01HNQRTMWWV5JCNT7YWR7XCNVD: test-repair

BasicConfig {
    output_rules: [
        "/tmp/*.txt",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HNQRV32MR7PFMHAKN1324G9B

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-02-03T15:49:16.677Zjob dependencies complete; ready to run (waiting for 29 m 52 s)
22024-02-03T17:08:10.098Zjob assigned to worker 01HNQYZWGQHX9VXCNBGMYCSGCA (queued for 1 h 18 m 53 s)
32024-02-03T17:08:10.825Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-02-03T17:08:14.512Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-02-03T17:08:14.513Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-02-03T17:08:24.819Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-02-03T17:08:24.819Zdownloading input: /input/build/work/bins/crutest.gz
82024-02-03T17:08:41.543Zdownloaded input: /input/build/work/bins/crutest.gz
92024-02-03T17:08:41.544Zdownloading input: /input/build/work/bins/dsc.gz
102024-02-03T17:08:52.942Zdownloaded input: /input/build/work/bins/dsc.gz
112024-02-03T17:08:52.946Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-02-03T17:08:52.998Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-02-03T17:08:52.998Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-02-03T17:08:53.039Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-02-03T17:08:53.039Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-02-03T17:08:53.070Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-02-03T17:08:53.070Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-02-03T17:08:53.099Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-02-03T17:08:53.099Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-02-03T17:08:53.133Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-02-03T17:08:53.134Zdownloading input: /input/build/work/scripts/test_up.sh
222024-02-03T17:08:53.162Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-02-03T17:08:53.162Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-02-03T17:08:53.202Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-02-03T17:08:53.205Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-02-03T17:09:11.162Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-02-03T17:09:11.162Zstarting task 0: "setup"
282024-02-03T17:09:11.283Z++ uname -s
292024-02-03T17:09:11.287Z+ kern=SunOS
302024-02-03T17:09:11.287Z+ case "$kern" in
312024-02-03T17:09:11.287Z+ groupadd -g 12345 build
322024-02-03T17:09:11.290Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-02-03T17:09:13.440Z+ zfs create -o mountpoint=/work rpool/work
342024-02-03T17:09:13.805Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-02-03T17:09:13.810Z+ home_fs=zfs
362024-02-03T17:09:13.810Z+ [[ zfs == autofs ]]
372024-02-03T17:09:13.810Z+ mkdir -p /home/build
382024-02-03T17:09:13.813Z+ chown build:build /home/build /work
392024-02-03T17:09:14.822Z+ chmod 0700 /home/build /work
402024-02-03T17:09:14.830Zprocess exited: duration 3667 ms, exit code 0
 
412024-02-03T17:09:14.847Zstarting task 1: "authentication"
422024-02-03T17:09:14.880Zprocess exited: duration 33 ms, exit code 0
 
432024-02-03T17:09:14.901Zstarting task 2: "build"
442024-02-03T17:09:14.907Z+ banner cores
452024-02-03T17:09:14.989Z
462024-02-03T17:09:14.990Z #### #### ##### ###### ####
472024-02-03T17:09:14.990Z # # # # # # # #
482024-02-03T17:09:14.990Z # # # # # ##### ####
492024-02-03T17:09:14.990Z # # # ##### # #
502024-02-03T17:09:14.990Z # # # # # # # # #
512024-02-03T17:09:14.990Z #### #### # # ###### ####
522024-02-03T17:09:14.990Z
532024-02-03T17:09:14.993Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-02-03T17:09:15.002Z+ echo 'input bins dir contains:'
552024-02-03T17:09:15.002Zinput bins dir contains:
562024-02-03T17:09:15.005Z+ ls -ltr /input/build/work/bins
572024-02-03T17:09:15.005Ztotal 762218
582024-02-03T17:09:15.005Z-rw-r--r-- 1 root root 121487815 Feb 3 17:08 crucible-downstairs.gz
592024-02-03T17:09:15.005Z-rw-r--r-- 1 root root 91386138 Feb 3 17:08 crucible-hammer.gz
602024-02-03T17:09:15.005Z-rw-r--r-- 1 root root 105093135 Feb 3 17:08 crutest.gz
612024-02-03T17:09:15.005Z-rw-r--r-- 1 root root 72023915 Feb 3 17:08 dsc.gz
622024-02-03T17:09:15.005Z+ echo 'input script dir contains:'
632024-02-03T17:09:15.005Zinput script dir contains:
642024-02-03T17:09:15.005Z+ ls -ltr /input/build/work/scripts
652024-02-03T17:09:15.011Ztotal 38
662024-02-03T17:09:15.012Z-rw-r--r-- 1 root root 3056 Feb 3 17:08 perf-downstairs-tick.d
672024-02-03T17:09:15.012Z-rw-r--r-- 1 root root 939 Feb 3 17:08 test_ds.sh
682024-02-03T17:09:15.012Z-rw-r--r-- 1 root root 3746 Feb 3 17:08 test_live_repair.sh
692024-02-03T17:09:15.012Z-rw-r--r-- 1 root root 6416 Feb 3 17:08 test_repair.sh
702024-02-03T17:09:15.012Z-rw-r--r-- 1 root root 2969 Feb 3 17:08 test_replay.sh
712024-02-03T17:09:15.012Z-rw-r--r-- 1 root root 9465 Feb 3 17:08 test_up.sh
722024-02-03T17:09:15.012Z-rw-r--r-- 1 root root 2824 Feb 3 17:08 upstairs_info.d
732024-02-03T17:09:15.012Z+ pfexec chmod +x /input/build/work/scripts/perf-downstairs-tick.d /input/build/work/scripts/test_ds.sh /input/build/work/scripts/test_live_repair.sh /input/build/work/scripts/test_repair.sh /input/build/work/scripts/test_replay.sh /input/build/work/scripts/test_up.sh /input/build/work/scripts/upstairs_info.d
742024-02-03T17:09:15.015Z+ echo ' chmod input script dir contains:'
752024-02-03T17:09:15.015Z chmod input script dir contains:
762024-02-03T17:09:15.015Z+ ls -ltr /input/build/work/scripts
772024-02-03T17:09:15.015Ztotal 38
782024-02-03T17:09:15.015Z-rwxr-xr-x 1 root root 3056 Feb 3 17:08 perf-downstairs-tick.d
792024-02-03T17:09:15.015Z-rwxr-xr-x 1 root root 939 Feb 3 17:08 test_ds.sh
802024-02-03T17:09:15.015Z-rwxr-xr-x 1 root root 3746 Feb 3 17:08 test_live_repair.sh
812024-02-03T17:09:15.015Z-rwxr-xr-x 1 root root 6416 Feb 3 17:08 test_repair.sh
822024-02-03T17:09:15.015Z-rwxr-xr-x 1 root root 2969 Feb 3 17:08 test_replay.sh
832024-02-03T17:09:15.015Z-rwxr-xr-x 1 root root 9465 Feb 3 17:08 test_up.sh
842024-02-03T17:09:15.015Z-rwxr-xr-x 1 root root 2824 Feb 3 17:08 upstairs_info.d
852024-02-03T17:09:15.015Z+ banner unpack
862024-02-03T17:09:15.019Z
872024-02-03T17:09:15.019Z # # # # ##### ## #### # #
882024-02-03T17:09:15.019Z # # ## # # # # # # # # #
892024-02-03T17:09:15.019Z # # # # # # # # # # ####
902024-02-03T17:09:15.019Z # # # # # ##### ###### # # #
912024-02-03T17:09:15.019Z # # # ## # # # # # # #
922024-02-03T17:09:15.019Z #### # # # # # #### # #
932024-02-03T17:09:15.019Z
942024-02-03T17:09:15.019Z+ mkdir -p /var/tmp/bins
952024-02-03T17:09:15.019Z+ for t in "$input/bins/"*.gz
962024-02-03T17:09:15.022Z++ basename /input/build/work/bins/crucible-downstairs.gz
972024-02-03T17:09:15.022Z+ b=crucible-downstairs.gz
982024-02-03T17:09:15.022Z+ b=crucible-downstairs
992024-02-03T17:09:15.022Z+ gunzip
1002024-02-03T17:09:19.427Z+ chmod +x /var/tmp/bins/crucible-downstairs
1012024-02-03T17:09:19.431Z+ for t in "$input/bins/"*.gz
1022024-02-03T17:09:19.431Z++ basename /input/build/work/bins/crucible-hammer.gz
1032024-02-03T17:09:19.434Z+ b=crucible-hammer.gz
1042024-02-03T17:09:19.434Z+ b=crucible-hammer
1052024-02-03T17:09:19.435Z+ gunzip
1062024-02-03T17:09:22.573Z+ chmod +x /var/tmp/bins/crucible-hammer
1072024-02-03T17:09:22.584Z+ for t in "$input/bins/"*.gz
1082024-02-03T17:09:22.584Z++ basename /input/build/work/bins/crutest.gz
1092024-02-03T17:09:22.589Z+ b=crutest.gz
1102024-02-03T17:09:22.589Z+ b=crutest
1112024-02-03T17:09:22.589Z+ gunzip
1122024-02-03T17:09:26.267Z+ chmod +x /var/tmp/bins/crutest
1132024-02-03T17:09:26.270Z+ for t in "$input/bins/"*.gz
1142024-02-03T17:09:26.270Z++ basename /input/build/work/bins/dsc.gz
1152024-02-03T17:09:26.270Z+ b=dsc.gz
1162024-02-03T17:09:26.270Z+ b=dsc
1172024-02-03T17:09:26.270Z+ gunzip
1182024-02-03T17:09:28.779Z+ chmod +x /var/tmp/bins/dsc
1192024-02-03T17:09:28.790Z+ export BINDIR=/var/tmp/bins
1202024-02-03T17:09:28.791Z+ BINDIR=/var/tmp/bins
1212024-02-03T17:09:28.791Z+ banner repair
1222024-02-03T17:09:28.791Z
1232024-02-03T17:09:28.791Z ##### ###### ##### ## # #####
1242024-02-03T17:09:28.791Z # # # # # # # # # #
1252024-02-03T17:09:28.791Z # # ##### # # # # # # #
1262024-02-03T17:09:28.791Z ##### # ##### ###### # #####
1272024-02-03T17:09:28.791Z # # # # # # # # #
1282024-02-03T17:09:28.791Z # # ###### # # # # # #
1292024-02-03T17:09:28.791Z
1302024-02-03T17:09:28.791Z+ ptime -m bash /input/build/work/scripts/test_repair.sh -N
1312024-02-03T17:09:28.927ZTurn off color for downstairs dump
1322024-02-03T17:09:28.973ZCreating region directory at: /var/tmp/test_repair
1332024-02-03T17:09:28.977ZCreating dsc directory at: /tmp/dsc
1342024-02-03T17:09:29.558ZDownstairs region 0 created at /var/tmp/test_repair/8810 in 0.58412236
1352024-02-03T17:09:29.626ZDownstairs region 1 created at /var/tmp/test_repair/8820 in 0.062061504
1362024-02-03T17:09:29.688ZDownstairs region 2 created at /var/tmp/test_repair/8830 in 0.06394132
1372024-02-03T17:09:29.688ZRegion set with 3 regions was created
1382024-02-03T17:09:29.727Z/var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1131 1132 1133
1392024-02-03T17:09:29.727ZDump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
1402024-02-03T17:09:30.183Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.288353482Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135}
1412024-02-03T17:09:30.183Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.289670773Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135}
1422024-02-03T17:09:30.183Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.289702313Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135}
1432024-02-03T17:09:30.183Z{"msg":"Crucible 8a47c22d-3acf-4f00-9e92-6f7aac716ba6 has session id: a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.289723772Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1442024-02-03T17:09:30.183Z{"msg":"Upstairs opts: Upstairs UUID: 8a47c22d-3acf-4f00-9e92-6f7aac716ba6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.289782657Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1452024-02-03T17:09:30.183Z{"msg":"Crucible stats registered with UUID: 8a47c22d-3acf-4f00-9e92-6f7aac716ba6","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.289807714Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1462024-02-03T17:09:30.183ZCrucible runtime is spawned
1472024-02-03T17:09:30.183Z{"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.290568456Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135}
1482024-02-03T17:09:30.184Z{"msg":"8a47c22d-3acf-4f00-9e92-6f7aac716ba6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.291215677Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1492024-02-03T17:09:30.184Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.291294209Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1502024-02-03T17:09:30.184Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.291323659Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1512024-02-03T17:09:30.184Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.291392814Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1522024-02-03T17:09:30.184Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.291421608Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1532024-02-03T17:09:30.184Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.291456168Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1542024-02-03T17:09:30.184Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.291516254Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1552024-02-03T17:09:30.184Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.291542662Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"":"io task","client":"1","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1562024-02-03T17:09:30.184Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.291566227Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"":"io task","client":"0","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1572024-02-03T17:09:30.184Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.291673671Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"":"io task","client":"2","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1582024-02-03T17:09:30.184Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.292456973Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"":"io task","client":"2","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1592024-02-03T17:09:30.184Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.292518695Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"":"io task","client":"0","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1602024-02-03T17:09:30.184Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.292562443Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"":"io task","client":"1","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1612024-02-03T17:09:30.184Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.296043036Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1622024-02-03T17:09:30.184Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.296184746Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1632024-02-03T17:09:30.184Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.296212029Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1642024-02-03T17:09:30.184Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.296234088Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1652024-02-03T17:09:30.184Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.296329794Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1662024-02-03T17:09:30.184Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.296378632Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1672024-02-03T17:09:30.184Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.298874988Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1682024-02-03T17:09:30.184Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.2989504Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1692024-02-03T17:09:30.184Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.299113546Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1702024-02-03T17:09:30.184Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.300911962Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1712024-02-03T17:09:30.185Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.300969417Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1722024-02-03T17:09:30.185Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.301055451Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1732024-02-03T17:09:30.185Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.301100238Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1742024-02-03T17:09:30.185Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.301126993Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1752024-02-03T17:09:30.185Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.301178154Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1762024-02-03T17:09:30.185Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.301205726Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1772024-02-03T17:09:30.185Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.30122954Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1782024-02-03T17:09:30.185Z{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.301426984Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1792024-02-03T17:09:30.185Z{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.301587457Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1802024-02-03T17:09:30.185Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.301852235Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1812024-02-03T17:09:30.185Z{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.301935842Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1822024-02-03T17:09:30.185Z{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.301990722Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1832024-02-03T17:09:30.185Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.302017843Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1842024-02-03T17:09:30.185Z{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.302043484Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1852024-02-03T17:09:30.185Z{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.302091518Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1862024-02-03T17:09:30.185Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.30211804Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1872024-02-03T17:09:30.185Z{"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.302173462Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1882024-02-03T17:09:30.185Z{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.302198384Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1892024-02-03T17:09:30.185Z{"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.302244726Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1902024-02-03T17:09:30.185Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.302268837Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1912024-02-03T17:09:30.185Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.302290992Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1922024-02-03T17:09:30.185Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.302338275Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1932024-02-03T17:09:30.185Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.302362658Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1942024-02-03T17:09:30.185Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
1952024-02-03T17:09:30.186Z,"time":"2024-02-03T17:09:29.302446087Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1962024-02-03T17:09:30.186Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.302626552Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1972024-02-03T17:09:30.186Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.302654818Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1982024-02-03T17:09:30.186Z{"msg":"8a47c22d-3acf-4f00-9e92-6f7aac716ba6 is now active with session: a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.302676599Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
1992024-02-03T17:09:30.186Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.302696762Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135,"session_id":"a7ab960e-86b2-4ea7-b535-bc0a80e1f9cc"}
2002024-02-03T17:09:30.186Z{"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:29.30277716Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135}
2012024-02-03T17:09:30.186ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2022024-02-03T17:09:30.186ZFill test
2032024-02-03T17:09:30.186ZRead and Verify all blocks (0..600 range:false)
2042024-02-03T17:09:30.186ZWrote out file "/tmp/test_repair_verify.data"
2052024-02-03T17:09:30.186ZCLIENT: Tests done. All submitted work has been ACK'd
2062024-02-03T17:09:30.186Z----------------------------------------------------------------
2072024-02-03T17:09:30.186Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:6
2082024-02-03T17:09:30.186ZStates: Active Active Active
2092024-02-03T17:09:30.186ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
2102024-02-03T17:09:30.186Z 8 Acked 1007 Read 100 Done Done Done false
2112024-02-03T17:09:30.186Z 9 Acked 1008 Read 100 Done Done Done false
2122024-02-03T17:09:30.186Z 10 Acked 1009 Read 100 Done Done Done false
2132024-02-03T17:09:30.186Z 11 Acked 1010 Read 100 Done Done Done false
2142024-02-03T17:09:30.186Z 12 Acked 1011 Read 100 Done Done Done false
2152024-02-03T17:09:30.186Z 13 Acked 1012 Read 100 Done Done Done false
2162024-02-03T17:09:30.186Z STATES DS:0 DS:1 DS:2 TOTAL
2172024-02-03T17:09:30.186Z New 0 0 0 0
2182024-02-03T17:09:30.186Z Sent 0 0 0 0
2192024-02-03T17:09:30.187Z Done 6 6 6 18
2202024-02-03T17:09:30.187Z Skipped 0 0 0 0
2212024-02-03T17:09:30.187Z Error 0 0 0 0
2222024-02-03T17:09:30.187ZLast Flush: 1006 1006 1006
2232024-02-03T17:09:30.187ZDownstairs last five completed: 1006 1005 1004 1003 1002
2242024-02-03T17:09:30.187ZUpstairs last five completed: 13 12 11 10 9
2252024-02-03T17:09:30.187ZCLIENT: Up:0 ds:6 act:3
2262024-02-03T17:09:33.962Z----------------------------------------------------------------
2272024-02-03T17:09:33.962Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0
2282024-02-03T17:09:33.962ZDownstairs last five completed: 1013 1012 1011 1010 1009
2292024-02-03T17:09:33.962ZUpstairs last five completed: 14 13 12 11 10
2302024-02-03T17:09:33.962ZCLIENT: Up:0 ds:0 act:3
2312024-02-03T17:09:33.962ZCLIENT: All crucible jobs finished, exiting program
2322024-02-03T17:09:33.966Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:09:33.500036541Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1135}
2332024-02-03T17:09:33.973Z
2342024-02-03T17:09:33.973ZBegin loop 1
2352024-02-03T17:09:33.973ZDownstairs to restart: 2
2362024-02-03T17:09:34.012Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.538438752Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137}
2372024-02-03T17:09:34.012Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.538921271Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137}
2382024-02-03T17:09:34.012Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.538957085Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137}
2392024-02-03T17:09:34.019Z{"msg":"Crucible bb6b740f-da17-4600-afa0-3f63539944da has session id: de5a188a-0482-4972-b97f-165e4e937238","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.539036803Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2402024-02-03T17:09:34.019Z{"msg"Crucible runtime is spawned
2412024-02-03T17:09:34.019Z:"Upstairs opts: Upstairs UUID: bb6b740f-da17-4600-afa0-3f63539944da, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.539275776Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2422024-02-03T17:09:34.019Z{"msg":"Crucible stats registered with UUID: bb6b740f-da17-4600-afa0-3f63539944da","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.539354337Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2432024-02-03T17:09:34.019Z{"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.539380131Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137}
2442024-02-03T17:09:34.020Z{"msg":"bb6b740f-da17-4600-afa0-3f63539944da active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.539636621Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2452024-02-03T17:09:34.020Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.539700613Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2462024-02-03T17:09:34.020Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.53972715Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2472024-02-03T17:09:34.020Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.539794364Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2482024-02-03T17:09:34.020Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.539976773Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2492024-02-03T17:09:34.020Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.540032233Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2502024-02-03T17:09:34.020Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.540100258Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2512024-02-03T17:09:34.020Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.540148107Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"":"io task","client":"2","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2522024-02-03T17:09:34.020Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.540193468Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"":"io task","client":"0","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2532024-02-03T17:09:34.020Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.540276406Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"":"io task","client":"1","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2542024-02-03T17:09:34.020Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.540324962Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"":"io task","client":"2","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2552024-02-03T17:09:34.020Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.540366348Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"":"io task","client":"0","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2562024-02-03T17:09:34.021Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.540620151Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"":"io task","client":"1","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2572024-02-03T17:09:34.021Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.541609152Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2582024-02-03T17:09:34.021Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.541774061Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2592024-02-03T17:09:34.021Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.541861143Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2602024-02-03T17:09:34.021Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.541903462Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2612024-02-03T17:09:34.021Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.542039457Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2622024-02-03T17:09:34.021Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.542211232Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2632024-02-03T17:09:34.021Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.542700807Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2642024-02-03T17:09:34.021Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.543036145Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2652024-02-03T17:09:34.021Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.543576265Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2662024-02-03T17:09:34.021Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.543866383Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2672024-02-03T17:09:34.021Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.543940061Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2682024-02-03T17:09:34.021Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.543967306Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2692024-02-03T17:09:34.022Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.543988732Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2702024-02-03T17:09:34.022Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.544011217Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2712024-02-03T17:09:34.022Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.544059129Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2722024-02-03T17:09:34.022Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.544486999Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2732024-02-03T17:09:34.022Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.544553113Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2742024-02-03T17:09:34.022Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.544580401Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2752024-02-03T17:09:34.022Z{"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.544604256Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2762024-02-03T17:09:34.022Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.544653528Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2772024-02-03T17:09:34.022Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.544679402Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2782024-02-03T17:09:34.022Z{"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.544702641Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2792024-02-03T17:09:34.022Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.544751603Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2802024-02-03T17:09:34.022Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.544777184Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2812024-02-03T17:09:34.022ZWait for a query_work_queue command to finish before sending IO
2822024-02-03T17:09:34.023Z{"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.544829207Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2832024-02-03T17:09:34.023Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.544855469Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2842024-02-03T17:09:34.023Z{"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.544877562Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2852024-02-03T17:09:34.023Z{"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.544925532Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2862024-02-03T17:09:34.023Z{"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.544949372Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2872024-02-03T17:09:34.023Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.544995947Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2882024-02-03T17:09:34.023Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.545020527Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2892024-02-03T17:09:34.023Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.54506551Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2902024-02-03T17:09:34.023Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.545089833Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2912024-02-03T17:09:34.023Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.54513776Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2922024-02-03T17:09:34.023Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.545162595Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2932024-02-03T17:09:34.023Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.545183352Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2942024-02-03T17:09:34.023Z{"msg":"bb6b740f-da17-4600-afa0-3f63539944da is now active with session: de5a188a-0482-4972-b97f-165e4e937238","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.545229574Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2952024-02-03T17:09:34.024Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.545251635Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137,"session_id":"de5a188a-0482-4972-b97f-165e4e937238"}
2962024-02-03T17:09:34.024Z{"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:33.5452736Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137}
2972024-02-03T17:09:34.024ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2982024-02-03T17:09:34.024ZLoading write count information from file "/tmp/test_repair_verify.data"
2992024-02-03T17:09:34.024ZRun Repair workload
3002024-02-03T17:09:34.024Z01/30 Flush
3012024-02-03T17:09:34.024Z02/30 Write block 568 len 36864 data: 2 2 2 2 2 2 2 2 2
3022024-02-03T17:09:34.024Z03/30 Write block 510 len 32768 data: 2 2 2 2 2 2 2 2
3032024-02-03T17:09:34.024Z04/30 Flush
3042024-02-03T17:09:34.032Z05/30 Write block 19 len 32768 data: 2 2 2 2 2 2 2 2
3052024-02-03T17:09:34.032Z06/30 Write block 187 len 24576 data: 2 2 2 2 2 2
3062024-02-03T17:09:34.032Z07/30 Read block 586 len 12288
3072024-02-03T17:09:34.032Z08/30 Read block 396 len 4096
3082024-02-03T17:09:34.036Z09/30 Read block 213 len 4096
3092024-02-03T17:09:34.036Z10/30 Read block 463 len 40960
3102024-02-03T17:09:34.036Z11/30 Read block 301 len 12288
3112024-02-03T17:09:34.036Z12/30 Read block 22 len 4096
3122024-02-03T17:09:34.036Z13/30 Read block 37 len 24576
3132024-02-03T17:09:34.037Z14/30 Write block 231 len 8192 data: 2 2
3142024-02-03T17:09:34.042Z15/30 Write block 67 len 20480 data: 2 2 2 2 2
3152024-02-03T17:09:34.042Z16/30 Flush
3162024-02-03T17:09:34.082Z17/30 Write block 111 len 28672 data: 2 2 2 2 2 2 2
3172024-02-03T17:09:34.085Z18/30 Write block 490 len 40960 data: 2 2 2 2 2 2 2 2 2 2
3182024-02-03T17:09:34.085Z19/30 Flush
3192024-02-03T17:09:34.091Z20/30 Write block 387 len 40960 data: 2 2 2 2 2 2 2 2 2 2
3202024-02-03T17:09:34.094Z21/30 Read block 42 len 36864
3212024-02-03T17:09:34.094Z22/30 Read block 413 len 16384
3222024-02-03T17:09:34.097Z23/30 Read block 169 len 32768
3232024-02-03T17:09:34.097Z24/30 Flush
3242024-02-03T17:09:34.154Z25/30 Write block 276 len 20480 data: 2 2 2 2 2
3252024-02-03T17:09:34.158Z26/30 Read block 135 len 36864
3262024-02-03T17:09:34.158Z27/30 Write block 312 len 16384 data: 2 2 2 2
3272024-02-03T17:09:34.158Z28/30 Write block 496 len 12288 data: 3 3 3
3282024-02-03T17:09:34.162Z29/30 Write block 491 len 24576 data: 3 3 3 3 3 4
3292024-02-03T17:09:34.162Z30/30 Write block 306 len 4096 data: 2
3302024-02-03T17:09:34.162Z----------------------------------------------------------------
3312024-02-03T17:09:34.162Z Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:30
3322024-02-03T17:09:34.162ZStates: Active Active Active
3332024-02-03T17:09:34.162ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
3342024-02-03T17:09:34.162Z 1 Acked 1000 Flush 0 Done Done Sent false
3352024-02-03T17:09:34.162Z 2 Acked 1001 Write 9 Done Done Sent false
3362024-02-03T17:09:34.162Z 3 Acked 1002 Write 8 Done Done Sent false
3372024-02-03T17:09:34.162Z 4 Acked 1003 Flush 0 Done Done Sent false
3382024-02-03T17:09:34.162Z 5 Acked 1004 Write 8 Done Done Sent false
3392024-02-03T17:09:34.162Z 6 Acked 1005 Write 6 Done Done Sent false
3402024-02-03T17:09:34.163Z 7 Acked 1006 Read 3 Done Done Sent false
3412024-02-03T17:09:34.163Z 8 Acked 1007 Read 1 Done Done Sent false
3422024-02-03T17:09:34.163Z 9 Acked 1008 Read 1 Done Done Sent false
3432024-02-03T17:09:34.163Z 10 Acked 1009 Read 10 Done Done Sent false
3442024-02-03T17:09:34.163Z 11 Acked 1010 Read 3 Done Done Sent false
3452024-02-03T17:09:34.163Z 12 Acked 1011 Read 1 Done Done Sent false
3462024-02-03T17:09:34.163Z 13 Acked 1012 Read 6 Done Done Sent false
3472024-02-03T17:09:34.163Z 14 Acked 1013 Write 2 Done Done Sent false
3482024-02-03T17:09:34.163Z 15 Acked 1014 Write 5 Done Done Sent false
3492024-02-03T17:09:34.163Z 16 Acked 1015 Flush 0 Done Done Sent false
3502024-02-03T17:09:34.163Z 17 Acked 1016 Write 7 Done Done Sent false
3512024-02-03T17:09:34.163Z 18 Acked 1017 Write 10 Done Done Sent false
3522024-02-03T17:09:34.163Z 19 Acked 1018 Flush 0 Done Done Sent false
3532024-02-03T17:09:34.164Z 20 Acked 1019 Write 10 Done Done Sent false
3542024-02-03T17:09:34.164Z 21 Acked 1020 Read 9 Done Done Sent false
3552024-02-03T17:09:34.164Z 22 Acked 1021 Read 4 Done Done Sent false
3562024-02-03T17:09:34.164Z 23 Acked 1022 Read 8 Done Done Sent false
3572024-02-03T17:09:34.164Z 24 Acked 1023 Flush 0 Done Done Sent false
3582024-02-03T17:09:34.164Z 25 Acked 1024 Write 5 Done Done Sent false
3592024-02-03T17:09:34.164Z 26 Acked 1025 Read 9 Done Done Sent false
3602024-02-03T17:09:34.164Z 27 Acked 1026 Write 4 Done Done Sent false
3612024-02-03T17:09:34.164Z 28 Acked 1027 Write 3 Done Done Sent false
3622024-02-03T17:09:34.164Z 29 Acked 1028 Write 6 Done Done Sent false
3632024-02-03T17:09:34.164Z 30 Acked 1029 Write 1 Sent Sent Sent false
3642024-02-03T17:09:34.164Z STATES DS:0 DS:1 DS:2 TOTAL
3652024-02-03T17:09:34.164Z New 0 0 0 0
3662024-02-03T17:09:34.165Z Sent 1 1 30 32
3672024-02-03T17:09:34.165Z Done 29 29 0 58
3682024-02-03T17:09:34.165Z Skipped 0 0 0 0
3692024-02-03T17:09:34.165Z Error 0 0 0 0
3702024-02-03T17:09:34.165ZLast Flush: 1023 1023 0
3712024-02-03T17:09:34.165ZDownstairs last five completed:
3722024-02-03T17:09:34.165ZUpstairs last five completed: 30 29 28 27 26
3732024-02-03T17:09:34.169Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:09:33.699261837Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1137}
3742024-02-03T17:09:34.169ZWrote out file "/tmp/test_repair_verify.data"
3752024-02-03T17:09:34.173Z
3762024-02-03T17:09:34.188ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
3772024-02-03T17:09:34.201ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3782024-02-03T17:09:34.205Z 0 000-019 2 2 1 4 4 1 F F F <---
3792024-02-03T17:09:34.205Z 1 020-039 2 2 1 4 4 1 F F F <---
3802024-02-03T17:09:34.205Z 2 040-059 1 1 1 1 1 1 F F F
3812024-02-03T17:09:34.205Z 3 060-079 2 2 1 4 4 1 F F F <---
3822024-02-03T17:09:34.205Z 4 080-099 1 1 1 1 1 1 F F F
3832024-02-03T17:09:34.205Z 5 100-119 2 2 1 5 5 1 F F F <---
3842024-02-03T17:09:34.205Z 6 120-139 1 1 1 1 1 1 F F F
3852024-02-03T17:09:34.205Z 7 140-159 1 1 1 1 1 1 F F F
3862024-02-03T17:09:34.205Z 8 160-179 1 1 1 1 1 1 F F F
3872024-02-03T17:09:34.206Z 9 180-199 2 2 1 4 4 1 F F F <---
3882024-02-03T17:09:34.206Z 10 200-219 1 1 1 1 1 1 F F F
3892024-02-03T17:09:34.206Z 11 220-239 2 2 1 4 4 1 F F F <---
3902024-02-03T17:09:34.206Z 12 240-259 1 1 1 1 1 1 F F F
3912024-02-03T17:09:34.206Z 13 260-279 1 1 1 1 1 1 T T F <---
3922024-02-03T17:09:34.206Z 14 280-299 1 1 1 1 1 1 T T F <---
3932024-02-03T17:09:34.206Z 15 300-319 1 1 1 1 1 1 T T F <---
3942024-02-03T17:09:34.206Z 16 320-339 1 1 1 1 1 1 F F F
3952024-02-03T17:09:34.206Z 17 340-359 1 1 1 1 1 1 F F F
3962024-02-03T17:09:34.206Z 18 360-379 1 1 1 1 1 1 F F F
3972024-02-03T17:09:34.206Z 19 380-399 2 2 1 6 6 1 F F F <---
3982024-02-03T17:09:34.206Z 20 400-419 1 1 1 1 1 1 F F F
3992024-02-03T17:09:34.206Z 21 420-439 1 1 1 1 1 1 F F F
4002024-02-03T17:09:34.207Z 22 440-459 1 1 1 1 1 1 F F F
4012024-02-03T17:09:34.207Z 23 460-479 1 1 1 1 1 1 F F F
4022024-02-03T17:09:34.207Z 24 480-499 2 2 1 5 5 1 T T F <---
4032024-02-03T17:09:34.207Z 25 500-519 2 2 1 3 3 1 F F F <---
4042024-02-03T17:09:34.207Z 26 520-539 1 1 1 1 1 1 F F F
4052024-02-03T17:09:34.207Z 27 540-559 1 1 1 1 1 1 F F F
4062024-02-03T17:09:34.207Z 28 560-579 2 2 1 3 3 1 F F F <---
4072024-02-03T17:09:34.207Z 29 580-599 1 1 1 1 1 1 F F F
4082024-02-03T17:09:34.207ZMax gen: 2, Max flush: 6
4092024-02-03T17:09:34.207ZError: Difference in extent metadata found!
4102024-02-03T17:09:34.213ZA Difference in extent metadata is expected here
4112024-02-03T17:09:34.213ZOn loop 1
4122024-02-03T17:09:34.213Z
4132024-02-03T17:09:34.214ZVerifying data now
4142024-02-03T17:09:40.374ZNo change after verify
4152024-02-03T17:09:40.374ZLoop: 1 Downstairs dump after verify (and repair):
4162024-02-03T17:09:40.397ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4172024-02-03T17:09:40.397Z 0 000-019 2 2 2 4 4 4 F F F
4182024-02-03T17:09:40.398Z 1 020-039 2 2 2 4 4 4 F F F
4192024-02-03T17:09:40.398Z 2 040-059 1 1 1 1 1 1 F F F
4202024-02-03T17:09:40.398Z 3 060-079 2 2 2 4 4 4 F F F
4212024-02-03T17:09:40.401Z 4 080-099 1 1 1 1 1 1 F F F
4222024-02-03T17:09:40.401Z 5 100-119 2 2 2 5 5 5 F F F
4232024-02-03T17:09:40.401Z 6 120-139 1 1 1 1 1 1 F F F
4242024-02-03T17:09:40.401Z 7 140-159 1 1 1 1 1 1 F F F
4252024-02-03T17:09:40.401Z 8 160-179 1 1 1 1 1 1 F F F
4262024-02-03T17:09:40.401Z 9 180-199 2 2 2 4 4 4 F F F
4272024-02-03T17:09:40.401Z 10 200-219 1 1 1 1 1 1 F F F
4282024-02-03T17:09:40.401Z 11 220-239 2 2 2 4 4 4 F F F
4292024-02-03T17:09:40.401Z 12 240-259 1 1 1 1 1 1 F F F
4302024-02-03T17:09:40.401Z 13 260-279 3 3 3 7 7 7 F F F
4312024-02-03T17:09:40.401Z 14 280-299 3 3 3 7 7 7 F F F
4322024-02-03T17:09:40.401Z 15 300-319 3 3 3 7 7 7 F F F
4332024-02-03T17:09:40.401Z 16 320-339 1 1 1 1 1 1 F F F
4342024-02-03T17:09:40.401Z 17 340-359 1 1 1 1 1 1 F F F
4352024-02-03T17:09:40.401Z 18 360-379 1 1 1 1 1 1 F F F
4362024-02-03T17:09:40.401Z 19 380-399 2 2 2 6 6 6 F F F
4372024-02-03T17:09:40.401Z 20 400-419 1 1 1 1 1 1 F F F
4382024-02-03T17:09:40.401Z 21 420-439 1 1 1 1 1 1 F F F
4392024-02-03T17:09:40.402Z 22 440-459 1 1 1 1 1 1 F F F
4402024-02-03T17:09:40.402Z 23 460-479 1 1 1 1 1 1 F F F
4412024-02-03T17:09:40.402Z 24 480-499 3 3 3 7 7 7 F F F
4422024-02-03T17:09:40.402Z 25 500-519 2 2 2 3 3 3 F F F
4432024-02-03T17:09:40.402Z 26 520-539 1 1 1 1 1 1 F F F
4442024-02-03T17:09:40.402Z 27 540-559 1 1 1 1 1 1 F F F
4452024-02-03T17:09:40.402Z 28 560-579 2 2 2 3 3 3 F F F
4462024-02-03T17:09:40.402Z 29 580-599 1 1 1 1 1 1 F F F
4472024-02-03T17:09:40.402ZMax gen: 3, Max flush: 7
4482024-02-03T17:09:40.405Z
4492024-02-03T17:09:40.405ZBegin loop 2
4502024-02-03T17:09:40.405ZDownstairs to restart: 0
4512024-02-03T17:09:40.432Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.968912672Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145}
4522024-02-03T17:09:40.432Z{Crucible runtime is spawned
4532024-02-03T17:09:40.432Z"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.9696244Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145}
4542024-02-03T17:09:40.432Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.969674058Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145}
4552024-02-03T17:09:40.437Z{"msg":"Crucible b948790c-ab6c-47bb-84b6-172f6f9e10d6 has session id: 11cb94c6-b16d-46da-8a6f-692c9ae105f9","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.96971423Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4562024-02-03T17:09:40.437Z{"msg":"Upstairs opts: Upstairs UUID: b948790c-ab6c-47bb-84b6-172f6f9e10d6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.969807707Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4572024-02-03T17:09:40.437Z{"msg":"Crucible stats registered with UUID: b948790c-ab6c-47bb-84b6-172f6f9e10d6","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.969886617Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4582024-02-03T17:09:40.437Z{"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.969967474Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145}
4592024-02-03T17:09:40.437Z{"msg":"b948790c-ab6c-47bb-84b6-172f6f9e10d6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.97003935Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4602024-02-03T17:09:40.437Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.970091028Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4612024-02-03T17:09:40.437Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.970119798Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4622024-02-03T17:09:40.437Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.970158785Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4632024-02-03T17:09:40.438Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.97018942Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4642024-02-03T17:09:40.438Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.97025944Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4652024-02-03T17:09:40.438Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.970334964Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4662024-02-03T17:09:40.438Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.970446861Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"":"io task","client":"0","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4672024-02-03T17:09:40.438Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.970495096Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"":"io task","client":"2","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4682024-02-03T17:09:40.438Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.970631453Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"":"io task","client":"1","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4692024-02-03T17:09:40.438Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.970727313Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"":"io task","client":"0","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4702024-02-03T17:09:40.438Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.970829712Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"":"io task","client":"2","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4712024-02-03T17:09:40.438Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.970879531Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"":"io task","client":"1","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4722024-02-03T17:09:40.438Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.971963695Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4732024-02-03T17:09:40.438Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.972143893Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4742024-02-03T17:09:40.438Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.9721907Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4752024-02-03T17:09:40.438Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.97234461Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4762024-02-03T17:09:40.438Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.973079688Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4772024-02-03T17:09:40.438Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.973391722Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4782024-02-03T17:09:40.438Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.973474213Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4792024-02-03T17:09:40.438Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.973501761Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4802024-02-03T17:09:40.438Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.973882604Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4812024-02-03T17:09:40.438Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.973982262Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4822024-02-03T17:09:40.444Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.974101518Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4832024-02-03T17:09:40.444Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.974578122Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4842024-02-03T17:09:40.444Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.974644435Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4852024-02-03T17:09:40.444Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.974718482Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4862024-02-03T17:09:40.444Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.974769127Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4872024-02-03T17:09:40.444Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.975589631Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4882024-02-03T17:09:40.444Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.975651078Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4892024-02-03T17:09:40.444Z{"msg":"[0]R flush_numbers[0..12]: [4, 4, 1, 4, 1, 5, 1, 1, 1, 4, 1, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.975677547Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4902024-02-03T17:09:40.444Z{"msg":"[0]R generation[0..12]: [2, 2, 1, 2, 1, 2, 1, 1, 1, 2, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.975700253Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4912024-02-03T17:09:40.444Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.975749808Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4922024-02-03T17:09:40.444Z{"msg":"[1]R flush_numbers[0..12]: [4, 4, 1, 4, 1, 5, 1, 1, 1, 4, 1, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.97577521Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4932024-02-03T17:09:40.444Z{"msg":"[1]R generation[0..12]: [2, 2, 1, 2, 1, 2, 1, 1, 1, 2, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.975823124Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4942024-02-03T17:09:40.444Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.975849126Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4952024-02-03T17:09:40.444Z{"msg":"[2]R flush_numbers[0..12]: [4, 4, 1, 4, 1, 5, 1, 1, 1, 4, 1, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.975896685Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4962024-02-03T17:09:40.444Z{"msg":"[2]R generation[0..12]: [2, 2, 1, 2, 1, 2, 1, 1, 1, 2, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.975921634Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4972024-02-03T17:09:40.444Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.9759697Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
4982024-02-03T17:09:40.444Z{"msg":"Max found gen is 4","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
4992024-02-03T17:09:40.444Z,"time":"2024-02-03T17:09:39.975993788Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
5002024-02-03T17:09:40.444Z{"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.976041981Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
5012024-02-03T17:09:40.444Z{"msg":"Next flush: 8","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.976065357Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
5022024-02-03T17:09:40.444Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.97611439Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
5032024-02-03T17:09:40.444Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.976139177Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
5042024-02-03T17:09:40.444Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.976159071Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
5052024-02-03T17:09:40.445Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.976180613Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
5062024-02-03T17:09:40.445Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.97620195Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
5072024-02-03T17:09:40.445Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.976224826Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
5082024-02-03T17:09:40.445Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.976280129Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
5092024-02-03T17:09:40.445Z{"msg":"b948790c-ab6c-47bb-84b6-172f6f9e10d6 is now active with session: 11cb94c6-b16d-46da-8a6f-692c9ae105f9","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.976304923Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
5102024-02-03T17:09:40.445Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.976359753Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145,"session_id":"11cb94c6-b16d-46da-8a6f-692c9ae105f9"}
5112024-02-03T17:09:40.445Z{"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:39.976395095Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145}
5122024-02-03T17:09:40.445ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
5132024-02-03T17:09:40.445ZLoading write count information from file "/tmp/test_repair_verify.data"
5142024-02-03T17:09:40.445ZRun Repair workload
5152024-02-03T17:09:40.445Z01/30 Write block 499 len 16384 data: 3 2 2 2
5162024-02-03T17:09:40.449Z02/30 Flush
5172024-02-03T17:09:40.449Z03/30 Write block 436 len 32768 data: 2 2 2 2 2 2 2 2
5182024-02-03T17:09:40.449Z04/30 Read block 99 len 36864
5192024-02-03T17:09:40.452Z05/30 Read block 4 len 20480
5202024-02-03T17:09:40.452Z06/30 Read block 291 len 40960
5212024-02-03T17:09:40.455Z07/30 Write block 525 len 36864 data: 2 2 2 2 2 2 2 2 2
5222024-02-03T17:09:40.455Z08/30 Flush
5232024-02-03T17:09:40.489Z09/30 Write block 321 len 4096 data: 2
5242024-02-03T17:09:40.492Z10/30 Read block 390 len 28672
5252024-02-03T17:09:40.493Z11/30 Write block 9 len 24576 data: 2 2 2 2 2 2
5262024-02-03T17:09:40.496Z12/30 Read block 229 len 4096
5272024-02-03T17:09:40.496Z13/30 Write block 561 len 40960 data: 2 2 2 2 2 2 2 3 3 3
5282024-02-03T17:09:40.499Z14/30 Write block 508 len 24576 data: 2 2 3 3 3 3
5292024-02-03T17:09:40.499Z15/30 Write block 127 len 36864 data: 2 2 2 2 2 2 2 2 2
5302024-02-03T17:09:40.502Z16/30 Write block 130 len 4096 data: 3
5312024-02-03T17:09:40.502Z17/30 Read block 505 len 32768
5322024-02-03T17:09:40.505Z18/30 Read block 230 len 20480
5332024-02-03T17:09:40.505Z19/30 Write block 590 len 4096 data: 2
5342024-02-03T17:09:40.505Z20/30 Read block 415 len 36864
5352024-02-03T17:09:40.508Z21/30 Write block 470 len 4096 data: 2
5362024-02-03T17:09:40.508Z22/30 Flush
5372024-02-03T17:09:40.512Z23/30 Write block 74 len 40960 data: 2 2 2 2 2 2 2 2 2 2
5382024-02-03T17:09:40.515Z24/30 Read block 407 len 36864
5392024-02-03T17:09:40.518Z25/30 Read block 543 len 40960
5402024-02-03T17:09:40.518Z26/30 Read block 250 len 4096
5412024-02-03T17:09:40.521Z27/30 Read block 226 len 32768
5422024-02-03T17:09:40.522Z28/30 Write block 488 len 12288 data: 2 2 3
5432024-02-03T17:09:40.525Z29/30 Write block 252 len 28672 data: 2 2 2 2 2 2 2
5442024-02-03T17:09:40.525Z30/30 Write block 183 len 12288 data: 2 2 2
5452024-02-03T17:09:40.525Z----------------------------------------------------------------
5462024-02-03T17:09:40.525Z Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:28
5472024-02-03T17:09:40.525ZStates: Active Active Active
5482024-02-03T17:09:40.526ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
5492024-02-03T17:09:40.526Z 3 Acked 1002 Write 8 Done Done Done false
5502024-02-03T17:09:40.526Z 4 Acked 1003 Read 9 Sent Done Done false
5512024-02-03T17:09:40.526Z 5 Acked 1004 Read 5 Sent Done Done false
5522024-02-03T17:09:40.526Z 6 Acked 1005 Read 10 Sent Done Done false
5532024-02-03T17:09:40.526Z 7 Acked 1006 Write 9 Sent Done Done false
5542024-02-03T17:09:40.526Z 8 Acked 1007 Flush 0 Sent Done Done false
5552024-02-03T17:09:40.526Z 9 Acked 1008 Write 1 Sent Done Done false
5562024-02-03T17:09:40.526Z 10 Acked 1009 Read 7 Sent Done Done false
5572024-02-03T17:09:40.526Z 11 Acked 1010 Write 6 Sent Done Done false
5582024-02-03T17:09:40.526Z 12 Acked 1011 Read 1 Sent Done Done false
5592024-02-03T17:09:40.526Z 13 Acked 1012 Write 10 Sent Done Done false
5602024-02-03T17:09:40.526Z 14 Acked 1013 Write 6 Sent Done Done false
5612024-02-03T17:09:40.526Z 15 Acked 1014 Write 9 Sent Done Done false
5622024-02-03T17:09:40.526Z 16 Acked 1015 Write 1 Sent Done Done false
5632024-02-03T17:09:40.526Z 17 Acked 1016 Read 8 Sent Done Done false
5642024-02-03T17:09:40.526Z 18 Acked 1017 Read 5 Sent Done Done false
5652024-02-03T17:09:40.526Z 19 Acked 1018 Write 1 Sent Done Done false
5662024-02-03T17:09:40.526Z 20 Acked 1019 Read 9 Sent Done Done false
5672024-02-03T17:09:40.526Z 21 Acked 1020 Write 1 Sent Done Done false
5682024-02-03T17:09:40.526Z 22 Acked 1021 Flush 0 Sent Done Done false
5692024-02-03T17:09:40.526Z 23 Acked 1022 Write 10 Sent Done Done false
5702024-02-03T17:09:40.526Z 24 Acked 1023 Read 9 Sent Done Done false
5712024-02-03T17:09:40.527Z 25 Acked 1024 Read 10 Sent Done Done false
5722024-02-03T17:09:40.527Z 26 Acked 1025 Read 1 Sent Done Done false
5732024-02-03T17:09:40.527Z 27 Acked 1026 Read 8 Sent Done Done false
5742024-02-03T17:09:40.527Z 28 Acked 1027 Write 3 Sent Done Done false
5752024-02-03T17:09:40.527Z 29 Acked 1028 Write 7 Sent Sent Sent false
5762024-02-03T17:09:40.527Z 30 Acked 1029 Write 3 Sent Sent Sent false
5772024-02-03T17:09:40.527Z STATES DS:0 DS:1 DS:2 TOTAL
5782024-02-03T17:09:40.527Z New 0 0 0 0
5792024-02-03T17:09:40.527Z Sent 27 2 2 31
5802024-02-03T17:09:40.527Z Done 1 26 26 53
5812024-02-03T17:09:40.527Z Skipped 0 0 0 0
5822024-02-03T17:09:40.527Z Error 0 0 0 0
5832024-02-03T17:09:40.527ZLast Flush: 1001 1021 1021
5842024-02-03T17:09:40.527ZDownstairs last five completed: 1001 1000
5852024-02-03T17:09:40.527ZUpstairs last five completed: 30 29 28 27 26
5862024-02-03T17:09:40.527Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:09:40.062467495Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1145}
5872024-02-03T17:09:40.531ZWrote out file "/tmp/test_repair_verify.data"
5882024-02-03T17:09:40.534Z
5892024-02-03T17:09:40.541ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
5902024-02-03T17:09:40.565ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5912024-02-03T17:09:40.565Z 0 000-019 2 4 4 4 10 10 F F F <---
5922024-02-03T17:09:40.565Z 1 020-039 2 2 2 4 4 4 F F F
5932024-02-03T17:09:40.565Z 2 040-059 1 1 1 1 1 1 F F F
5942024-02-03T17:09:40.565Z 3 060-079 2 2 2 4 4 4 F T T <---
5952024-02-03T17:09:40.569Z 4 080-099 1 1 1 1 1 1 F T T <---
5962024-02-03T17:09:40.569Z 5 100-119 2 2 2 5 5 5 F F F
5972024-02-03T17:09:40.569Z 6 120-139 1 4 4 1 10 10 F F F <---
5982024-02-03T17:09:40.569Z 7 140-159 1 1 1 1 1 1 F F F
5992024-02-03T17:09:40.569Z 8 160-179 1 1 1 1 1 1 F F F
6002024-02-03T17:09:40.569Z 9 180-199 2 2 2 4 4 4 F T T <---
6012024-02-03T17:09:40.569Z 10 200-219 1 1 1 1 1 1 F F F
6022024-02-03T17:09:40.569Z 11 220-239 2 2 2 4 4 4 F F F
6032024-02-03T17:09:40.569Z 12 240-259 1 1 1 1 1 1 F T T <---
6042024-02-03T17:09:40.569Z 13 260-279 3 3 3 7 7 7 F F F
6052024-02-03T17:09:40.569Z 14 280-299 3 3 3 7 7 7 F F F
6062024-02-03T17:09:40.569Z 15 300-319 3 3 3 7 7 7 F F F
6072024-02-03T17:09:40.569Z 16 320-339 1 4 4 1 10 10 F F F <---
6082024-02-03T17:09:40.569Z 17 340-359 1 1 1 1 1 1 F F F
6092024-02-03T17:09:40.569Z 18 360-379 1 1 1 1 1 1 F F F
6102024-02-03T17:09:40.569Z 19 380-399 2 2 2 6 6 6 F F F
6112024-02-03T17:09:40.570Z 20 400-419 1 1 1 1 1 1 F F F
6122024-02-03T17:09:40.570Z 21 420-439 1 4 4 1 9 9 T F F <---
6132024-02-03T17:09:40.570Z 22 440-459 1 4 4 1 9 9 T F F <---
6142024-02-03T17:09:40.570Z 23 460-479 1 4 4 1 10 10 F F F <---
6152024-02-03T17:09:40.570Z 24 480-499 4 4 4 8 8 8 F T T <---
6162024-02-03T17:09:40.570Z 25 500-519 4 4 4 8 10 10 F F F <---
6172024-02-03T17:09:40.570Z 26 520-539 1 4 4 1 9 9 F F F <---
6182024-02-03T17:09:40.570Z 27 540-559 1 1 1 1 1 1 F F F
6192024-02-03T17:09:40.570Z 28 560-579 2 4 4 3 10 10 F F F <---
6202024-02-03T17:09:40.570Z 29 580-599 1 4 4 1 10 10 F F F <---
6212024-02-03T17:09:40.570ZMax gen: 4, Max flush: 10
6222024-02-03T17:09:40.570ZError: Difference in extent metadata found!
6232024-02-03T17:09:40.573ZA Difference in extent metadata is expected here
6242024-02-03T17:09:40.573ZOn loop 2
6252024-02-03T17:09:40.573Z
6262024-02-03T17:09:41.580ZVerifying data now
6272024-02-03T17:09:47.747ZNo change after verify
6282024-02-03T17:09:47.747ZLoop: 2 Downstairs dump after verify (and repair):
6292024-02-03T17:09:47.770ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6302024-02-03T17:09:47.771Z 0 000-019 4 4 4 10 10 10 F F F
6312024-02-03T17:09:47.771Z 1 020-039 2 2 2 4 4 4 F F F
6322024-02-03T17:09:47.771Z 2 040-059 1 1 1 1 1 1 F F F
6332024-02-03T17:09:47.774Z 3 060-079 5 5 5 11 11 11 F F F
6342024-02-03T17:09:47.774Z 4 080-099 5 5 5 11 11 11 F F F
6352024-02-03T17:09:47.774Z 5 100-119 2 2 2 5 5 5 F F F
6362024-02-03T17:09:47.775Z 6 120-139 4 4 4 10 10 10 F F F
6372024-02-03T17:09:47.775Z 7 140-159 1 1 1 1 1 1 F F F
6382024-02-03T17:09:47.775Z 8 160-179 1 1 1 1 1 1 F F F
6392024-02-03T17:09:47.775Z 9 180-199 5 5 5 11 11 11 F F F
6402024-02-03T17:09:47.775Z 10 200-219 1 1 1 1 1 1 F F F
6412024-02-03T17:09:47.775Z 11 220-239 2 2 2 4 4 4 F F F
6422024-02-03T17:09:47.775Z 12 240-259 5 5 5 11 11 11 F F F
6432024-02-03T17:09:47.775Z 13 260-279 3 3 3 7 7 7 F F F
6442024-02-03T17:09:47.775Z 14 280-299 3 3 3 7 7 7 F F F
6452024-02-03T17:09:47.775Z 15 300-319 3 3 3 7 7 7 F F F
6462024-02-03T17:09:47.775Z 16 320-339 4 4 4 10 10 10 F F F
6472024-02-03T17:09:47.775Z 17 340-359 1 1 1 1 1 1 F F F
6482024-02-03T17:09:47.776Z 18 360-379 1 1 1 1 1 1 F F F
6492024-02-03T17:09:47.776Z 19 380-399 2 2 2 6 6 6 F F F
6502024-02-03T17:09:47.776Z 20 400-419 1 1 1 1 1 1 F F F
6512024-02-03T17:09:47.776Z 21 420-439 4 4 4 9 9 9 F F F
6522024-02-03T17:09:47.776Z 22 440-459 4 4 4 9 9 9 F F F
6532024-02-03T17:09:47.776Z 23 460-479 4 4 4 10 10 10 F F F
6542024-02-03T17:09:47.776Z 24 480-499 5 5 5 11 11 11 F F F
6552024-02-03T17:09:47.776Z 25 500-519 4 4 4 10 10 10 F F F
6562024-02-03T17:09:47.776Z 26 520-539 4 4 4 9 9 9 F F F
6572024-02-03T17:09:47.776Z 27 540-559 1 1 1 1 1 1 F F F
6582024-02-03T17:09:47.776Z 28 560-579 4 4 4 10 10 10 F F F
6592024-02-03T17:09:47.776Z 29 580-599 4 4 4 10 10 10 F F F
6602024-02-03T17:09:47.776ZMax gen: 5, Max flush: 11
6612024-02-03T17:09:47.780Z
6622024-02-03T17:09:47.780ZBegin loop 3
6632024-02-03T17:09:47.780ZDownstairs to restart: 0
6642024-02-03T17:09:47.806Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.344565686Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153}
6652024-02-03T17:09:47.807Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.345089474Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153}
6662024-02-03T17:09:47.807Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.345119573Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153}
6672024-02-03T17:09:47.807Z{"msg":"Crucible bb31b06d-7e44-4938-943d-94340ed9ee20 has session id: 524b124c-1dbb-47b6-9af5-b88ee3431821"Crucible runtime is spawned
6682024-02-03T17:09:47.811Z,"v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.345296118Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6692024-02-03T17:09:47.811Z{"msg":"Upstairs opts: Upstairs UUID: bb31b06d-7e44-4938-943d-94340ed9ee20, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.345392168Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6702024-02-03T17:09:47.811Z{"msg":"Crucible stats registered with UUID: bb31b06d-7e44-4938-943d-94340ed9ee20","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.345422398Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6712024-02-03T17:09:47.811Z{"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.345487773Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153}
6722024-02-03T17:09:47.812Z{"msg":"bb31b06d-7e44-4938-943d-94340ed9ee20 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.345555247Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6732024-02-03T17:09:47.812Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.345632438Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6742024-02-03T17:09:47.812Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.345677663Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6752024-02-03T17:09:47.812Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.345717981Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6762024-02-03T17:09:47.812Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.345828189Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6772024-02-03T17:09:47.812Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.345874248Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6782024-02-03T17:09:47.812Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.345916526Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6792024-02-03T17:09:47.812Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.345993031Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"":"io task","client":"1","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6802024-02-03T17:09:47.812Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.346036854Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"":"io task","client":"0","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6812024-02-03T17:09:47.812Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.346120602Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"":"io task","client":"2","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6822024-02-03T17:09:47.812Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.346277494Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"":"io task","client":"2","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6832024-02-03T17:09:47.812Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.346396931Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"":"io task","client":"1","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6842024-02-03T17:09:47.813Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.346442729Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"":"io task","client":"0","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6852024-02-03T17:09:47.813Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.347595418Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6862024-02-03T17:09:47.813Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.347757061Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6872024-02-03T17:09:47.813Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.347881307Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6882024-02-03T17:09:47.813Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.347933831Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6892024-02-03T17:09:47.813Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.348538103Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6902024-02-03T17:09:47.813Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.348844256Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6912024-02-03T17:09:47.813Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.348914345Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6922024-02-03T17:09:47.813Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.348942134Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6932024-02-03T17:09:47.819Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.349416893Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6942024-02-03T17:09:47.819Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.349529011Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6952024-02-03T17:09:47.819Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.349602559Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6962024-02-03T17:09:47.819Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.349968763Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6972024-02-03T17:09:47.819Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.350040272Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6982024-02-03T17:09:47.819Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.350067218Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
6992024-02-03T17:09:47.819Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.350191768Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
7002024-02-03T17:09:47.819Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.350902032Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
7012024-02-03T17:09:47.819Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.351016776Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
7022024-02-03T17:09:47.819Z{"msg":"[0]R flush_numbers[0..12]: [10, 4, 1, 11, 11, 5, 10, 1, 1, 11, 1, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.351104529Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
7032024-02-03T17:09:47.819Z{"msg":"[0]R generation[0..12]: [4, 2, 1, 5, 5, 2, 4, 1, 1, 5, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.351149209Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"Wait for a query_work_queue command to finish before sending IO
7042024-02-03T17:09:47.819Z}
7052024-02-03T17:09:47.820Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.351281884Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
7062024-02-03T17:09:47.820Z{"msg":"[1]R flush_numbers[0..12]: [10, 4, 1, 11, 11, 5, 10, 1, 1, 11, 1, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.351419104Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
7072024-02-03T17:09:47.820Z{"msg":"[1]R generation[0..12]: [4, 2, 1, 5, 5, 2, 4, 1, 1, 5, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.351481918Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
7082024-02-03T17:09:47.820Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.351527346Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
7092024-02-03T17:09:47.820Z{"msg":"[2]R flush_numbers[0..12]: [10, 4, 1, 11, 11, 5, 10, 1, 1, 11, 1, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.351600065Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
7102024-02-03T17:09:47.820Z{"msg":"[2]R generation[0..12]: [4, 2, 1, 5, 5, 2, 4, 1, 1, 5, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.35164519Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
7112024-02-03T17:09:47.820Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.351711173Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
7122024-02-03T17:09:47.820Z{"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.351753204Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
7132024-02-03T17:09:47.820Z{"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.3518164Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
7142024-02-03T17:09:47.820Z{"msg":"Next flush: 12","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.351883929Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
7152024-02-03T17:09:47.820Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.351926354Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
7162024-02-03T17:09:47.820Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.351989904Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
7172024-02-03T17:09:47.820Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.352079707Z","Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
7182024-02-03T17:09:47.821Zhostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
7192024-02-03T17:09:47.821Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.352117569Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
7202024-02-03T17:09:47.821Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.352169891Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
7212024-02-03T17:09:47.821Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.352194767Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
7222024-02-03T17:09:47.821Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.352240434Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
7232024-02-03T17:09:47.821Z{"msg":"bb31b06d-7e44-4938-943d-94340ed9ee20 is now active with session: 524b124c-1dbb-47b6-9af5-b88ee3431821","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.352264838Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
7242024-02-03T17:09:47.821Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.352286274Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153,"session_id":"524b124c-1dbb-47b6-9af5-b88ee3431821"}
7252024-02-03T17:09:47.821Z{"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:47.352306879Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153}
7262024-02-03T17:09:47.821ZLoading write count information from file "/tmp/test_repair_verify.data"
7272024-02-03T17:09:47.821ZRun Repair workload
7282024-02-03T17:09:47.821Z01/30 Write block 433 len 16384 data: 2 2 2 3
7292024-02-03T17:09:47.821Z02/30 Write block 24 len 28672 data: 3 3 3 2 2 2 2
7302024-02-03T17:09:47.827Z03/30 Write block 76 len 36864 data: 3 3 3 3 3 3 3 3 2
7312024-02-03T17:09:47.827Z04/30 Flush
7322024-02-03T17:09:47.827Z05/30 Write block 329 len 24576 data: 2 2 2 2 2 2
7332024-02-03T17:09:47.827Z06/30 Write block 546 len 8192 data: 2 2
7342024-02-03T17:09:47.831Z07/30 Read block 210 len 20480
7352024-02-03T17:09:47.831Z08/30 Read block 260 len 40960
7362024-02-03T17:09:47.831Z09/30 Read block 35 len 20480
7372024-02-03T17:09:47.834Z10/30 Read block 562 len 12288
7382024-02-03T17:09:47.834Z11/30 Read block 588 len 12288
7392024-02-03T17:09:47.838Z12/30 Write block 248 len 24576 data: 2 2 2 2 3 3
7402024-02-03T17:09:47.838Z13/30 Write block 390 len 4096 data: 3
7412024-02-03T17:09:47.838Z14/30 Write block 50 len 4096 data: 2
7422024-02-03T17:09:47.838Z15/30 Write block 313 len 20480 data: 3 3 3 2 2
7432024-02-03T17:09:47.841Z16/30 Read block 202 len 8192
7442024-02-03T17:09:47.841Z17/30 Read block 473 len 24576
7452024-02-03T17:09:47.841Z18/30 Flush
7462024-02-03T17:09:47.915Z19/30 Write block 317 len 28672 data: 3 2 2 2 3 2 2
7472024-02-03T17:09:47.918Z20/30 Read block 462 len 40960
7482024-02-03T17:09:47.918Z21/30 Read block 104 len 36864
7492024-02-03T17:09:47.921Z22/30 Read block 580 len 36864
7502024-02-03T17:09:47.924Z23/30 Write block 108 len 20480 data: 2 2 2 3 3
7512024-02-03T17:09:47.925Z24/30 Write block 429 len 12288 data: 2 2 2
7522024-02-03T17:09:47.925Z25/30 Write block 283 len 36864 data: 2 2 2 2 2 2 2 2 2
7532024-02-03T17:09:47.928Z26/30 Read block 420 len 20480
7542024-02-03T17:09:47.928Z27/30 Write block 3 len 32768 data: 2 2 2 2 2 2 3 3
7552024-02-03T17:09:47.931Z28/30 Read block 499 len 16384
7562024-02-03T17:09:47.932Z29/30 Write block 596 len 4096 data: 2
7572024-02-03T17:09:47.935Z30/30 Write block 259 len 40960 data: 2 2 2 2 2 2 2 2 2 2
7582024-02-03T17:09:47.935Z----------------------------------------------------------------
7592024-02-03T17:09:47.935Z Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:26
7602024-02-03T17:09:47.935ZStates: Active Active Active
7612024-02-03T17:09:47.935ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
7622024-02-03T17:09:47.935Z 5 Acked 1004 Write 6 Done Done Done false
7632024-02-03T17:09:47.935Z 6 Acked 1005 Write 2 Done Done Done false
7642024-02-03T17:09:47.935Z 7 Acked 1006 Read 5 Done Done Done false
7652024-02-03T17:09:47.935Z 8 Acked 1007 Read 10 Done Done Done false
7662024-02-03T17:09:47.935Z 9 Acked 1008 Read 5 Done Done Done false
7672024-02-03T17:09:47.935Z 10 Acked 1009 Read 3 Done Done Done false
7682024-02-03T17:09:47.936Z 11 Acked 1010 Read 3 Done Done Done false
7692024-02-03T17:09:47.936Z 12 Acked 1011 Write 6 Done Done Done false
7702024-02-03T17:09:47.936Z 13 Acked 1012 Write 1 Done Done Done false
7712024-02-03T17:09:47.936Z 14 Acked 1013 Write 1 Done Done Done false
7722024-02-03T17:09:47.936Z 15 Acked 1014 Write 5 Done Done Done false
7732024-02-03T17:09:47.936Z 16 Acked 1015 Read 2 Sent Done Done false
7742024-02-03T17:09:47.936Z 17 Acked 1016 Read 6 Sent Done Done false
7752024-02-03T17:09:47.936Z 18 Acked 1017 Flush 0 Sent Done Done false
7762024-02-03T17:09:47.936Z 19 Acked 1018 Write 7 Sent Done Done false
7772024-02-03T17:09:47.936Z 20 Acked 1019 Read 10 Sent Done Done false
7782024-02-03T17:09:47.936Z 21 Acked 1020 Read 9 Sent Done Done false
7792024-02-03T17:09:47.936Z 22 Acked 1021 Read 9 Sent Done Done false
7802024-02-03T17:09:47.936Z 23 Acked 1022 Write 5 Sent Done Done false
7812024-02-03T17:09:47.936Z 24 Acked 1023 Write 3 Sent Done Done false
7822024-02-03T17:09:47.936Z 25 Acked 1024 Write 9 Sent Done Done false
7832024-02-03T17:09:47.936Z 26 Acked 1025 Read 5 Sent Done Done false
7842024-02-03T17:09:47.936Z 27 Acked 1026 Write 8 Sent Done Done false
7852024-02-03T17:09:47.936Z 28 Acked 1027 Read 4 Sent Done Done false
7862024-02-03T17:09:47.936Z 29 Acked 1028 Write 1 Sent Done Done false
7872024-02-03T17:09:47.937Z 30 Acked 1029 Write 10 Sent Sent Sent false
7882024-02-03T17:09:47.937Z STATES DS:0 DS:1 DS:2 TOTAL
7892024-02-03T17:09:47.937Z New 0 0 0 0
7902024-02-03T17:09:47.937Z Sent 15 1 1 17
7912024-02-03T17:09:47.937Z Done 11 25 25 61
7922024-02-03T17:09:47.937Z Skipped 0 0 0 0
7932024-02-03T17:09:47.937Z Error 0 0 0 0
7942024-02-03T17:09:47.937ZLast Flush: 1003 1017 1017
7952024-02-03T17:09:47.937ZDownstairs last five completed: 1003 1002 1001 1000
7962024-02-03T17:09:47.937ZUpstairs last five completed: 30 29 28 27 26
7972024-02-03T17:09:47.937Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:09:47.47277892Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1153}
7982024-02-03T17:09:47.937ZWrote out file "/tmp/test_repair_verify.data"
7992024-02-03T17:09:47.942Z
8002024-02-03T17:09:47.948ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
8012024-02-03T17:09:47.981ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8022024-02-03T17:09:47.981Z 0 000-019 4 4 4 10 10 10 F T T <---
8032024-02-03T17:09:47.981Z 1 020-039 6 6 6 12 12 12 F F F
8042024-02-03T17:09:47.981Z 2 040-059 1 6 6 1 13 13 T F F <---
8052024-02-03T17:09:47.985Z 3 060-079 6 6 6 12 12 12 F F F
8062024-02-03T17:09:47.985Z 4 080-099 6 6 6 12 12 12 F F F
8072024-02-03T17:09:47.985Z 5 100-119 2 2 2 5 5 5 F T T <---
8082024-02-03T17:09:47.985Z 6 120-139 4 4 4 10 10 10 F F F
8092024-02-03T17:09:47.985Z 7 140-159 1 1 1 1 1 1 F F F
8102024-02-03T17:09:47.985Z 8 160-179 1 1 1 1 1 1 F F F
8112024-02-03T17:09:47.985Z 9 180-199 5 5 5 11 11 11 F F F
8122024-02-03T17:09:47.985Z 10 200-219 1 1 1 1 1 1 F F F
8132024-02-03T17:09:47.986Z 11 220-239 2 2 2 4 4 4 F F F
8142024-02-03T17:09:47.986Z 12 240-259 5 6 6 11 13 13 T T T <---
8152024-02-03T17:09:47.986Z 13 260-279 3 3 3 7 7 7 F T T <---
8162024-02-03T17:09:47.986Z 14 280-299 3 3 3 7 7 7 F T T <---
8172024-02-03T17:09:47.986Z 15 300-319 3 6 6 7 13 13 T T T <---
8182024-02-03T17:09:47.986Z 16 320-339 4 6 6 10 13 13 T T T <---
8192024-02-03T17:09:47.986Z 17 340-359 1 1 1 1 1 1 F F F
8202024-02-03T17:09:47.986Z 18 360-379 1 1 1 1 1 1 F F F
8212024-02-03T17:09:47.986Z 19 380-399 2 6 6 6 13 13 T F F <---
8222024-02-03T17:09:47.986Z 20 400-419 1 1 1 1 1 1 F F F
8232024-02-03T17:09:47.986Z 21 420-439 6 6 6 12 12 12 F T T <---
8242024-02-03T17:09:47.986Z 22 440-459 4 4 4 9 9 9 F F F
8252024-02-03T17:09:47.986Z 23 460-479 4 4 4 10 10 10 F F F
8262024-02-03T17:09:47.987Z 24 480-499 5 5 5 11 11 11 F F F
8272024-02-03T17:09:47.987Z 25 500-519 4 4 4 10 10 10 F F F
8282024-02-03T17:09:47.987Z 26 520-539 4 4 4 9 9 9 F F F
8292024-02-03T17:09:47.987Z 27 540-559 1 6 6 1 13 13 T F F <---
8302024-02-03T17:09:47.987Z 28 560-579 4 4 4 10 10 10 F F F
8312024-02-03T17:09:47.987Z 29 580-599 4 4 4 10 10 10 F T T <---
8322024-02-03T17:09:47.987ZMax gen: 6, Max flush: 13
8332024-02-03T17:09:47.987ZError: Difference in extent metadata found!
8342024-02-03T17:09:47.987ZA Difference in extent metadata is expected here
8352024-02-03T17:09:47.987ZOn loop 3
8362024-02-03T17:09:47.987Z
8372024-02-03T17:09:47.991ZVerifying data now
8382024-02-03T17:09:54.111ZNo change after verify
8392024-02-03T17:09:54.111ZLoop: 3 Downstairs dump after verify (and repair):
8402024-02-03T17:09:54.134ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8412024-02-03T17:09:54.134Z 0 000-019 7 7 7 14 14 14 F F F
8422024-02-03T17:09:54.134Z 1 020-039 6 6 6 12 12 12 F F F
8432024-02-03T17:09:54.134Z 2 040-059 6 6 6 13 13 13 F F F
8442024-02-03T17:09:54.134Z 3 060-079 6 6 6 12 12 12 F F F
8452024-02-03T17:09:54.137Z 4 080-099 6 6 6 12 12 12 F F F
8462024-02-03T17:09:54.138Z 5 100-119 7 7 7 14 14 14 F F F
8472024-02-03T17:09:54.138Z 6 120-139 4 4 4 10 10 10 F F F
8482024-02-03T17:09:54.138Z 7 140-159 1 1 1 1 1 1 F F F
8492024-02-03T17:09:54.138Z 8 160-179 1 1 1 1 1 1 F F F
8502024-02-03T17:09:54.138Z 9 180-199 5 5 5 11 11 11 F F F
8512024-02-03T17:09:54.138Z 10 200-219 1 1 1 1 1 1 F F F
8522024-02-03T17:09:54.138Z 11 220-239 2 2 2 4 4 4 F F F
8532024-02-03T17:09:54.138Z 12 240-259 7 7 7 14 14 14 F F F
8542024-02-03T17:09:54.138Z 13 260-279 7 7 7 14 14 14 F F F
8552024-02-03T17:09:54.138Z 14 280-299 7 7 7 14 14 14 F F F
8562024-02-03T17:09:54.138Z 15 300-319 7 7 7 14 14 14 F F F
8572024-02-03T17:09:54.138Z 16 320-339 7 7 7 14 14 14 F F F
8582024-02-03T17:09:54.138Z 17 340-359 1 1 1 1 1 1 F F F
8592024-02-03T17:09:54.138Z 18 360-379 1 1 1 1 1 1 F F F
8602024-02-03T17:09:54.138Z 19 380-399 6 6 6 13 13 13 F F F
8612024-02-03T17:09:54.138Z 20 400-419 1 1 1 1 1 1 F F F
8622024-02-03T17:09:54.138Z 21 420-439 7 7 7 14 14 14 F F F
8632024-02-03T17:09:54.138Z 22 440-459 4 4 4 9 9 9 F F F
8642024-02-03T17:09:54.138Z 23 460-479 4 4 4 10 10 10 F F F
8652024-02-03T17:09:54.138Z 24 480-499 5 5 5 11 11 11 F F F
8662024-02-03T17:09:54.138Z 25 500-519 4 4 4 10 10 10 F F F
8672024-02-03T17:09:54.139Z 26 520-539 4 4 4 9 9 9 F F F
8682024-02-03T17:09:54.139Z 27 540-559 6 6 6 13 13 13 F F F
8692024-02-03T17:09:54.139Z 28 560-579 4 4 4 10 10 10 F F F
8702024-02-03T17:09:54.139Z 29 580-599 7 7 7 14 14 14 F F F
8712024-02-03T17:09:54.139ZMax gen: 7, Max flush: 14
8722024-02-03T17:09:54.142Z
8732024-02-03T17:09:54.142ZBegin loop 4
8742024-02-03T17:09:54.142ZDownstairs to restart: 1
8752024-02-03T17:09:54.175Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.711248441Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161}
8762024-02-03T17:09:54.182Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.711748279Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161}
8772024-02-03T17:09:54.182Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
8782024-02-03T17:09:54.182Z,"time":"2024-02-03T17:09:53.711885404Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161}
8792024-02-03T17:09:54.182Z{"msg":"Crucible d0f140de-a058-4530-8c9f-44e872b7446e has session id: cc94a362-897c-4f56-a5a3-2e879134477a","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.712149235Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
8802024-02-03T17:09:54.182Z{"msg":"Upstairs opts: Upstairs UUID: d0f140de-a058-4530-8c9f-44e872b7446e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.712320773Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
8812024-02-03T17:09:54.182Z{"msg":"Crucible stats registered with UUID: d0f140de-a058-4530-8c9f-44e872b7446e","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.712350892Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
8822024-02-03T17:09:54.182Z{"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.712383727Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161}
8832024-02-03T17:09:54.182Z{"msg":"d0f140de-a058-4530-8c9f-44e872b7446e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.712533634Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
8842024-02-03T17:09:54.182Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.712579881Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
8852024-02-03T17:09:54.182Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.712622687Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
8862024-02-03T17:09:54.182Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.71266141Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
8872024-02-03T17:09:54.182Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.712702653Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
8882024-02-03T17:09:54.182Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.712738339Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
8892024-02-03T17:09:54.182Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.712773521Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
8902024-02-03T17:09:54.182Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.712809881Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"":"io task","client":"2","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
8912024-02-03T17:09:54.182Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.712851811Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"":"io task","client":"0","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
8922024-02-03T17:09:54.183Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.71292658Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"":"io task","client":"1","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
8932024-02-03T17:09:54.183Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.712968442Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"":"io task","client":"1","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
8942024-02-03T17:09:54.183Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.713110118Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"":"io task","client":"2","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
8952024-02-03T17:09:54.183Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.713223684Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"":"io task","client":"0","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
8962024-02-03T17:09:54.183Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.714455903Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
8972024-02-03T17:09:54.183Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.714598411Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
8982024-02-03T17:09:54.183Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.714668702Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
8992024-02-03T17:09:54.183Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.714737263Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9002024-02-03T17:09:54.183Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.714781199Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9012024-02-03T17:09:54.183Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.714891838Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9022024-02-03T17:09:54.183Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.715436256Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9032024-02-03T17:09:54.183Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.715928303Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9042024-02-03T17:09:54.183Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.716469329Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9052024-02-03T17:09:54.183Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.716563837Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9062024-02-03T17:09:54.183Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.716652653Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9072024-02-03T17:09:54.183Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.716735897Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9082024-02-03T17:09:54.183Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.717016537Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9092024-02-03T17:09:54.183Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.717083023Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9102024-02-03T17:09:54.183Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.717109465Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9112024-02-03T17:09:54.183Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.717569707Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9122024-02-03T17:09:54.183Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.717683359Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9132024-02-03T17:09:54.183Z{"msg":"[0]R flush_numbers[0..12]: [14, 12, 13, 12, 12, 14, 10, 1, 1, 11, 1, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.717710705Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9142024-02-03T17:09:54.183Z{"msg":"[0]R generation[0..12]: [7, 6, 6, 6, 6, 7, 4, 1, 1, 5, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.717734085Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9152024-02-03T17:09:54.183Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.717783383Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9162024-02-03T17:09:54.184Z{"msg":"[1]R flush_numbers[0..12]: [14, 12, 13, 12, 12, 14, 10, 1, 1, 11, 1, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.717809215Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9172024-02-03T17:09:54.184Z{"msg":"[1]R generation[0..12]: [7, 6, 6, 6, 6, 7, 4, 1, 1, 5, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.717856801Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9182024-02-03T17:09:54.184Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.717882994Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9192024-02-03T17:09:54.184ZWait for a query_work_queue command to finish before sending IO
9202024-02-03T17:09:54.184Z{"msg":"[2]R flush_numbers[0..12]: [14, 12, 13, 12, 12, 14, 10, 1, 1, 11, 1, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.717935573Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9212024-02-03T17:09:54.184Z{"msg":"[2]R generation[0..12]: [7, 6, 6, 6, 6, 7, 4, 1, 1, 5, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.717960644Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9222024-02-03T17:09:54.184Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.718009364Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9232024-02-03T17:09:54.184Z{"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.718033784Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9242024-02-03T17:09:54.184Z{"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.718081286Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9252024-02-03T17:09:54.184Z{"msg":"Next flush: 15","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.718104931Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9262024-02-03T17:09:54.184Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.718153985Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9272024-02-03T17:09:54.184Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.718178207Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9282024-02-03T17:09:54.184Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.718198555Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9292024-02-03T17:09:54.184Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.71822197Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9302024-02-03T17:09:54.188Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.718244366Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9312024-02-03T17:09:54.188Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.718268253Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9322024-02-03T17:09:54.188Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.718288992Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9332024-02-03T17:09:54.188Z{"msg":"d0f140de-a058-4530-8c9f-44e872b7446e is now active with session: cc94a362-897c-4f56-a5a3-2e879134477a","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.718311681Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9342024-02-03T17:09:54.188Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.718356249Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161,"session_id":"cc94a362-897c-4f56-a5a3-2e879134477a"}
9352024-02-03T17:09:54.188Z{"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:53.718378854Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161}
9362024-02-03T17:09:54.188ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
9372024-02-03T17:09:54.188ZLoading write count information from file "/tmp/test_repair_verify.data"
9382024-02-03T17:09:54.188ZRun Repair workload
9392024-02-03T17:09:54.188Z01/30 Write block 322 len 8192 data: 3 3
9402024-02-03T17:09:54.188Z02/30 Write block 440 len 12288 data: 3 3 3
9412024-02-03T17:09:54.188Z03/30 Write block 283 len 20480 data: 3 3 3 3 3
9422024-02-03T17:09:54.188Z04/30 Write block 180 len 16384 data: 2 2 2 3
9432024-02-03T17:09:54.188Z05/30 Read block 346 len 4096
9442024-02-03T17:09:54.188Z06/30 Write block 146 len 4096 data: 2
9452024-02-03T17:09:54.189Z07/30 Read block 508 len 16384
9462024-02-03T17:09:54.192Z08/30 Flush
9472024-02-03T17:09:54.195Z09/30 Write block 427 len 16384 data: 2 2 3 3
9482024-02-03T17:09:54.195Z10/30 Read block 117 len 12288
9492024-02-03T17:09:54.198Z11/30 Read block 3 len 40960
9502024-02-03T17:09:54.198Z12/30 Read block 456 len 28672
9512024-02-03T17:09:54.201Z13/30 Read block 335 len 40960
9522024-02-03T17:09:54.201Z14/30 Read block 119 len 24576
9532024-02-03T17:09:54.204Z15/30 Write block 491 len 4096 data: 4
9542024-02-03T17:09:54.207Z16/30 Write block 367 len 16384 data: 2 2 2 2
9552024-02-03T17:09:54.207Z17/30 Read block 252 len 40960
9562024-02-03T17:09:54.210Z18/30 Read block 375 len 8192
9572024-02-03T17:09:54.210Z19/30 Write block 368 len 12288 data: 3 3 3
9582024-02-03T17:09:54.210Z20/30 Write block 173 len 20480 data: 2 2 2 2 2
9592024-02-03T17:09:54.210Z21/30 Read block 274 len 16384
9602024-02-03T17:09:54.213Z22/30 Read block 346 len 20480
9612024-02-03T17:09:54.213Z23/30 Read block 134 len 8192
9622024-02-03T17:09:54.216Z24/30 Write block 173 len 24576 data: 3 3 3 3 3 2
9632024-02-03T17:09:54.216Z25/30 Read block 501 len 40960
9642024-02-03T17:09:54.219Z26/30 Read block 37 len 36864
9652024-02-03T17:09:54.219Z27/30 Read block 125 len 36864
9662024-02-03T17:09:54.222Z28/30 Read block 100 len 28672
9672024-02-03T17:09:54.225Z29/30 Write block 392 len 16384 data: 3 3 3 3
9682024-02-03T17:09:54.225Z30/30 Read block 57 len 20480
9692024-02-03T17:09:54.225Z----------------------------------------------------------------
9702024-02-03T17:09:54.226Z Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:30
9712024-02-03T17:09:54.226ZStates: Active Active Active
9722024-02-03T17:09:54.226ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
9732024-02-03T17:09:54.226Z 1 Acked 1000 Write 2 Done Done Done false
9742024-02-03T17:09:54.226Z 2 Acked 1001 Write 3 Done Sent Done false
9752024-02-03T17:09:54.226Z 3 Acked 1002 Write 5 Done Sent Done false
9762024-02-03T17:09:54.226Z 4 Acked 1003 Write 4 Done Sent Done false
9772024-02-03T17:09:54.226Z 5 Acked 1004 Read 1 Done Sent Done false
9782024-02-03T17:09:54.226Z 6 Acked 1005 Write 1 Done Sent Done false
9792024-02-03T17:09:54.226Z 7 Acked 1006 Read 4 Done Sent Done false
9802024-02-03T17:09:54.226Z 8 Acked 1007 Flush 0 Done Sent Done false
9812024-02-03T17:09:54.226Z 9 Acked 1008 Write 4 Done Sent Done false
9822024-02-03T17:09:54.226Z 10 Acked 1009 Read 3 Done Sent Done false
9832024-02-03T17:09:54.230Z 11 Acked 1010 Read 10 Done Sent Done false
9842024-02-03T17:09:54.230Z 12 Acked 1011 Read 7 Done Sent Done false
9852024-02-03T17:09:54.230Z 13 Acked 1012 Read 10 Done Sent Done false
9862024-02-03T17:09:54.230Z 14 Acked 1013 Read 6 Done Sent Done false
9872024-02-03T17:09:54.230Z 15 Acked 1014 Write 1 Done Sent Done false
9882024-02-03T17:09:54.230Z 16 Acked 1015 Write 4 Done Sent Done false
9892024-02-03T17:09:54.230Z 17 Acked 1016 Read 10 Done Sent Done false
9902024-02-03T17:09:54.230Z 18 Acked 1017 Read 2 Done Sent Done false
9912024-02-03T17:09:54.230Z 19 Acked 1018 Write 3 Done Sent Done false
9922024-02-03T17:09:54.230Z 20 Acked 1019 Write 5 Done Sent Done false
9932024-02-03T17:09:54.230Z 21 Acked 1020 Read 4 Done Sent Done false
9942024-02-03T17:09:54.230Z 22 Acked 1021 Read 5 Done Sent Done false
9952024-02-03T17:09:54.230Z 23 Acked 1022 Read 2 Done Sent Done false
9962024-02-03T17:09:54.230Z 24 Acked 1023 Write 6 Done Sent Done false
9972024-02-03T17:09:54.230Z 25 Acked 1024 Read 10 Done Sent Done false
9982024-02-03T17:09:54.230Z 26 Acked 1025 Read 9 Done Sent Done false
9992024-02-03T17:09:54.230Z 27 Acked 1026 Read 9 Done Sent Done false
10002024-02-03T17:09:54.230Z 28 Acked 1027 Read 7 Done Sent Done false
10012024-02-03T17:09:54.230Z 29 Acked 1028 Write 4 Done Sent Done false
10022024-02-03T17:09:54.230Z 30 Acked 1029 Read 5 Done Sent Done false
10032024-02-03T17:09:54.230Z STATES DS:0 DS:1 DS:2 TOTAL
10042024-02-03T17:09:54.230Z New 0 0 0 0
10052024-02-03T17:09:54.231Z Sent 0 29 0 29
10062024-02-03T17:09:54.231Z Done 30 1 30 61
10072024-02-03T17:09:54.231Z Skipped 0 0 0 0
10082024-02-03T17:09:54.231Z Error 0 0 0 0
10092024-02-03T17:09:54.231ZLast Flush: 1007 0 1007
10102024-02-03T17:09:54.231ZDownstairs last five completed:
10112024-02-03T17:09:54.231ZUpstairs last five completed: 30 29 28 27 26
10122024-02-03T17:09:54.231Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:09:53.765402609Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1161}
10132024-02-03T17:09:54.231ZWrote out file "/tmp/test_repair_verify.data"
10142024-02-03T17:09:54.237Z
10152024-02-03T17:09:54.240ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
10162024-02-03T17:09:54.265ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10172024-02-03T17:09:54.265Z 0 000-019 7 7 7 14 14 14 F F F
10182024-02-03T17:09:54.265Z 1 020-039 6 6 6 12 12 12 F F F
10192024-02-03T17:09:54.265Z 2 040-059 6 6 6 13 13 13 F F F
10202024-02-03T17:09:54.265Z 3 060-079 6 6 6 12 12 12 F F F
10212024-02-03T17:09:54.265Z 4 080-099 6 6 6 12 12 12 F F F
10222024-02-03T17:09:54.265Z 5 100-119 7 7 7 14 14 14 F F F
10232024-02-03T17:09:54.265Z 6 120-139 4 4 4 10 10 10 F F F
10242024-02-03T17:09:54.265Z 7 140-159 8 1 8 15 1 15 F F F <---
10252024-02-03T17:09:54.265Z 8 160-179 1 1 1 1 1 1 T F T <---
10262024-02-03T17:09:54.265Z 9 180-199 8 5 8 15 11 15 F F F <---
10272024-02-03T17:09:54.269Z 10 200-219 1 1 1 1 1 1 F F F
10282024-02-03T17:09:54.269Z 11 220-239 2 2 2 4 4 4 F F F
10292024-02-03T17:09:54.269Z 12 240-259 7 7 7 14 14 14 F F F
10302024-02-03T17:09:54.269Z 13 260-279 7 7 7 14 14 14 F F F
10312024-02-03T17:09:54.269Z 14 280-299 8 7 8 15 14 15 F F F <---
10322024-02-03T17:09:54.269Z 15 300-319 7 7 7 14 14 14 F F F
10332024-02-03T17:09:54.269Z 16 320-339 8 7 8 15 14 15 F T F <---
10342024-02-03T17:09:54.269Z 17 340-359 1 1 1 1 1 1 F F F
10352024-02-03T17:09:54.269Z 18 360-379 1 1 1 1 1 1 T F T <---
10362024-02-03T17:09:54.269Z 19 380-399 6 6 6 13 13 13 T F T <---
10372024-02-03T17:09:54.269Z 20 400-419 1 1 1 1 1 1 F F F
10382024-02-03T17:09:54.269Z 21 420-439 7 7 7 14 14 14 T F T <---
10392024-02-03T17:09:54.269Z 22 440-459 8 4 8 15 9 15 F F F <---
10402024-02-03T17:09:54.269Z 23 460-479 4 4 4 10 10 10 F F F
10412024-02-03T17:09:54.269Z 24 480-499 5 5 5 11 11 11 T F T <---
10422024-02-03T17:09:54.269Z 25 500-519 4 4 4 10 10 10 F F F
10432024-02-03T17:09:54.269Z 26 520-539 4 4 4 9 9 9 F F F
10442024-02-03T17:09:54.269Z 27 540-559 6 6 6 13 13 13 F F F
10452024-02-03T17:09:54.269Z 28 560-579 4 4 4 10 10 10 F F F
10462024-02-03T17:09:54.269Z 29 580-599 7 7 7 14 14 14 F F F
10472024-02-03T17:09:54.269ZMax gen: 8, Max flush: 15
10482024-02-03T17:09:54.269ZError: Difference in extent metadata found!
10492024-02-03T17:09:54.273ZA Difference in extent metadata is expected here
10502024-02-03T17:09:54.273ZOn loop 4
10512024-02-03T17:09:54.273Z
10522024-02-03T17:09:54.276ZVerifying data now
10532024-02-03T17:09:59.187ZNo change after verify
10542024-02-03T17:09:59.187ZLoop: 4 Downstairs dump after verify (and repair):
10552024-02-03T17:09:59.210ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10562024-02-03T17:09:59.210Z 0 000-019 7 7 7 14 14 14 F F F
10572024-02-03T17:09:59.214Z 1 020-039 6 6 6 12 12 12 F F F
10582024-02-03T17:09:59.214Z 2 040-059 6 6 6 13 13 13 F F F
10592024-02-03T17:09:59.214Z 3 060-079 6 6 6 12 12 12 F F F
10602024-02-03T17:09:59.214Z 4 080-099 6 6 6 12 12 12 F F F
10612024-02-03T17:09:59.214Z 5 100-119 7 7 7 14 14 14 F F F
10622024-02-03T17:09:59.214Z 6 120-139 4 4 4 10 10 10 F F F
10632024-02-03T17:09:59.214Z 7 140-159 8 8 8 15 15 15 F F F
10642024-02-03T17:09:59.214Z 8 160-179 9 9 9 16 16 16 F F F
10652024-02-03T17:09:59.214Z 9 180-199 8 8 8 15 15 15 F F F
10662024-02-03T17:09:59.214Z 10 200-219 1 1 1 1 1 1 F F F
10672024-02-03T17:09:59.214Z 11 220-239 2 2 2 4 4 4 F F F
10682024-02-03T17:09:59.214Z 12 240-259 7 7 7 14 14 14 F F F
10692024-02-03T17:09:59.215Z 13 260-279 7 7 7 14 14 14 F F F
10702024-02-03T17:09:59.215Z 14 280-299 8 8 8 15 15 15 F F F
10712024-02-03T17:09:59.215Z 15 300-319 7 7 7 14 14 14 F F F
10722024-02-03T17:09:59.215Z 16 320-339 8 8 8 15 15 15 F F F
10732024-02-03T17:09:59.215Z 17 340-359 1 1 1 1 1 1 F F F
10742024-02-03T17:09:59.215Z 18 360-379 9 9 9 16 16 16 F F F
10752024-02-03T17:09:59.215Z 19 380-399 9 9 9 16 16 16 F F F
10762024-02-03T17:09:59.215Z 20 400-419 1 1 1 1 1 1 F F F
10772024-02-03T17:09:59.215Z 21 420-439 9 9 9 16 16 16 F F F
10782024-02-03T17:09:59.215Z 22 440-459 8 8 8 15 15 15 F F F
10792024-02-03T17:09:59.215Z 23 460-479 4 4 4 10 10 10 F F F
10802024-02-03T17:09:59.215Z 24 480-499 9 9 9 16 16 16 F F F
10812024-02-03T17:09:59.215Z 25 500-519 4 4 4 10 10 10 F F F
10822024-02-03T17:09:59.215Z 26 520-539 4 4 4 9 9 9 F F F
10832024-02-03T17:09:59.216Z 27 540-559 6 6 6 13 13 13 F F F
10842024-02-03T17:09:59.216Z 28 560-579 4 4 4 10 10 10 F F F
10852024-02-03T17:09:59.216Z 29 580-599 7 7 7 14 14 14 F F F
10862024-02-03T17:09:59.216ZMax gen: 9, Max flush: 16
10872024-02-03T17:09:59.219Z
10882024-02-03T17:09:59.219ZBegin loop 5
10892024-02-03T17:09:59.219ZDownstairs to restart: 1
10902024-02-03T17:09:59.246Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.784776841Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169}
10912024-02-03T17:09:59.246Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.785328322Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169}
10922024-02-03T17:09:59.246Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.785362897Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169}
10932024-02-03T17:09:59.246Z{"msg":"Crucible runtime is spawned
10942024-02-03T17:09:59.246ZCrucible c97f8d75-6f43-42fd-a293-8a62b33c123e has session id: e70927bd-d2ec-42b9-890c-896e476eafc5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.785408792Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
10952024-02-03T17:09:59.251Z{"msg":"Upstairs opts: Upstairs UUID: c97f8d75-6f43-42fd-a293-8a62b33c123e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.785554185Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
10962024-02-03T17:09:59.251Z{"msg":"Crucible stats registered with UUID: c97f8d75-6f43-42fd-a293-8a62b33c123e","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.785647167Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
10972024-02-03T17:09:59.251Z{"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.785750474Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169}
10982024-02-03T17:09:59.251Z{"msg":"c97f8d75-6f43-42fd-a293-8a62b33c123e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.785819463Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
10992024-02-03T17:09:59.252Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.785843852Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11002024-02-03T17:09:59.252Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.785867134Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11012024-02-03T17:09:59.252Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.785920801Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11022024-02-03T17:09:59.252Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.785963954Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11032024-02-03T17:09:59.252Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.786044997Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11042024-02-03T17:09:59.252Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.786091667Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11052024-02-03T17:09:59.252Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.786175897Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"":"io task","client":"2","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11062024-02-03T17:09:59.252Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.786219352Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"":"io task","client":"0","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11072024-02-03T17:09:59.252Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.786263307Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"":"io task","client":"1","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11082024-02-03T17:09:59.252Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.786379694Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"":"io task","client":"1","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11092024-02-03T17:09:59.252Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.786702151Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"":"io task","client":"0","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11102024-02-03T17:09:59.252Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.786819664Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"":"io task","client":"2","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11112024-02-03T17:09:59.252Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.787617499Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11122024-02-03T17:09:59.252Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.787745716Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11132024-02-03T17:09:59.252Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.787856011Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11142024-02-03T17:09:59.252Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.788065605Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11152024-02-03T17:09:59.252Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.788111725Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11162024-02-03T17:09:59.252Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.788154601Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11172024-02-03T17:09:59.252Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.788561147Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11182024-02-03T17:09:59.253Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.788630333Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11192024-02-03T17:09:59.253Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.789063201Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11202024-02-03T17:09:59.258Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.789564577Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11212024-02-03T17:09:59.258Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.789655035Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11222024-02-03T17:09:59.258Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.78970133Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11232024-02-03T17:09:59.258Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.789740936Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11242024-02-03T17:09:59.258Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.789821118Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11252024-02-03T17:09:59.258Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.789866167Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11262024-02-03T17:09:59.258Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.789940006Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11272024-02-03T17:09:59.258Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.790080461Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11282024-02-03T17:09:59.258Z{"msg":"[0]R flush_numbers[0..12]: [14, 12, 13, 12, 12, 14, 10, 15, 16, 15, 1, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.790165612Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11292024-02-03T17:09:59.258Z{"msg":"[0]R generation[0..12]: [7, 6, 6, 6, 6, 7, 4, 8, 9, 8, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.790196361Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11302024-02-03T17:09:59.259Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.790220628Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11312024-02-03T17:09:59.259Z{"msg":"[1]R flush_numbers[0..12]: [14, 12, 13, 12, 12, 14, 10, 15, 16, 15, 1, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.790243337Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11322024-02-03T17:09:59.259Z{"msg":"[1]R generation[0..12]: [7, 6, 6, 6, 6, 7, 4, 8, 9, 8, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.79029331Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11332024-02-03T17:09:59.259Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.790319714Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11342024-02-03T17:09:59.259Z{"msg":"[2]R flush_numbers[0..12]: [14, 12, 13, 12, 12, 14, 10, 15, 16, 15, 1, 4]","v":0,"name":Wait for a query_work_queue command to finish before sending IO
11352024-02-03T17:09:59.259Z"crucible","level":30,"time":"2024-02-03T17:09:58.790347002Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11362024-02-03T17:09:59.259Z{"msg":"[2]R generation[0..12]: [7, 6, 6, 6, 6, 7, 4, 8, 9, 8, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.790478196Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11372024-02-03T17:09:59.259Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.790508599Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11382024-02-03T17:09:59.259Z{"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.79053207Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11392024-02-03T17:09:59.259Z{"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.790581042Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11402024-02-03T17:09:59.259Z{"msg":"Next flush: 17","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.790604351Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11412024-02-03T17:09:59.259Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.790625623Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11422024-02-03T17:09:59.259Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.790648159Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11432024-02-03T17:09:59.260Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.790682378Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11442024-02-03T17:09:59.260Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.790721917Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11452024-02-03T17:09:59.260Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.79075615Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11462024-02-03T17:09:59.260Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.79077954Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11472024-02-03T17:09:59.260Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.790799912Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11482024-02-03T17:09:59.260Z{"msg":"c97f8d75-6f43-42fd-a293-8a62b33c123e is now active with session: e70927bd-d2ec-42b9-890c-896e476eafc5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.790835125Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11492024-02-03T17:09:59.260Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.790873128Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169,"session_id":"e70927bd-d2ec-42b9-890c-896e476eafc5"}
11502024-02-03T17:09:59.260Z{"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:09:58.79093082Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169}
11512024-02-03T17:09:59.260ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
11522024-02-03T17:09:59.260ZLoading write count information from file "/tmp/test_repair_verify.data"
11532024-02-03T17:09:59.260ZRun Repair workload
11542024-02-03T17:09:59.260Z01/30 Write block 390 len 24576 data: 4 3 4 4 4 4
11552024-02-03T17:09:59.260Z02/30 Read block 565 len 16384
11562024-02-03T17:09:59.264Z03/30 Write block 316 len 16384 data: 3 4 3 3
11572024-02-03T17:09:59.264Z04/30 Read block 220 len 28672
11582024-02-03T17:09:59.264Z05/30 Read block 155 len 4096
11592024-02-03T17:09:59.264Z06/30 Write block 22 len 28672 data: 3 3 4 4 4 3 3
11602024-02-03T17:09:59.264Z07/30 Read block 172 len 12288
11612024-02-03T17:09:59.265Z08/30 Write block 274 len 8192 data: 2 2
11622024-02-03T17:09:59.268Z09/30 Write block 348 len 32768 data: 2 2 2 2 2 2 2 2
11632024-02-03T17:09:59.268Z10/30 Read block 383 len 8192
11642024-02-03T17:09:59.271Z11/30 Read block 468 len 8192
11652024-02-03T17:09:59.271Z12/30 Read block 452 len 16384
11662024-02-03T17:09:59.272Z13/30 Read block 236 len 40960
11672024-02-03T17:09:59.275Z14/30 Read block 154 len 32768
11682024-02-03T17:09:59.275Z15/30 Read block 571 len 24576
11692024-02-03T17:09:59.279Z16/30 Read block 11 len 24576
11702024-02-03T17:09:59.279Z17/30 Read block 330 len 8192
11712024-02-03T17:09:59.279Z18/30 Write block 483 len 28672 data: 2 2 2 2 2 3 3
11722024-02-03T17:09:59.282Z19/30 Read block 182 len 16384
11732024-02-03T17:09:59.282Z20/30 Write block 85 len 8192 data: 2 2
11742024-02-03T17:09:59.282Z21/30 Read block 291 len 16384
11752024-02-03T17:09:59.286Z22/30 Write block 414 len 8192 data: 2 2
11762024-02-03T17:09:59.287Z23/30 Read block 2 len 32768
11772024-02-03T17:09:59.287Z24/30 Read block 528 len 12288
11782024-02-03T17:09:59.290Z25/30 Read block 85 len 20480
11792024-02-03T17:09:59.290Z26/30 Read block 66 len 8192
11802024-02-03T17:09:59.299Z27/30 Write block 271 len 28672 data: 2 2 2 3 3 3 3
11812024-02-03T17:09:59.299Z28/30 Write block 31 len 16384 data: 2 2 2 2
11822024-02-03T17:09:59.299Z29/30 Read block 547 len 28672
11832024-02-03T17:09:59.302Z30/30 Read block 507 len 36864
11842024-02-03T17:09:59.303Z----------------------------------------------------------------
11852024-02-03T17:09:59.303Z Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:30
11862024-02-03T17:09:59.303ZStates: Active Active Active
11872024-02-03T17:09:59.303ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
11882024-02-03T17:09:59.303Z 1 Acked 1000 Write 6 Done Done Done false
11892024-02-03T17:09:59.303Z 2 Acked 1001 Read 4 Done Done Done false
11902024-02-03T17:09:59.303Z 3 Acked 1002 Write 4 Done Done Done false
11912024-02-03T17:09:59.303Z 4 Acked 1003 Read 7 Done Done Done false
11922024-02-03T17:09:59.303Z 5 Acked 1004 Read 1 Done Done Done false
11932024-02-03T17:09:59.303Z 6 Acked 1005 Write 7 Done Done Done false
11942024-02-03T17:09:59.303Z 7 Acked 1006 Read 3 Done Done Done false
11952024-02-03T17:09:59.303Z 8 Acked 1007 Write 2 Done Done Done false
11962024-02-03T17:09:59.304Z 9 Acked 1008 Write 8 Done Sent Done false
11972024-02-03T17:09:59.304Z 10 Acked 1009 Read 2 Done Sent Done false
11982024-02-03T17:09:59.304Z 11 Acked 1010 Read 2 Done Sent Done false
11992024-02-03T17:09:59.304Z 12 Acked 1011 Read 4 Done Sent Done false
12002024-02-03T17:09:59.304Z 13 Acked 1012 Read 10 Done Sent Done false
12012024-02-03T17:09:59.304Z 14 Acked 1013 Read 8 Done Sent Done false
12022024-02-03T17:09:59.304Z 15 Acked 1014 Read 6 Done Sent Done false
12032024-02-03T17:09:59.304Z 16 Acked 1015 Read 6 Done Sent Done false
12042024-02-03T17:09:59.304Z 17 Acked 1016 Read 2 Done Sent Done false
12052024-02-03T17:09:59.304Z 18 Acked 1017 Write 7 Done Sent Done false
12062024-02-03T17:09:59.304Z 19 Acked 1018 Read 4 Done Sent Done false
12072024-02-03T17:09:59.304Z 20 Acked 1019 Write 2 Done Sent Done false
12082024-02-03T17:09:59.304Z 21 Acked 1020 Read 4 Done Sent Done false
12092024-02-03T17:09:59.305Z 22 Acked 1021 Write 2 Done Sent Done false
12102024-02-03T17:09:59.305Z 23 Acked 1022 Read 8 Done Sent Done false
12112024-02-03T17:09:59.305Z 24 Acked 1023 Read 3 Done Sent Done false
12122024-02-03T17:09:59.305Z 25 Acked 1024 Read 5 Done Sent Done false
12132024-02-03T17:09:59.305Z 26 Acked 1025 Read 2 Done Sent Done false
12142024-02-03T17:09:59.305Z 27 Acked 1026 Write 7 Done Sent Done false
12152024-02-03T17:09:59.305Z 28 Acked 1027 Write 4 Done Sent Done false
12162024-02-03T17:09:59.305Z 29 Acked 1028 Read 7 Done Sent Done false
12172024-02-03T17:09:59.305Z 30 Acked 1029 Read 9 Done Sent Done false
12182024-02-03T17:09:59.305Z STATES DS:0 DS:1 DS:2 TOTAL
12192024-02-03T17:09:59.305Z New 0 0 0 0
12202024-02-03T17:09:59.305Z Sent 0 22 0 22
12212024-02-03T17:09:59.305Z Done 30 8 30 68
12222024-02-03T17:09:59.306Z Skipped 0 0 0 0
12232024-02-03T17:09:59.306Z Error 0 0 0 0
12242024-02-03T17:09:59.306ZLast Flush: 0 0 0
12252024-02-03T17:09:59.306ZDownstairs last five completed:
12262024-02-03T17:09:59.306ZUpstairs last five completed: 30 29 28 27 26
12272024-02-03T17:09:59.306Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:09:58.836650906Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1169}
12282024-02-03T17:09:59.306ZWrote out file "/tmp/test_repair_verify.data"
12292024-02-03T17:09:59.309Z
12302024-02-03T17:09:59.313ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
12312024-02-03T17:09:59.339ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12322024-02-03T17:09:59.339Z 0 000-019 7 7 7 14 14 14 F F F
12332024-02-03T17:09:59.339Z 1 020-039 6 6 6 12 12 12 T T T
12342024-02-03T17:09:59.339Z 2 040-059 6 6 6 13 13 13 F F F
12352024-02-03T17:09:59.339Z 3 060-079 6 6 6 12 12 12 F F F
12362024-02-03T17:09:59.343Z 4 080-099 6 6 6 12 12 12 T F T <---
12372024-02-03T17:09:59.343Z 5 100-119 7 7 7 14 14 14 F F F
12382024-02-03T17:09:59.343Z 6 120-139 4 4 4 10 10 10 F F F
12392024-02-03T17:09:59.343Z 7 140-159 8 8 8 15 15 15 F F F
12402024-02-03T17:09:59.343Z 8 160-179 9 9 9 16 16 16 F F F
12412024-02-03T17:09:59.343Z 9 180-199 8 8 8 15 15 15 F F F
12422024-02-03T17:09:59.343Z 10 200-219 1 1 1 1 1 1 F F F
12432024-02-03T17:09:59.343Z 11 220-239 2 2 2 4 4 4 F F F
12442024-02-03T17:09:59.343Z 12 240-259 7 7 7 14 14 14 F F F
12452024-02-03T17:09:59.343Z 13 260-279 7 7 7 14 14 14 T T T
12462024-02-03T17:09:59.343Z 14 280-299 8 8 8 15 15 15 F F F
12472024-02-03T17:09:59.343Z 15 300-319 7 7 7 14 14 14 T T T
12482024-02-03T17:09:59.344Z 16 320-339 8 8 8 15 15 15 F F F
12492024-02-03T17:09:59.344Z 17 340-359 1 1 1 1 1 1 T F T <---
12502024-02-03T17:09:59.344Z 18 360-379 9 9 9 16 16 16 F F F
12512024-02-03T17:09:59.344Z 19 380-399 9 9 9 16 16 16 T T T
12522024-02-03T17:09:59.344Z 20 400-419 1 1 1 1 1 1 T F T <---
12532024-02-03T17:09:59.344Z 21 420-439 9 9 9 16 16 16 F F F
12542024-02-03T17:09:59.344Z 22 440-459 8 8 8 15 15 15 F F F
12552024-02-03T17:09:59.344Z 23 460-479 4 4 4 10 10 10 F F F
12562024-02-03T17:09:59.344Z 24 480-499 9 9 9 16 16 16 T F T <---
12572024-02-03T17:09:59.344Z 25 500-519 4 4 4 10 10 10 F F F
12582024-02-03T17:09:59.344Z 26 520-539 4 4 4 9 9 9 F F F
12592024-02-03T17:09:59.344Z 27 540-559 6 6 6 13 13 13 F F F
12602024-02-03T17:09:59.344Z 28 560-579 4 4 4 10 10 10 F F F
12612024-02-03T17:09:59.345Z 29 580-599 7 7 7 14 14 14 F F F
12622024-02-03T17:09:59.345ZMax gen: 9, Max flush: 16
12632024-02-03T17:09:59.345ZError: Difference in extent metadata found!
12642024-02-03T17:09:59.348ZA Difference in extent metadata is expected here
12652024-02-03T17:09:59.348ZOn loop 5
12662024-02-03T17:09:59.348Z
12672024-02-03T17:09:59.348ZVerifying data now
12682024-02-03T17:10:05.143ZNo change after verify
12692024-02-03T17:10:05.143ZLoop: 5 Downstairs dump after verify (and repair):
12702024-02-03T17:10:05.167ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12712024-02-03T17:10:05.167Z 0 000-019 7 7 7 14 14 14 F F F
12722024-02-03T17:10:05.167Z 1 020-039 10 10 10 17 17 17 F F F
12732024-02-03T17:10:05.167Z 2 040-059 6 6 6 13 13 13 F F F
12742024-02-03T17:10:05.167Z 3 060-079 6 6 6 12 12 12 F F F
12752024-02-03T17:10:05.170Z 4 080-099 10 10 10 17 17 17 F F F
12762024-02-03T17:10:05.171Z 5 100-119 7 7 7 14 14 14 F F F
12772024-02-03T17:10:05.171Z 6 120-139 4 4 4 10 10 10 F F F
12782024-02-03T17:10:05.171Z 7 140-159 8 8 8 15 15 15 F F F
12792024-02-03T17:10:05.171Z 8 160-179 9 9 9 16 16 16 F F F
12802024-02-03T17:10:05.171Z 9 180-199 8 8 8 15 15 15 F F F
12812024-02-03T17:10:05.171Z 10 200-219 1 1 1 1 1 1 F F F
12822024-02-03T17:10:05.171Z 11 220-239 2 2 2 4 4 4 F F F
12832024-02-03T17:10:05.171Z 12 240-259 7 7 7 14 14 14 F F F
12842024-02-03T17:10:05.171Z 13 260-279 10 10 10 17 17 17 F F F
12852024-02-03T17:10:05.171Z 14 280-299 8 8 8 15 15 15 F F F
12862024-02-03T17:10:05.171Z 15 300-319 10 10 10 17 17 17 F F F
12872024-02-03T17:10:05.171Z 16 320-339 8 8 8 15 15 15 F F F
12882024-02-03T17:10:05.171Z 17 340-359 10 10 10 17 17 17 F F F
12892024-02-03T17:10:05.171Z 18 360-379 9 9 9 16 16 16 F F F
12902024-02-03T17:10:05.171Z 19 380-399 10 10 10 17 17 17 F F F
12912024-02-03T17:10:05.171Z 20 400-419 10 10 10 17 17 17 F F F
12922024-02-03T17:10:05.171Z 21 420-439 9 9 9 16 16 16 F F F
12932024-02-03T17:10:05.171Z 22 440-459 8 8 8 15 15 15 F F F
12942024-02-03T17:10:05.171Z 23 460-479 4 4 4 10 10 10 F F F
12952024-02-03T17:10:05.171Z 24 480-499 10 10 10 17 17 17 F F F
12962024-02-03T17:10:05.171Z 25 500-519 4 4 4 10 10 10 F F F
12972024-02-03T17:10:05.171Z 26 520-539 4 4 4 9 9 9 F F F
12982024-02-03T17:10:05.171Z 27 540-559 6 6 6 13 13 13 F F F
12992024-02-03T17:10:05.172Z 28 560-579 4 4 4 10 10 10 F F F
13002024-02-03T17:10:05.172Z 29 580-599 7 7 7 14 14 14 F F F
13012024-02-03T17:10:05.172ZMax gen: 10, Max flush: 17
13022024-02-03T17:10:05.175Z
13032024-02-03T17:10:05.175ZBegin loop 6
13042024-02-03T17:10:05.175ZDownstairs to restart: 1
13052024-02-03T17:10:05.205Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.743125892Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177}
13062024-02-03T17:10:05.205Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.743563792Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177}
13072024-02-03T17:10:05.209Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.743648836Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177}
13082024-02-03T17:10:05.213Z{"msg":"Crucible afa7e68f-8217-4db4-9b35-f1c32779ca58 has session id: 8d245f27-60ce-4ced-96de-0dfd32565d25","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.750069692Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13092024-02-03T17:10:05.213Z{"msg":"Upstairs opts: Upstairs UUID: afa7e68f-8217-4db4-9b35-f1c32779ca58, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.750153448Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13102024-02-03T17:10:05.213Z{"msg":"Crucible stats registered with UUID: afa7e68f-8217-4db4-9b35-f1c32779ca58","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.750181638Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13112024-02-03T17:10:05.213ZCrucible runtime is spawned
13122024-02-03T17:10:05.213Z{"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.750663407Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177}
13132024-02-03T17:10:05.213Z{"msg":"afa7e68f-8217-4db4-9b35-f1c32779ca58 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.750860096Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13142024-02-03T17:10:05.213Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.750921953Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13152024-02-03T17:10:05.213Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.750965012Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13162024-02-03T17:10:05.213Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.751048969Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13172024-02-03T17:10:05.213Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.751093605Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13182024-02-03T17:10:05.213Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.751133653Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13192024-02-03T17:10:05.213Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.751251751Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13202024-02-03T17:10:05.213Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.751282342Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"":"io task","client":"0","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13212024-02-03T17:10:05.213Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.751310194Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"":"io task","client":"1","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13222024-02-03T17:10:05.213Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.751573707Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"":"io task","client":"0","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13232024-02-03T17:10:05.213Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.75172454Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"":"io task","client":"1","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13242024-02-03T17:10:05.213Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.75185087Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"":"io task","client":"2","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13252024-02-03T17:10:05.213Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.751902533Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"":"io task","client":"2","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13262024-02-03T17:10:05.219Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.753055701Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13272024-02-03T17:10:05.219Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.753165348Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13282024-02-03T17:10:05.219Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.753192614Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13292024-02-03T17:10:05.219Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.753216359Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13302024-02-03T17:10:05.219Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.753238588Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13312024-02-03T17:10:05.220Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.753408891Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13322024-02-03T17:10:05.220Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.754561163Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13332024-02-03T17:10:05.220Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.754695554Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13342024-02-03T17:10:05.220Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.754828294Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13352024-02-03T17:10:05.220Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.755672691Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13362024-02-03T17:10:05.220Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.755775212Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13372024-02-03T17:10:05.220Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.755824907Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13382024-02-03T17:10:05.220Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.755946432Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13392024-02-03T17:10:05.220Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.755997893Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13402024-02-03T17:10:05.220Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.756074704Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13412024-02-03T17:10:05.220Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.756141689Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13422024-02-03T17:10:05.220Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.756207965Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13432024-02-03T17:10:05.220Z{"msg":"[0]R flush_numbers[0..12]: [14, 17, 13, 12, 17, 14, 10, 15, 16, 15, 1, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.756254946Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13442024-02-03T17:10:05.220Z{"msg":"[0]R generation[0..12]: [7, 10, 6, 6, 10, 7, 4, 8, 9, 8, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.756298469Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13452024-02-03T17:10:05.220Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.756371554Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13462024-02-03T17:10:05.220Z{"msg":"[1]R flush_numbers[0..12]: [14, 17, 13, 12, 17, 14, 10, 15, 16, 15, 1, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.756418712Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13472024-02-03T17:10:05.220ZWait for a query_work_queue command to finish before sending IO
13482024-02-03T17:10:05.220Z{"msg":"[1]R generation[0..12]: [7, 10, 6, 6, 10, 7, 4, 8, 9, 8, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.756483255Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13492024-02-03T17:10:05.220Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.756530214Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13502024-02-03T17:10:05.220Z{"msg":"[2]R flush_numbers[0..12]: [14, 17, 13, 12, 17, 14, 10, 15, 16, 15, 1, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.756603672Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13512024-02-03T17:10:05.220Z{"msg":"[2]R generation[0..12]: [7, 10, 6, 6, 10, 7, 4, 8, 9, 8, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.756649888Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13522024-02-03T17:10:05.220Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.756717897Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13532024-02-03T17:10:05.220Z{"msg":"Max found gen is 11","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.756761716Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13542024-02-03T17:10:05.221Z{"msg":"Generation requested: 12 >= found:11","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.756829793Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13552024-02-03T17:10:05.221Z{"msg":"Next flush: 18","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.756873106Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13562024-02-03T17:10:05.221Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.756940397Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13572024-02-03T17:10:05.221Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.756984823Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13582024-02-03T17:10:05.221Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.757043816Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13592024-02-03T17:10:05.221Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.757070165Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13602024-02-03T17:10:05.221Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.757092317Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13612024-02-03T17:10:05.221Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.757144418Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13622024-02-03T17:10:05.221Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.757167958Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13632024-02-03T17:10:05.221Z{"msg":"afa7e68f-8217-4db4-9b35-f1c32779ca58 is now active with session: 8d245f27-60ce-4ced-96de-0dfd32565d25","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.7571895Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13642024-02-03T17:10:05.221Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.757209212Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177,"session_id":"8d245f27-60ce-4ced-96de-0dfd32565d25"}
13652024-02-03T17:10:05.221Z{"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:04.757229636Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177}
13662024-02-03T17:10:05.221ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
13672024-02-03T17:10:05.225ZLoading write count information from file "/tmp/test_repair_verify.data"
13682024-02-03T17:10:05.225ZRun Repair workload
13692024-02-03T17:10:05.225Z01/30 Write block 503 len 20480 data: 2 2 2 2 2
13702024-02-03T17:10:05.225Z02/30 Write block 353 len 36864 data: 3 3 3 2 2 2 2 2 2
13712024-02-03T17:10:05.225Z03/30 Write block 190 len 24576 data: 3 3 3 2 2 2
13722024-02-03T17:10:05.225Z04/30 Write block 505 len 28672 data: 3 3 3 3 3 4 4
13732024-02-03T17:10:05.229Z05/30 Write block 2 len 40960 data: 2 3 3 3 3 3 3 4 4 3
13742024-02-03T17:10:05.229Z06/30 Read block 286 len 8192
13752024-02-03T17:10:05.229Z07/30 Read block 59 len 24576
13762024-02-03T17:10:05.233Z08/30 Read block 211 len 24576
13772024-02-03T17:10:05.233Z09/30 Read block 337 len 4096
13782024-02-03T17:10:05.236Z10/30 Flush
13792024-02-03T17:10:05.242Z11/30 Write block 336 len 12288 data: 2 2 2
13802024-02-03T17:10:05.242Z12/30 Read block 1 len 12288
13812024-02-03T17:10:05.242Z13/30 Read block 493 len 28672
13822024-02-03T17:10:05.245Z14/30 Read block 348 len 20480
13832024-02-03T17:10:05.245Z15/30 Read block 98 len 36864
13842024-02-03T17:10:05.245Z16/30 Write block 267 len 28672 data: 3 3 2 2 3 3 3
13852024-02-03T17:10:05.246Z17/30 Write block 568 len 12288 data: 4 4 4
13862024-02-03T17:10:05.250Z18/30 Read block 185 len 16384
13872024-02-03T17:10:05.250Z19/30 Flush
13882024-02-03T17:10:05.259Z20/30 Write block 372 len 32768 data: 2 2 2 2 2 2 2 2
13892024-02-03T17:10:05.259Z21/30 Write block 6 len 4096 data: 4
13902024-02-03T17:10:05.262Z22/30 Write block 272 len 16384 data: 4 4 4 4
13912024-02-03T17:10:05.262Z23/30 Read block 460 len 36864
13922024-02-03T17:10:05.262Z24/30 Read block 452 len 12288
13932024-02-03T17:10:05.262Z25/30 Write block 332 len 8192 data: 3 3
13942024-02-03T17:10:05.263Z26/30 Write block 202 len 36864 data: 2 2 2 2 2 2 2 2 2
13952024-02-03T17:10:05.263Z27/30 Write block 248 len 16384 data: 3 3 3 3
13962024-02-03T17:10:05.263Z28/30 Read block 220 len 32768
13972024-02-03T17:10:05.266Z29/30 Read block 457 len 32768
13982024-02-03T17:10:05.266Z30/30 Read block 302 len 20480
13992024-02-03T17:10:05.270Z----------------------------------------------------------------
14002024-02-03T17:10:05.270Z Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30
14012024-02-03T17:10:05.270ZStates: Active Active Active
14022024-02-03T17:10:05.270ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
14032024-02-03T17:10:05.270Z 1 Acked 1000 Write 5 Done Done Done false
14042024-02-03T17:10:05.270Z 2 Acked 1001 Write 9 Done Done Done false
14052024-02-03T17:10:05.270Z 3 Acked 1002 Write 6 Done Done Done false
14062024-02-03T17:10:05.270Z 4 Acked 1003 Write 7 Done Done Done false
14072024-02-03T17:10:05.270Z 5 Acked 1004 Write 10 Done Sent Done false
14082024-02-03T17:10:05.270Z 6 Acked 1005 Read 2 Done Sent Done false
14092024-02-03T17:10:05.270Z 7 Acked 1006 Read 6 Done Sent Done false
14102024-02-03T17:10:05.270Z 8 Acked 1007 Read 6 Done Sent Done false
14112024-02-03T17:10:05.270Z 9 Acked 1008 Read 1 Done Sent Done false
14122024-02-03T17:10:05.270Z 10 Acked 1009 Flush 0 Done Sent Done false
14132024-02-03T17:10:05.270Z 11 Acked 1010 Write 3 Done Sent Done false
14142024-02-03T17:10:05.270Z 12 Acked 1011 Read 3 Done Sent Done false
14152024-02-03T17:10:05.270Z 13 Acked 1012 Read 7 Done Sent Done false
14162024-02-03T17:10:05.270Z 14 Acked 1013 Read 5 Done Sent Done false
14172024-02-03T17:10:05.270Z 15 Acked 1014 Read 9 Done Sent Done false
14182024-02-03T17:10:05.270Z 16 Acked 1015 Write 7 Done Sent Done false
14192024-02-03T17:10:05.270Z 17 Acked 1016 Write 3 Done Sent Done false
14202024-02-03T17:10:05.270Z 18 Acked 1017 Read 4 Done Sent Done false
14212024-02-03T17:10:05.270Z 19 Acked 1018 Flush 0 Done Sent Done false
14222024-02-03T17:10:05.271Z 20 Acked 1019 Write 8 Done Sent Done false
14232024-02-03T17:10:05.271Z 21 Acked 1020 Write 1 Done Sent Done false
14242024-02-03T17:10:05.271Z 22 Acked 1021 Write 4 Done Sent Done false
14252024-02-03T17:10:05.271Z 23 Acked 1022 Read 9 Done Sent Done false
14262024-02-03T17:10:05.271Z 24 Acked 1023 Read 3 Done Sent Done false
14272024-02-03T17:10:05.271Z 25 Acked 1024 Write 2 Done Sent Done false
14282024-02-03T17:10:05.271Z 26 Acked 1025 Write 9 Done Sent Done false
14292024-02-03T17:10:05.271Z 27 Acked 1026 Write 4 Done Sent Done false
14302024-02-03T17:10:05.271Z 28 Acked 1027 Read 8 Done Sent Done false
14312024-02-03T17:10:05.271Z 29 Acked 1028 Read 8 Done Sent Done false
14322024-02-03T17:10:05.271Z 30 Acked 1029 Read 5 Done Sent Done false
14332024-02-03T17:10:05.271Z STATES DS:0 DS:1 DS:2 TOTAL
14342024-02-03T17:10:05.271Z New 0 0 0 0
14352024-02-03T17:10:05.271Z Sent 0 26 0 26
14362024-02-03T17:10:05.271Z Done 30 4 30 64
14372024-02-03T17:10:05.271Z Skipped 0 0 0 0
14382024-02-03T17:10:05.271Z Error 0 0 0 0
14392024-02-03T17:10:05.271ZLast Flush: 1018 0 1018
14402024-02-03T17:10:05.271ZDownstairs last five completed:
14412024-02-03T17:10:05.271ZUpstairs last five completed: 30 29 28 27 26
14422024-02-03T17:10:05.271Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:10:04.807531481Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1177}
14432024-02-03T17:10:05.271ZWrote out file "/tmp/test_repair_verify.data"
14442024-02-03T17:10:05.277Z
14452024-02-03T17:10:05.282ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
14462024-02-03T17:10:05.311ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14472024-02-03T17:10:05.311Z 0 000-019 12 7 12 18 14 18 T F T <---
14482024-02-03T17:10:05.311Z 1 020-039 10 10 10 17 17 17 F F F
14492024-02-03T17:10:05.311Z 2 040-059 6 6 6 13 13 13 F F F
14502024-02-03T17:10:05.311Z 3 060-079 6 6 6 12 12 12 F F F
14512024-02-03T17:10:05.311Z 4 080-099 10 10 10 17 17 17 F F F
14522024-02-03T17:10:05.311Z 5 100-119 7 7 7 14 14 14 F F F
14532024-02-03T17:10:05.311Z 6 120-139 4 4 4 10 10 10 F F F
14542024-02-03T17:10:05.311Z 7 140-159 8 8 8 15 15 15 F F F
14552024-02-03T17:10:05.311Z 8 160-179 9 9 9 16 16 16 F F F
14562024-02-03T17:10:05.311Z 9 180-199 12 8 12 18 15 18 F T F <---
14572024-02-03T17:10:05.311Z 10 200-219 1 1 1 1 1 1 T F T <---
14582024-02-03T17:10:05.311Z 11 220-239 2 2 2 4 4 4 F F F
14592024-02-03T17:10:05.315Z 12 240-259 7 7 7 14 14 14 T F T <---
14602024-02-03T17:10:05.315Z 13 260-279 12 10 12 19 17 19 T F T <---
14612024-02-03T17:10:05.316Z 14 280-299 8 8 8 15 15 15 F F F
14622024-02-03T17:10:05.316Z 15 300-319 10 10 10 17 17 17 F F F
14632024-02-03T17:10:05.316Z 16 320-339 12 8 12 19 15 19 T F T <---
14642024-02-03T17:10:05.316Z 17 340-359 12 10 12 18 17 18 F T F <---
14652024-02-03T17:10:05.316Z 18 360-379 12 9 12 18 16 18 T T T <---
14662024-02-03T17:10:05.316Z 19 380-399 10 10 10 17 17 17 F F F
14672024-02-03T17:10:05.316Z 20 400-419 10 10 10 17 17 17 F F F
14682024-02-03T17:10:05.316Z 21 420-439 9 9 9 16 16 16 F F F
14692024-02-03T17:10:05.316Z 22 440-459 8 8 8 15 15 15 F F F
14702024-02-03T17:10:05.316Z 23 460-479 4 4 4 10 10 10 F F F
14712024-02-03T17:10:05.316Z 24 480-499 10 10 10 17 17 17 F F F
14722024-02-03T17:10:05.316Z 25 500-519 12 4 12 18 10 18 F T F <---
14732024-02-03T17:10:05.316Z 26 520-539 4 4 4 9 9 9 F F F
14742024-02-03T17:10:05.316Z 27 540-559 6 6 6 13 13 13 F F F
14752024-02-03T17:10:05.316Z 28 560-579 12 4 12 19 10 19 F F F <---
14762024-02-03T17:10:05.316Z 29 580-599 7 7 7 14 14 14 F F F
14772024-02-03T17:10:05.316ZMax gen: 12, Max flush: 19
14782024-02-03T17:10:05.316ZError: Difference in extent metadata found!
14792024-02-03T17:10:05.316ZA Difference in extent metadata is expected here
14802024-02-03T17:10:05.316ZOn loop 6
14812024-02-03T17:10:05.316Z
14822024-02-03T17:10:05.320ZVerifying data now
14832024-02-03T17:10:11.247ZNo change after verify
14842024-02-03T17:10:11.247ZLoop: 6 Downstairs dump after verify (and repair):
14852024-02-03T17:10:11.266ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14862024-02-03T17:10:11.266Z 0 000-019 13 13 13 20 20 20 F F F
14872024-02-03T17:10:11.266Z 1 020-039 10 10 10 17 17 17 F F F
14882024-02-03T17:10:11.266Z 2 040-059 6 6 6 13 13 13 F F F
14892024-02-03T17:10:11.266Z 3 060-079 6 6 6 12 12 12 F F F
14902024-02-03T17:10:11.274Z 4 080-099 10 10 10 17 17 17 F F F
14912024-02-03T17:10:11.274Z 5 100-119 7 7 7 14 14 14 F F F
14922024-02-03T17:10:11.274Z 6 120-139 4 4 4 10 10 10 F F F
14932024-02-03T17:10:11.275Z 7 140-159 8 8 8 15 15 15 F F F
14942024-02-03T17:10:11.275Z 8 160-179 9 9 9 16 16 16 F F F
14952024-02-03T17:10:11.275Z 9 180-199 12 12 12 18 18 18 F F F
14962024-02-03T17:10:11.275Z 10 200-219 13 13 13 20 20 20 F F F
14972024-02-03T17:10:11.275Z 11 220-239 2 2 2 4 4 4 F F F
14982024-02-03T17:10:11.275Z 12 240-259 13 13 13 20 20 20 F F F
14992024-02-03T17:10:11.275Z 13 260-279 13 13 13 20 20 20 F F F
15002024-02-03T17:10:11.275Z 14 280-299 8 8 8 15 15 15 F F F
15012024-02-03T17:10:11.275Z 15 300-319 10 10 10 17 17 17 F F F
15022024-02-03T17:10:11.275Z 16 320-339 13 13 13 20 20 20 F F F
15032024-02-03T17:10:11.275Z 17 340-359 12 12 12 18 18 18 F F F
15042024-02-03T17:10:11.275Z 18 360-379 13 13 13 20 20 20 F F F
15052024-02-03T17:10:11.275Z 19 380-399 10 10 10 17 17 17 F F F
15062024-02-03T17:10:11.276Z 20 400-419 10 10 10 17 17 17 F F F
15072024-02-03T17:10:11.276Z 21 420-439 9 9 9 16 16 16 F F F
15082024-02-03T17:10:11.276Z 22 440-459 8 8 8 15 15 15 F F F
15092024-02-03T17:10:11.276Z 23 460-479 4 4 4 10 10 10 F F F
15102024-02-03T17:10:11.276Z 24 480-499 10 10 10 17 17 17 F F F
15112024-02-03T17:10:11.276Z 25 500-519 12 12 12 18 18 18 F F F
15122024-02-03T17:10:11.276Z 26 520-539 4 4 4 9 9 9 F F F
15132024-02-03T17:10:11.276Z 27 540-559 6 6 6 13 13 13 F F F
15142024-02-03T17:10:11.276Z 28 560-579 12 12 12 19 19 19 F F F
15152024-02-03T17:10:11.276Z 29 580-599 7 7 7 14 14 14 F F F
15162024-02-03T17:10:11.276ZMax gen: 13, Max flush: 20
15172024-02-03T17:10:11.280Z
15182024-02-03T17:10:11.280ZBegin loop 7
15192024-02-03T17:10:11.280ZDownstairs to restart: 2
15202024-02-03T17:10:11.303Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.842573896Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185}
15212024-02-03T17:10:11.303Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.843198149Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185}
15222024-02-03T17:10:11.303Z{"msg":"Crucible runtime is spawned
15232024-02-03T17:10:11.307ZUpstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.843428281Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185}
15242024-02-03T17:10:11.307Z{"msg":"Crucible e05c2c39-1539-4659-bd67-069e14b0091a has session id: 5c26922f-483a-4947-8e12-aa74c3389f50","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.843512339Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15252024-02-03T17:10:11.308Z{"msg":"Upstairs opts: Upstairs UUID: e05c2c39-1539-4659-bd67-069e14b0091a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.843578185Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15262024-02-03T17:10:11.308Z{"msg":"Crucible stats registered with UUID: e05c2c39-1539-4659-bd67-069e14b0091a","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.843655145Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15272024-02-03T17:10:11.308Z{"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.843709909Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185}
15282024-02-03T17:10:11.308Z{"msg":"e05c2c39-1539-4659-bd67-069e14b0091a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.843838188Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15292024-02-03T17:10:11.308Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.843863867Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15302024-02-03T17:10:11.308Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.843898043Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15312024-02-03T17:10:11.308Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.843964728Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15322024-02-03T17:10:11.308Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.84400614Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15332024-02-03T17:10:11.308Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.844133581Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15342024-02-03T17:10:11.308Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.844217472Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15352024-02-03T17:10:11.308Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.844265187Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15362024-02-03T17:10:11.308Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.844366823Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15372024-02-03T17:10:11.308Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.844398711Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15382024-02-03T17:10:11.308Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.84442228Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15392024-02-03T17:10:11.308Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.844445259Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15402024-02-03T17:10:11.308Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.844467423Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15412024-02-03T17:10:11.308Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.845661759Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15422024-02-03T17:10:11.308Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.845916944Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15432024-02-03T17:10:11.308Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.846067737Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15442024-02-03T17:10:11.309Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.846130189Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15452024-02-03T17:10:11.309Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.846569168Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15462024-02-03T17:10:11.309Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.846709893Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15472024-02-03T17:10:11.314Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.846973986Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15482024-02-03T17:10:11.314Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.847328386Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15492024-02-03T17:10:11.314Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.847829044Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15502024-02-03T17:10:11.314Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.847929645Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15512024-02-03T17:10:11.314Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.848019804Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15522024-02-03T17:10:11.314Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.848065543Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15532024-02-03T17:10:11.315Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.848423766Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15542024-02-03T17:10:11.315Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.848527052Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15552024-02-03T17:10:11.315Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.848557426Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15562024-02-03T17:10:11.315Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.848753605Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15572024-02-03T17:10:11.315Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.848879981Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15582024-02-03T17:10:11.315Z{"msg":"[0]R flush_numbers[0..12]: [20, 17, 13, 12, 17, 14, 10, 15, 16, 18, 20, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.84892991Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15592024-02-03T17:10:11.315Z{"msg":"[0]R generation[0..12]: [13, 10, 6, 6, 10, 7, 4, 8, 9, 12, 13, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.848973236Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15602024-02-03T17:10:11.315Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.849043815Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15612024-02-03T17:10:11.315Z{"msg":"[1]R flush_numbers[0..12]: [20, 17, 13, 12, 17, 14, 10, 15, 16, 18, 20, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.849073301Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15622024-02-03T17:10:11.315Z{"msg":"[1]R generation[0..12]: [13, 10, 6, 6, 10, 7, 4, 8, 9, 12, 13, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.849096299Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15632024-02-03T17:10:11.315Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO
15642024-02-03T17:10:11.315Z,"level":30,"time":"2024-02-03T17:10:10.84917367Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15652024-02-03T17:10:11.315Z{"msg":"[2]R flush_numbers[0..12]: [20, 17, 13, 12, 17, 14, 10, 15, 16, 18, 20, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.849200153Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15662024-02-03T17:10:11.316Z{"msg":"[2]R generation[0..12]: [13, 10, 6, 6, 10, 7, 4, 8, 9, 12, 13, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.849249061Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15672024-02-03T17:10:11.316Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.849275179Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15682024-02-03T17:10:11.316Z{"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.849296821Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15692024-02-03T17:10:11.316Z{"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.849343398Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15702024-02-03T17:10:11.316Z{"msg":"Next flush: 21","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.84936663Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15712024-02-03T17:10:11.316Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.849388483Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15722024-02-03T17:10:11.316Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.849410249Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15732024-02-03T17:10:11.316Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.849430259Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15742024-02-03T17:10:11.316Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.849452127Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15752024-02-03T17:10:11.316Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.849500232Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15762024-02-03T17:10:11.316Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.849524328Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15772024-02-03T17:10:11.316Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.84954472Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15782024-02-03T17:10:11.316Z{"msg":"e05c2c39-1539-4659-bd67-069e14b0091a is now active with session: 5c26922f-483a-4947-8e12-aa74c3389f50","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.849590716Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15792024-02-03T17:10:11.316Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.84961293Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185,"session_id":"5c26922f-483a-4947-8e12-aa74c3389f50"}
15802024-02-03T17:10:11.316Z{"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:10.849633651Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185}
15812024-02-03T17:10:11.316ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
15822024-02-03T17:10:11.316ZLoading write count information from file "/tmp/test_repair_verify.data"
15832024-02-03T17:10:11.317ZRun Repair workload
15842024-02-03T17:10:11.317Z01/30 Write block 421 len 24576 data: 2 2 2 2 2 2
15852024-02-03T17:10:11.321Z02/30 Read block 143 len 12288
15862024-02-03T17:10:11.321Z03/30 Write block 398 len 24576 data: 2 2 2 2 2 2
15872024-02-03T17:10:11.321Z04/30 Read block 581 len 32768
15882024-02-03T17:10:11.321Z05/30 Write block 334 len 8192 data: 3 2
15892024-02-03T17:10:11.321Z06/30 Write block 275 len 24576 data: 5 4 4 3 3 3
15902024-02-03T17:10:11.324Z07/30 Read block 110 len 36864
15912024-02-03T17:10:11.325Z08/30 Write block 241 len 24576 data: 2 2 2 2 2 2
15922024-02-03T17:10:11.325Z09/30 Write block 563 len 8192 data: 3 3
15932024-02-03T17:10:11.328Z10/30 Read block 493 len 16384
15942024-02-03T17:10:11.328Z11/30 Flush
15952024-02-03T17:10:11.332Z12/30 Write block 465 len 8192 data: 2 2
15962024-02-03T17:10:11.336Z13/30 Flush
15972024-02-03T17:10:11.336Z14/30 Write block 463 len 20480 data: 2 2 3 3 2
15982024-02-03T17:10:11.339Z15/30 Write block 282 len 16384 data: 2 4 4 4
15992024-02-03T17:10:11.339Z16/30 Read block 12 len 32768
16002024-02-03T17:10:11.343Z17/30 Flush
16012024-02-03T17:10:11.343Z18/30 Write block 210 len 40960 data: 3 2 2 2 2 2 2 2 2 2
16022024-02-03T17:10:11.347Z19/30 Read block 63 len 40960
16032024-02-03T17:10:11.351Z20/30 Write block 320 len 20480 data: 3 4 4 4 2
16042024-02-03T17:10:11.351Z21/30 Write block 586 len 20480 data: 2 2 2 2 3
16052024-02-03T17:10:11.351Z22/30 Write block 275 len 28672 data: 6 5 5 4 4 4 2
16062024-02-03T17:10:11.351Z23/30 Write block 151 len 24576 data: 2 2 2 2 2 2
16072024-02-03T17:10:11.356Z24/30 Write block 572 len 40960 data: 3 3 3 3 3 2 2 2 2 2
16082024-02-03T17:10:11.357Z25/30 Read block 327 len 32768
16092024-02-03T17:10:11.360Z26/30 Write block 413 len 32768 data: 2 3 3 2 2 2 2 2
16102024-02-03T17:10:11.360Z27/30 Read block 139 len 8192
16112024-02-03T17:10:11.360Z28/30 Read block 425 len 8192
16122024-02-03T17:10:11.360Z29/30 Read block 557 len 24576
16132024-02-03T17:10:11.364Z30/30 Read block 220 len 12288
16142024-02-03T17:10:11.364Z----------------------------------------------------------------
16152024-02-03T17:10:11.364Z Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:30
16162024-02-03T17:10:11.364ZStates: Active Active Active
16172024-02-03T17:10:11.364ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
16182024-02-03T17:10:11.365Z 1 Acked 1000 Write 6 Done Done Done false
16192024-02-03T17:10:11.365Z 2 Acked 1001 Read 3 Done Done Done false
16202024-02-03T17:10:11.365Z 3 Acked 1002 Write 6 Done Done Done false
16212024-02-03T17:10:11.365Z 4 Acked 1003 Read 8 Done Done Done false
16222024-02-03T17:10:11.365Z 5 Acked 1004 Write 2 Done Done Done false
16232024-02-03T17:10:11.365Z 6 Acked 1005 Write 6 Done Done Done false
16242024-02-03T17:10:11.365Z 7 Acked 1006 Read 9 Done Done Done false
16252024-02-03T17:10:11.365Z 8 Acked 1007 Write 6 Done Done Done false
16262024-02-03T17:10:11.365Z 9 Acked 1008 Write 2 Done Done Done false
16272024-02-03T17:10:11.365Z 10 Acked 1009 Read 4 Done Done Done false
16282024-02-03T17:10:11.365Z 11 Acked 1010 Flush 0 Done Done Sent false
16292024-02-03T17:10:11.365Z 12 Acked 1011 Write 2 Done Done Sent false
16302024-02-03T17:10:11.366Z 13 Acked 1012 Flush 0 Done Done Sent false
16312024-02-03T17:10:11.366Z 14 Acked 1013 Write 5 Done Done Sent false
16322024-02-03T17:10:11.366Z 15 Acked 1014 Write 4 Done Done Sent false
16332024-02-03T17:10:11.366Z 16 Acked 1015 Read 8 Done Done Sent false
16342024-02-03T17:10:11.366Z 17 Acked 1016 Flush 0 Done Done Sent false
16352024-02-03T17:10:11.366Z 18 Acked 1017 Write 10 Done Done Sent false
16362024-02-03T17:10:11.366Z 19 Acked 1018 Read 10 Done Done Sent false
16372024-02-03T17:10:11.366Z 20 Acked 1019 Write 5 Done Done Sent false
16382024-02-03T17:10:11.366Z 21 Acked 1020 Write 5 Done Done Sent false
16392024-02-03T17:10:11.366Z 22 Acked 1021 Write 7 Done Done Sent false
16402024-02-03T17:10:11.366Z 23 Acked 1022 Write 6 Done Done Sent false
16412024-02-03T17:10:11.366Z 24 Acked 1023 Write 10 Done Done Sent false
16422024-02-03T17:10:11.366Z 25 Acked 1024 Read 8 Done Done Sent false
16432024-02-03T17:10:11.367Z 26 Acked 1025 Write 8 Done Done Sent false
16442024-02-03T17:10:11.367Z 27 Acked 1026 Read 2 Done Done Sent false
16452024-02-03T17:10:11.367Z 28 Acked 1027 Read 2 Done Done Sent false
16462024-02-03T17:10:11.367Z 29 Acked 1028 Read 6 Done Done Sent false
16472024-02-03T17:10:11.367Z 30 Acked 1029 Read 3 Done Done Sent false
16482024-02-03T17:10:11.367Z STATES DS:0 DS:1 DS:2 TOTAL
16492024-02-03T17:10:11.367Z New 0 0 0 0
16502024-02-03T17:10:11.367Z Sent 0 0 20 20
16512024-02-03T17:10:11.367Z Done 30 30 10 70
16522024-02-03T17:10:11.367Z Skipped 0 0 0 0
16532024-02-03T17:10:11.367Z Error 0 0 0 0
16542024-02-03T17:10:11.367ZLast Flush: 1016 1016 0
16552024-02-03T17:10:11.367ZDownstairs last five completed:
16562024-02-03T17:10:11.368ZUpstairs last five completed: 30 29 28 27 26
16572024-02-03T17:10:11.368Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:10:10.904114488Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1185}
16582024-02-03T17:10:11.372ZWrote out file "/tmp/test_repair_verify.data"
16592024-02-03T17:10:11.372Z
16602024-02-03T17:10:11.377ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
16612024-02-03T17:10:11.408ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
16622024-02-03T17:10:11.408Z 0 000-019 13 13 13 20 20 20 F F F
16632024-02-03T17:10:11.408Z 1 020-039 10 10 10 17 17 17 F F F
16642024-02-03T17:10:11.408Z 2 040-059 6 6 6 13 13 13 F F F
16652024-02-03T17:10:11.408Z 3 060-079 6 6 6 12 12 12 F F F
16662024-02-03T17:10:11.412Z 4 080-099 10 10 10 17 17 17 F F F
16672024-02-03T17:10:11.412Z 5 100-119 7 7 7 14 14 14 F F F
16682024-02-03T17:10:11.412Z 6 120-139 4 4 4 10 10 10 F F F
16692024-02-03T17:10:11.412Z 7 140-159 8 8 8 15 15 15 T T F <---
16702024-02-03T17:10:11.412Z 8 160-179 9 9 9 16 16 16 F F F
16712024-02-03T17:10:11.412Z 9 180-199 12 12 12 18 18 18 F F F
16722024-02-03T17:10:11.412Z 10 200-219 13 13 13 20 20 20 T T F <---
16732024-02-03T17:10:11.412Z 11 220-239 2 2 2 4 4 4 F F F
16742024-02-03T17:10:11.412Z 12 240-259 14 14 13 21 21 20 F F T <---
16752024-02-03T17:10:11.413Z 13 260-279 14 14 13 21 21 20 T T T <---
16762024-02-03T17:10:11.413Z 14 280-299 14 14 8 23 23 15 T T T <---
16772024-02-03T17:10:11.413Z 15 300-319 10 10 10 17 17 17 F F F
16782024-02-03T17:10:11.413Z 16 320-339 14 14 13 21 21 20 T T T <---
16792024-02-03T17:10:11.413Z 17 340-359 12 12 12 18 18 18 F F F
16802024-02-03T17:10:11.413Z 18 360-379 13 13 13 20 20 20 F F F
16812024-02-03T17:10:11.413Z 19 380-399 14 14 10 21 21 17 F F T <---
16822024-02-03T17:10:11.413Z 20 400-419 14 14 10 21 21 17 T T T <---
16832024-02-03T17:10:11.413Z 21 420-439 14 14 9 21 21 16 T T T <---
16842024-02-03T17:10:11.413Z 22 440-459 8 8 8 15 15 15 F F F
16852024-02-03T17:10:11.413Z 23 460-479 14 14 4 23 23 10 F F F <---
16862024-02-03T17:10:11.413Z 24 480-499 10 10 10 17 17 17 F F F
16872024-02-03T17:10:11.413Z 25 500-519 12 12 12 18 18 18 F F F
16882024-02-03T17:10:11.414Z 26 520-539 4 4 4 9 9 9 F F F
16892024-02-03T17:10:11.414Z 27 540-559 6 6 6 13 13 13 F F F
16902024-02-03T17:10:11.414Z 28 560-579 14 14 12 21 21 19 T T T <---
16912024-02-03T17:10:11.414Z 29 580-599 7 7 7 14 14 14 T T F <---
16922024-02-03T17:10:11.414ZMax gen: 14, Max flush: 23
16932024-02-03T17:10:11.414ZError: Difference in extent metadata found!
16942024-02-03T17:10:11.418ZA Difference in extent metadata is expected here
16952024-02-03T17:10:11.418ZOn loop 7
16962024-02-03T17:10:11.418Z
16972024-02-03T17:10:11.418ZVerifying data now
16982024-02-03T17:10:17.502ZNo change after verify
16992024-02-03T17:10:17.502ZLoop: 7 Downstairs dump after verify (and repair):
17002024-02-03T17:10:17.526ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
17012024-02-03T17:10:17.526Z 0 000-019 13 13 13 20 20 20 F F F
17022024-02-03T17:10:17.526Z 1 020-039 10 10 10 17 17 17 F F F
17032024-02-03T17:10:17.526Z 2 040-059 6 6 6 13 13 13 F F F
17042024-02-03T17:10:17.526Z 3 060-079 6 6 6 12 12 12 F F F
17052024-02-03T17:10:17.530Z 4 080-099 10 10 10 17 17 17 F F F
17062024-02-03T17:10:17.530Z 5 100-119 7 7 7 14 14 14 F F F
17072024-02-03T17:10:17.530Z 6 120-139 4 4 4 10 10 10 F F F
17082024-02-03T17:10:17.530Z 7 140-159 15 15 15 24 24 24 F F F
17092024-02-03T17:10:17.530Z 8 160-179 9 9 9 16 16 16 F F F
17102024-02-03T17:10:17.530Z 9 180-199 12 12 12 18 18 18 F F F
17112024-02-03T17:10:17.530Z 10 200-219 15 15 15 24 24 24 F F F
17122024-02-03T17:10:17.530Z 11 220-239 2 2 2 4 4 4 F F F
17132024-02-03T17:10:17.530Z 12 240-259 14 14 14 21 21 21 F F F
17142024-02-03T17:10:17.530Z 13 260-279 15 15 15 24 24 24 F F F
17152024-02-03T17:10:17.530Z 14 280-299 15 15 15 24 24 24 F F F
17162024-02-03T17:10:17.530Z 15 300-319 10 10 10 17 17 17 F F F
17172024-02-03T17:10:17.531Z 16 320-339 15 15 15 24 24 24 F F F
17182024-02-03T17:10:17.531Z 17 340-359 12 12 12 18 18 18 F F F
17192024-02-03T17:10:17.531Z 18 360-379 13 13 13 20 20 20 F F F
17202024-02-03T17:10:17.531Z 19 380-399 14 14 14 21 21 21 F F F
17212024-02-03T17:10:17.531Z 20 400-419 15 15 15 24 24 24 F F F
17222024-02-03T17:10:17.531Z 21 420-439 15 15 15 24 24 24 F F F
17232024-02-03T17:10:17.531Z 22 440-459 8 8 8 15 15 15 F F F
17242024-02-03T17:10:17.531Z 23 460-479 14 14 14 23 23 23 F F F
17252024-02-03T17:10:17.531Z 24 480-499 10 10 10 17 17 17 F F F
17262024-02-03T17:10:17.531Z 25 500-519 12 12 12 18 18 18 F F F
17272024-02-03T17:10:17.531Z 26 520-539 4 4 4 9 9 9 F F F
17282024-02-03T17:10:17.531Z 27 540-559 6 6 6 13 13 13 F F F
17292024-02-03T17:10:17.531Z 28 560-579 15 15 15 24 24 24 F F F
17302024-02-03T17:10:17.532Z 29 580-599 15 15 15 24 24 24 F F F
17312024-02-03T17:10:17.532ZMax gen: 15, Max flush: 24
17322024-02-03T17:10:17.535Z
17332024-02-03T17:10:17.535ZBegin loop 8
17342024-02-03T17:10:17.535ZDownstairs to restart: 0
17352024-02-03T17:10:17.564Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.104216797Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193}
17362024-02-03T17:10:17.564Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.104783182Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193}
17372024-02-03T17:10:17.564ZCrucible runtime is spawned
17382024-02-03T17:10:17.569Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.104899324Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193}
17392024-02-03T17:10:17.569Z{"msg":"Crucible 474a5ee0-a96d-49f8-adba-943b7626b5c4 has session id: fd7c0ecb-662f-41c7-8d07-b6543251f13a","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.105049519Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17402024-02-03T17:10:17.569Z{"msg":"Upstairs opts: Upstairs UUID: 474a5ee0-a96d-49f8-adba-943b7626b5c4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.105149432Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17412024-02-03T17:10:17.569Z{"msg":"Crucible stats registered with UUID: 474a5ee0-a96d-49f8-adba-943b7626b5c4","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.105192202Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17422024-02-03T17:10:17.569Z{"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.105229068Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193}
17432024-02-03T17:10:17.569Z{"msg":"474a5ee0-a96d-49f8-adba-943b7626b5c4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.105368901Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17442024-02-03T17:10:17.569Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.105416133Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17452024-02-03T17:10:17.569Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.105441938Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17462024-02-03T17:10:17.570Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.105526705Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17472024-02-03T17:10:17.570Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.105573714Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17482024-02-03T17:10:17.570Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.10565392Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17492024-02-03T17:10:17.570Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.105703317Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17502024-02-03T17:10:17.570Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.105879502Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17512024-02-03T17:10:17.570Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.105931711Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17522024-02-03T17:10:17.570Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.106008591Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17532024-02-03T17:10:17.570Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.106037711Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17542024-02-03T17:10:17.570Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.106064478Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17552024-02-03T17:10:17.570Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.10614177Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17562024-02-03T17:10:17.570Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.107490508Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17572024-02-03T17:10:17.570Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.107625539Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17582024-02-03T17:10:17.570Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.107792086Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17592024-02-03T17:10:17.570Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.107821398Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17602024-02-03T17:10:17.570Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.107844232Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17612024-02-03T17:10:17.570Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.107867832Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17622024-02-03T17:10:17.570Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.108738825Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17632024-02-03T17:10:17.570Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.1088102Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17642024-02-03T17:10:17.570Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.108934196Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17652024-02-03T17:10:17.575Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.109688728Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17662024-02-03T17:10:17.576Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.109790502Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17672024-02-03T17:10:17.576Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.109837102Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17682024-02-03T17:10:17.576Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.109915466Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17692024-02-03T17:10:17.576Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.10996265Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17702024-02-03T17:10:17.576Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.110007735Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17712024-02-03T17:10:17.576Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.110045038Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17722024-02-03T17:10:17.576Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.110070534Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17732024-02-03T17:10:17.576Z{"msg":"[0]R flush_numbers[0..12]: [20, 17, 13, 12, 17, 14, 10, 24, 16, 18, 24, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.110126138Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17742024-02-03T17:10:17.576Z{"msg":"[0]R generation[0..12]: [13, 10, 6, 6, 10, 7, 4, 15, 9, 12, 15, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.110151817Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17752024-02-03T17:10:17.576Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.110175564Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17762024-02-03T17:10:17.576Z{"msg":"[1]R flush_numbers[0..12]: [20, 17, 13, 12, 17, 14, 10, 24, 16, 18, 24, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.110225941Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17772024-02-03T17:10:17.576Z{"msg":"[1]R generation[0..12]: [13, 10, 6, 6, 10, 7, 4, 15, 9, 12, 15, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.110250701Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17782024-02-03T17:10:17.576Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.110301146Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17792024-02-03T17:10:17.576Z{"msg":"[2]R flush_numbers[0..12]: [20, 17, 13, 12, 17, 14, 10, 24, 16, 18, 24, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.110326524Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17802024-02-03T17:10:17.576Z{"msg":"[2]R generation[0..12]: [13, 10, 6, 6, 10, 7, 4, 15, 9, 12, 15, 2]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
17812024-02-03T17:10:17.576Z,"time":"2024-02-03T17:10:17.110402931Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17822024-02-03T17:10:17.576Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.11045835Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17832024-02-03T17:10:17.576Z{"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.110482335Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17842024-02-03T17:10:17.576Z{"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.110504382Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17852024-02-03T17:10:17.576Z{"msg":"Next flush: 25","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.110552635Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17862024-02-03T17:10:17.576Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.110576136Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17872024-02-03T17:10:17.576Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.110626551Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17882024-02-03T17:10:17.577Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.110648457Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17892024-02-03T17:10:17.577Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.110671579Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17902024-02-03T17:10:17.577Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.110693245Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17912024-02-03T17:10:17.577Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.110714985Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17922024-02-03T17:10:17.577Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.110735131Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17932024-02-03T17:10:17.577Z{"msg":"474a5ee0-a96d-49f8-adba-943b7626b5c4 is now active with session: fd7c0ecb-662f-41c7-8d07-b6543251f13a","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.110756328Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17942024-02-03T17:10:17.577Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.110805898Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193,"session_id":"fd7c0ecb-662f-41c7-8d07-b6543251f13a"}
17952024-02-03T17:10:17.577Z{"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:17.110828297Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193}
17962024-02-03T17:10:17.577ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
17972024-02-03T17:10:17.577ZLoading write count information from file "/tmp/test_repair_verify.data"
17982024-02-03T17:10:17.577ZRun Repair workload
17992024-02-03T17:10:17.577Z01/30 Write block 114 len 36864 data: 3 3 3 3 2 2 2 2 2
18002024-02-03T17:10:17.577Z02/30 Read block 119 len 28672
18012024-02-03T17:10:17.581Z03/30 Read block 330 len 32768
18022024-02-03T17:10:17.581Z04/30 Write block 35 len 32768 data: 2 2 2 2 2 2 2 2
18032024-02-03T17:10:17.581Z05/30 Write block 47 len 16384 data: 2 2 2 3
18042024-02-03T17:10:17.584Z06/30 Read block 337 len 20480
18052024-02-03T17:10:17.584Z07/30 Write block 547 len 16384 data: 3 2 2 2
18062024-02-03T17:10:17.587Z08/30 Write block 117 len 28672 data: 4 3 3 3 3 3 2
18072024-02-03T17:10:17.587Z09/30 Read block 114 len 4096
18082024-02-03T17:10:17.587Z10/30 Read block 143 len 28672
18092024-02-03T17:10:17.590Z11/30 Write block 30 len 12288 data: 3 3 3
18102024-02-03T17:10:17.590Z12/30 Flush
18112024-02-03T17:10:17.599Z13/30 Write block 162 len 36864 data: 2 2 2 2 2 2 2 2 2
18122024-02-03T17:10:17.599Z14/30 Read block 580 len 16384
18132024-02-03T17:10:17.600Z15/30 Write block 411 len 4096 data: 2
18142024-02-03T17:10:17.603Z16/30 Write block 553 len 40960 data: 2 2 2 2 2 2 2 2 3 3
18152024-02-03T17:10:17.603Z17/30 Read block 472 len 16384
18162024-02-03T17:10:17.603Z18/30 Write block 263 len 36864 data: 3 3 3 3 4 4 3 3 4
18172024-02-03T17:10:17.603Z19/30 Read block 69 len 12288
18182024-02-03T17:10:17.606Z20/30 Write block 263 len 24576 data: 4 4 4 4 5 5
18192024-02-03T17:10:17.606Z21/30 Read block 261 len 36864
18202024-02-03T17:10:17.609Z22/30 Read block 416 len 40960
18212024-02-03T17:10:17.609Z23/30 Read block 24 len 32768
18222024-02-03T17:10:17.612Z24/30 Flush
18232024-02-03T17:10:17.615Z25/30 Write block 232 len 12288 data: 3 2 2
18242024-02-03T17:10:17.617Z26/30 Flush
18252024-02-03T17:10:17.620Z27/30 Write block 365 len 40960 data: 2 2 3 4 4 4 2 3 3 3
18262024-02-03T17:10:17.620Z28/30 Write block 516 len 28672 data: 3 3 2 2 2 2 2
18272024-02-03T17:10:17.621Z29/30 Read block 469 len 32768
18282024-02-03T17:10:17.624Z30/30 Read block 124 len 12288
18292024-02-03T17:10:17.627Z----------------------------------------------------------------
18302024-02-03T17:10:17.627Z Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30
18312024-02-03T17:10:17.627ZStates: Active Active Active
18322024-02-03T17:10:17.627ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
18332024-02-03T17:10:17.627Z 1 Acked 1000 Write 9 Done Done Done false
18342024-02-03T17:10:17.627Z 2 Acked 1001 Read 7 Done Done Done false
18352024-02-03T17:10:17.627Z 3 Acked 1002 Read 8 Done Done Done false
18362024-02-03T17:10:17.627Z 4 Acked 1003 Write 8 Done Done Done false
18372024-02-03T17:10:17.627Z 5 Acked 1004 Write 4 Sent Done Done false
18382024-02-03T17:10:17.627Z 6 Acked 1005 Read 5 Sent Done Done false
18392024-02-03T17:10:17.627Z 7 Acked 1006 Write 4 Sent Done Done false
18402024-02-03T17:10:17.627Z 8 Acked 1007 Write 7 Sent Done Done false
18412024-02-03T17:10:17.627Z 9 Acked 1008 Read 1 Sent Done Done false
18422024-02-03T17:10:17.627Z 10 Acked 1009 Read 7 Sent Done Done false
18432024-02-03T17:10:17.627Z 11 Acked 1010 Write 3 Sent Done Done false
18442024-02-03T17:10:17.627Z 12 Acked 1011 Flush 0 Sent Done Done false
18452024-02-03T17:10:17.627Z 13 Acked 1012 Write 9 Sent Done Done false
18462024-02-03T17:10:17.627Z 14 Acked 1013 Read 4 Sent Done Done false
18472024-02-03T17:10:17.628Z 15 Acked 1014 Write 1 Sent Done Done false
18482024-02-03T17:10:17.628Z 16 Acked 1015 Write 10 Sent Done Done false
18492024-02-03T17:10:17.628Z 17 Acked 1016 Read 4 Sent Done Done false
18502024-02-03T17:10:17.628Z 18 Acked 1017 Write 9 Sent Done Done false
18512024-02-03T17:10:17.628Z 19 Acked 1018 Read 3 Sent Done Done false
18522024-02-03T17:10:17.628Z 20 Acked 1019 Write 6 Sent Done Done false
18532024-02-03T17:10:17.628Z 21 Acked 1020 Read 9 Sent Done Done false
18542024-02-03T17:10:17.628Z 22 Acked 1021 Read 10 Sent Done Done false
18552024-02-03T17:10:17.628Z 23 Acked 1022 Read 8 Sent Done Done false
18562024-02-03T17:10:17.628Z 24 Acked 1023 Flush 0 Sent Done Done false
18572024-02-03T17:10:17.628Z 25 Acked 1024 Write 3 Sent Done Done false
18582024-02-03T17:10:17.628Z 26 Acked 1025 Flush 0 Sent Done Done false
18592024-02-03T17:10:17.628Z 27 Acked 1026 Write 10 Sent Done Done false
18602024-02-03T17:10:17.628Z 28 Acked 1027 Write 7 Sent Done Done false
18612024-02-03T17:10:17.628Z 29 Acked 1028 Read 8 Sent Done Done false
18622024-02-03T17:10:17.628Z 30 Acked 1029 Read 3 Sent Sent Done false
18632024-02-03T17:10:17.628Z STATES DS:0 DS:1 DS:2 TOTAL
18642024-02-03T17:10:17.628Z New 0 0 0 0
18652024-02-03T17:10:17.628Z Sent 26 1 0 27
18662024-02-03T17:10:17.628Z Done 4 29 30 63
18672024-02-03T17:10:17.628Z Skipped 0 0 0 0
18682024-02-03T17:10:17.628Z Error 0 0 0 0
18692024-02-03T17:10:17.628ZLast Flush: 0 1025 1025
18702024-02-03T17:10:17.628ZDownstairs last five completed:
18712024-02-03T17:10:17.629ZUpstairs last five completed: 30 29 28 27 26
18722024-02-03T17:10:17.629Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:10:17.166793124Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1193}
18732024-02-03T17:10:17.629ZWrote out file "/tmp/test_repair_verify.data"
18742024-02-03T17:10:17.633Z
18752024-02-03T17:10:17.639ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
18762024-02-03T17:10:17.666ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
18772024-02-03T17:10:17.666Z 0 000-019 13 13 13 20 20 20 F F F
18782024-02-03T17:10:17.666Z 1 020-039 10 16 16 17 25 25 T F F <---
18792024-02-03T17:10:17.666Z 2 040-059 6 16 16 13 25 25 T F F <---
18802024-02-03T17:10:17.666Z 3 060-079 6 6 6 12 12 12 F F F
18812024-02-03T17:10:17.666Z 4 080-099 10 10 10 17 17 17 F F F
18822024-02-03T17:10:17.666Z 5 100-119 7 16 16 14 25 25 T F F <---
18832024-02-03T17:10:17.666Z 6 120-139 4 16 16 10 25 25 T F F <---
18842024-02-03T17:10:17.666Z 7 140-159 15 15 15 24 24 24 F F F
18852024-02-03T17:10:17.666Z 8 160-179 9 16 16 16 26 26 F F F <---
18862024-02-03T17:10:17.669Z 9 180-199 12 12 12 18 18 18 F F F
18872024-02-03T17:10:17.669Z 10 200-219 15 15 15 24 24 24 F F F
18882024-02-03T17:10:17.669Z 11 220-239 2 16 16 4 27 27 F F F <---
18892024-02-03T17:10:17.670Z 12 240-259 14 14 14 21 21 21 F F F
18902024-02-03T17:10:17.670Z 13 260-279 15 16 16 24 26 26 F F F <---
18912024-02-03T17:10:17.670Z 14 280-299 15 15 15 24 24 24 F F F
18922024-02-03T17:10:17.670Z 15 300-319 10 10 10 17 17 17 F F F
18932024-02-03T17:10:17.670Z 16 320-339 15 15 15 24 24 24 F F F
18942024-02-03T17:10:17.670Z 17 340-359 12 12 12 18 18 18 F F F
18952024-02-03T17:10:17.670Z 18 360-379 13 13 13 20 20 20 F T T <---
18962024-02-03T17:10:17.670Z 19 380-399 14 14 14 21 21 21 F F F
18972024-02-03T17:10:17.670Z 20 400-419 15 16 16 24 26 26 F F F <---
18982024-02-03T17:10:17.670Z 21 420-439 15 15 15 24 24 24 F F F
18992024-02-03T17:10:17.670Z 22 440-459 8 8 8 15 15 15 F F F
19002024-02-03T17:10:17.670Z 23 460-479 14 14 14 23 23 23 F F F
19012024-02-03T17:10:17.670Z 24 480-499 10 10 10 17 17 17 F F F
19022024-02-03T17:10:17.670Z 25 500-519 12 12 12 18 18 18 F T T <---
19032024-02-03T17:10:17.670Z 26 520-539 4 4 4 9 9 9 F T T <---
19042024-02-03T17:10:17.670Z 27 540-559 6 16 16 13 26 26 F F F <---
19052024-02-03T17:10:17.670Z 28 560-579 15 16 16 24 26 26 F F F <---
19062024-02-03T17:10:17.670Z 29 580-599 15 15 15 24 24 24 F F F
19072024-02-03T17:10:17.670ZMax gen: 16, Max flush: 27
19082024-02-03T17:10:17.670ZError: Difference in extent metadata found!
19092024-02-03T17:10:17.670ZA Difference in extent metadata is expected here
19102024-02-03T17:10:17.670ZOn loop 8
19112024-02-03T17:10:17.671Z
19122024-02-03T17:10:17.674ZVerifying data now
19132024-02-03T17:10:23.815ZNo change after verify
19142024-02-03T17:10:23.815ZLoop: 8 Downstairs dump after verify (and repair):
19152024-02-03T17:10:23.838ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
19162024-02-03T17:10:23.838Z 0 000-019 13 13 13 20 20 20 F F F
19172024-02-03T17:10:23.838Z 1 020-039 16 16 16 25 25 25 F F F
19182024-02-03T17:10:23.838Z 2 040-059 16 16 16 25 25 25 F F F
19192024-02-03T17:10:23.841Z 3 060-079 6 6 6 12 12 12 F F F
19202024-02-03T17:10:23.841Z 4 080-099 10 10 10 17 17 17 F F F
19212024-02-03T17:10:23.841Z 5 100-119 16 16 16 25 25 25 F F F
19222024-02-03T17:10:23.841Z 6 120-139 16 16 16 25 25 25 F F F
19232024-02-03T17:10:23.841Z 7 140-159 15 15 15 24 24 24 F F F
19242024-02-03T17:10:23.841Z 8 160-179 16 16 16 26 26 26 F F F
19252024-02-03T17:10:23.841Z 9 180-199 12 12 12 18 18 18 F F F
19262024-02-03T17:10:23.842Z 10 200-219 15 15 15 24 24 24 F F F
19272024-02-03T17:10:23.842Z 11 220-239 16 16 16 27 27 27 F F F
19282024-02-03T17:10:23.842Z 12 240-259 14 14 14 21 21 21 F F F
19292024-02-03T17:10:23.842Z 13 260-279 16 16 16 26 26 26 F F F
19302024-02-03T17:10:23.842Z 14 280-299 15 15 15 24 24 24 F F F
19312024-02-03T17:10:23.842Z 15 300-319 10 10 10 17 17 17 F F F
19322024-02-03T17:10:23.842Z 16 320-339 15 15 15 24 24 24 F F F
19332024-02-03T17:10:23.842Z 17 340-359 12 12 12 18 18 18 F F F
19342024-02-03T17:10:23.842Z 18 360-379 17 17 17 28 28 28 F F F
19352024-02-03T17:10:23.842Z 19 380-399 14 14 14 21 21 21 F F F
19362024-02-03T17:10:23.842Z 20 400-419 16 16 16 26 26 26 F F F
19372024-02-03T17:10:23.842Z 21 420-439 15 15 15 24 24 24 F F F
19382024-02-03T17:10:23.842Z 22 440-459 8 8 8 15 15 15 F F F
19392024-02-03T17:10:23.842Z 23 460-479 14 14 14 23 23 23 F F F
19402024-02-03T17:10:23.842Z 24 480-499 10 10 10 17 17 17 F F F
19412024-02-03T17:10:23.842Z 25 500-519 17 17 17 28 28 28 F F F
19422024-02-03T17:10:23.842Z 26 520-539 17 17 17 28 28 28 F F F
19432024-02-03T17:10:23.842Z 27 540-559 16 16 16 26 26 26 F F F
19442024-02-03T17:10:23.842Z 28 560-579 16 16 16 26 26 26 F F F
19452024-02-03T17:10:23.842Z 29 580-599 15 15 15 24 24 24 F F F
19462024-02-03T17:10:23.842ZMax gen: 17, Max flush: 28
19472024-02-03T17:10:23.845Z
19482024-02-03T17:10:23.845ZBegin loop 9
19492024-02-03T17:10:23.845ZDownstairs to restart: 0
19502024-02-03T17:10:23.874Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.415613147Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201}
19512024-02-03T17:10:23.875Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.416126743Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201}
19522024-02-03T17:10:23.875Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.416205446Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201}
19532024-02-03T17:10:23.879Z{"msg":"Crucible 673cf3f5-6fb4-4d99-8f95-eeb2a13bd792 has session id: c278f639-326a-4251-8697-f84829b08b8f","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.416285156Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19542024-02-03T17:10:23.879Z{"msg":"Upstairs opts: Upstairs UUID: 673cf3f5-6fb4-4d99-8f95-eeb2a13bd792, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.416341024Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19552024-02-03T17:10:23.879Z{"msg":"Crucible stats registered with UUID: 673cf3f5-6fb4-4d99-8f95-eeb2a13bd792","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.416428544Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19562024-02-03T17:10:23.879ZCrucible runtime is spawned
19572024-02-03T17:10:23.879Z{"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.416681553Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201}
19582024-02-03T17:10:23.879Z{"msg":"673cf3f5-6fb4-4d99-8f95-eeb2a13bd792 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.41682536Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19592024-02-03T17:10:23.879Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.416904409Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19602024-02-03T17:10:23.879Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.416987343Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19612024-02-03T17:10:23.879Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.417031769Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19622024-02-03T17:10:23.879Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.417074238Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19632024-02-03T17:10:23.879Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.417199592Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19642024-02-03T17:10:23.879Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.417269925Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19652024-02-03T17:10:23.880Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.417485514Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"":"io task","client":"2","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19662024-02-03T17:10:23.880Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.417626121Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"":"io task","client":"0","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19672024-02-03T17:10:23.880Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.417711188Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"":"io task","client":"1","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19682024-02-03T17:10:23.880Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.417748924Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"":"io task","client":"0","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19692024-02-03T17:10:23.880Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.417773296Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"":"io task","client":"2","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19702024-02-03T17:10:23.880Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.417795576Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"":"io task","client":"1","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19712024-02-03T17:10:23.880Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.418894482Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19722024-02-03T17:10:23.880Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.418960643Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19732024-02-03T17:10:23.880Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.418985193Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19742024-02-03T17:10:23.880Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.41907205Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19752024-02-03T17:10:23.880Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.419728391Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19762024-02-03T17:10:23.885Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.419820138Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19772024-02-03T17:10:23.885Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.420215376Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19782024-02-03T17:10:23.885Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.420293799Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19792024-02-03T17:10:23.885Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.421144197Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19802024-02-03T17:10:23.885Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.421358476Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19812024-02-03T17:10:23.885Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.421433934Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19822024-02-03T17:10:23.885Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.421460613Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19832024-02-03T17:10:23.885Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.421481427Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19842024-02-03T17:10:23.885Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.421503838Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19852024-02-03T17:10:23.886Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.421525773Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19862024-02-03T17:10:23.886Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.422011388Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19872024-02-03T17:10:23.886Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.422114034Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19882024-02-03T17:10:23.886Z{"msg":"[0]R flush_numbers[0..12]: [20, 25, 25, 12, 17, 25, 25, 24, 26, 18, 24, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.42220049Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19892024-02-03T17:10:23.886Z{"msg":"[0]R generation[0..12]: [13, 16, 16, 6, 10, 16, 16, 15, 16, 12, 15, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.422280539ZWait for a query_work_queue command to finish before sending IO
19902024-02-03T17:10:23.886Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19912024-02-03T17:10:23.886Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.42237446Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19922024-02-03T17:10:23.886Z{"msg":"[1]R flush_numbers[0..12]: [20, 25, 25, 12, 17, 25, 25, 24, 26, 18, 24, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.42245813Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19932024-02-03T17:10:23.886Z{"msg":"[1]R generation[0..12]: [13, 16, 16, 6, 10, 16, 16, 15, 16, 12, 15, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.422540558Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19942024-02-03T17:10:23.886Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.422624638Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19952024-02-03T17:10:23.886Z{"msg":"[2]R flush_numbers[0..12]: [20, 25, 25, 12, 17, 25, 25, 24, 26, 18, 24, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.42270673Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19962024-02-03T17:10:23.886Z{"msg":"[2]R generation[0..12]: [13, 16, 16, 6, 10, 16, 16, 15, 16, 12, 15, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.422791714Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19972024-02-03T17:10:23.886Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.422874911Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19982024-02-03T17:10:23.886Z{"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.422921087Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
19992024-02-03T17:10:23.886Z{"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.423002404Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
20002024-02-03T17:10:23.886Z{"msg":"Next flush: 29","v":0,"name":"crucible","level":30,"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
20012024-02-03T17:10:23.886Ztime":"2024-02-03T17:10:23.423084216Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
20022024-02-03T17:10:23.886Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.423168292Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
20032024-02-03T17:10:23.886Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.423250025Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
20042024-02-03T17:10:23.886Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.423279859Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
20052024-02-03T17:10:23.886Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.423329945Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
20062024-02-03T17:10:23.886Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.423354412Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
20072024-02-03T17:10:23.886Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.423401345Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
20082024-02-03T17:10:23.887Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.423424027Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
20092024-02-03T17:10:23.887Z{"msg":"673cf3f5-6fb4-4d99-8f95-eeb2a13bd792 is now active with session: c278f639-326a-4251-8697-f84829b08b8f","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.423445764Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
20102024-02-03T17:10:23.887Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.423465739Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201,"session_id":"c278f639-326a-4251-8697-f84829b08b8f"}
20112024-02-03T17:10:23.887Z{"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:23.423487703Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201}
20122024-02-03T17:10:23.887ZLoading write count information from file "/tmp/test_repair_verify.data"
20132024-02-03T17:10:23.887ZRun Repair workload
20142024-02-03T17:10:23.887Z01/30 Flush
20152024-02-03T17:10:23.890Z02/30 Write block 299 len 32768 data: 2 2 2 2 2 2 2 3
20162024-02-03T17:10:23.890Z03/30 Flush
20172024-02-03T17:10:23.891Z04/30 Write block 117 len 8192 data: 5 4
20182024-02-03T17:10:23.891Z05/30 Flush
20192024-02-03T17:10:23.894Z06/30 Flush
20202024-02-03T17:10:23.897Z07/30 Write block 169 len 24576 data: 3 3 2 2 4 4
20212024-02-03T17:10:23.897Z08/30 Read block 519 len 28672
20222024-02-03T17:10:23.900Z09/30 Read block 182 len 8192
20232024-02-03T17:10:23.900Z10/30 Write block 591 len 28672 data: 2 2 2 2 2 3 2
20242024-02-03T17:10:23.900Z11/30 Read block 96 len 12288
20252024-02-03T17:10:23.906Z12/30 Read block 455 len 20480
20262024-02-03T17:10:23.906Z13/30 Read block 67 len 32768
20272024-02-03T17:10:23.909Z14/30 Read block 167 len 40960
20282024-02-03T17:10:23.909Z15/30 Write block 439 len 4096 data: 3
20292024-02-03T17:10:23.909Z16/30 Read block 13 len 24576
20302024-02-03T17:10:23.912Z17/30 Flush
20312024-02-03T17:10:23.915Z18/30 Write block 323 len 4096 data: 5
20322024-02-03T17:10:23.915Z19/30 Write block 377 len 4096 data: 3
20332024-02-03T17:10:23.915Z20/30 Write block 359 len 8192 data: 3 3
20342024-02-03T17:10:23.915Z21/30 Read block 572 len 16384
20352024-02-03T17:10:23.918Z22/30 Write block 348 len 8192 data: 3 3
20362024-02-03T17:10:23.918Z23/30 Write block 364 len 28672 data: 2 3 3 4 5 5 5
20372024-02-03T17:10:23.921Z24/30 Write block 85 len 40960 data: 3 3 2 2 2 2 2 2 2 2
20382024-02-03T17:10:23.921Z25/30 Read block 278 len 24576
20392024-02-03T17:10:23.924Z26/30 Read block 391 len 8192
20402024-02-03T17:10:23.924Z27/30 Read block 194 len 24576
20412024-02-03T17:10:23.927Z28/30 Write block 213 len 4096 data: 3
20422024-02-03T17:10:23.927Z29/30 Read block 81 len 20480
20432024-02-03T17:10:23.931Z30/30 Read block 588 len 12288
20442024-02-03T17:10:23.931Z----------------------------------------------------------------
20452024-02-03T17:10:23.931Z Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:24
20462024-02-03T17:10:23.931ZStates: Active Active Active
20472024-02-03T17:10:23.931ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
20482024-02-03T17:10:23.931Z 7 Acked 1006 Write 6 Done Done Done false
20492024-02-03T17:10:23.931Z 8 Acked 1007 Read 7 Done Done Done false
20502024-02-03T17:10:23.931Z 9 Acked 1008 Read 2 Done Done Done false
20512024-02-03T17:10:23.931Z 10 Acked 1009 Write 7 Done Done Done false
20522024-02-03T17:10:23.931Z 11 Acked 1010 Read 3 Sent Done Done false
20532024-02-03T17:10:23.931Z 12 Acked 1011 Read 5 Sent Done Done false
20542024-02-03T17:10:23.931Z 13 Acked 1012 Read 8 Sent Done Done false
20552024-02-03T17:10:23.931Z 14 Acked 1013 Read 10 Sent Done Done false
20562024-02-03T17:10:23.931Z 15 Acked 1014 Write 1 Sent Done Done false
20572024-02-03T17:10:23.931Z 16 Acked 1015 Read 6 Sent Done Done false
20582024-02-03T17:10:23.932Z 17 Acked 1016 Flush 0 Sent Done Done false
20592024-02-03T17:10:23.932Z 18 Acked 1017 Write 1 Sent Done Done false
20602024-02-03T17:10:23.932Z 19 Acked 1018 Write 1 Sent Done Done false
20612024-02-03T17:10:23.932Z 20 Acked 1019 Write 2 Sent Done Done false
20622024-02-03T17:10:23.932Z 21 Acked 1020 Read 4 Sent Done Done false
20632024-02-03T17:10:23.932Z 22 Acked 1021 Write 2 Sent Done Done false
20642024-02-03T17:10:23.932Z 23 Acked 1022 Write 7 Sent Done Done false
20652024-02-03T17:10:23.932Z 24 Acked 1023 Write 10 Sent Done Done false
20662024-02-03T17:10:23.932Z 25 Acked 1024 Read 6 Sent Done Done false
20672024-02-03T17:10:23.932Z 26 Acked 1025 Read 2 Sent Done Done false
20682024-02-03T17:10:23.932Z 27 Acked 1026 Read 6 Sent Done Done false
20692024-02-03T17:10:23.932Z 28 Acked 1027 Write 1 Sent Done Done false
20702024-02-03T17:10:23.932Z 29 Acked 1028 Read 5 Sent Done Done false
20712024-02-03T17:10:23.932Z 30 Acked 1029 Read 3 Sent Sent Done false
20722024-02-03T17:10:23.932Z STATES DS:0 DS:1 DS:2 TOTAL
20732024-02-03T17:10:23.932Z New 0 0 0 0
20742024-02-03T17:10:23.932Z Sent 20 1 0 21
20752024-02-03T17:10:23.932Z Done 4 23 24 51
20762024-02-03T17:10:23.932Z Skipped 0 0 0 0
20772024-02-03T17:10:23.932Z Error 0 0 0 0
20782024-02-03T17:10:23.932ZLast Flush: 1005 1016 1016
20792024-02-03T17:10:23.932ZDownstairs last five completed: 1005 1004 1003 1002 1001
20802024-02-03T17:10:23.932ZUpstairs last five completed: 30 29 28 27 26
20812024-02-03T17:10:23.932Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:10:23.471417925Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1201}
20822024-02-03T17:10:23.932ZWrote out file "/tmp/test_repair_verify.data"
20832024-02-03T17:10:23.937Z
20842024-02-03T17:10:23.943ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
20852024-02-03T17:10:23.970ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
20862024-02-03T17:10:23.970Z 0 000-019 13 13 13 20 20 20 F F F
20872024-02-03T17:10:23.970Z 1 020-039 16 16 16 25 25 25 F F F
20882024-02-03T17:10:23.970Z 2 040-059 16 16 16 25 25 25 F F F
20892024-02-03T17:10:23.970Z 3 060-079 6 6 6 12 12 12 F F F
20902024-02-03T17:10:23.973Z 4 080-099 10 10 10 17 17 17 F T T <---
20912024-02-03T17:10:23.974Z 5 100-119 18 18 18 31 31 31 F F F
20922024-02-03T17:10:23.974Z 6 120-139 16 16 16 25 25 25 F F F
20932024-02-03T17:10:23.974Z 7 140-159 15 15 15 24 24 24 F F F
20942024-02-03T17:10:23.974Z 8 160-179 16 18 18 26 33 33 T F F <---
20952024-02-03T17:10:23.974Z 9 180-199 12 12 12 18 18 18 F F F
20962024-02-03T17:10:23.974Z 10 200-219 15 15 15 24 24 24 F T T <---
20972024-02-03T17:10:23.974Z 11 220-239 16 16 16 27 27 27 F F F
20982024-02-03T17:10:23.974Z 12 240-259 14 14 14 21 21 21 F F F
20992024-02-03T17:10:23.974Z 13 260-279 16 16 16 26 26 26 F F F
21002024-02-03T17:10:23.974Z 14 280-299 18 18 18 30 30 30 F F F
21012024-02-03T17:10:23.974Z 15 300-319 18 18 18 30 30 30 F F F
21022024-02-03T17:10:23.974Z 16 320-339 15 15 15 24 24 24 F T T <---
21032024-02-03T17:10:23.974Z 17 340-359 12 12 12 18 18 18 F T T <---
21042024-02-03T17:10:23.974Z 18 360-379 17 17 17 28 28 28 F T T <---
21052024-02-03T17:10:23.974Z 19 380-399 14 14 14 21 21 21 F F F
21062024-02-03T17:10:23.974Z 20 400-419 16 16 16 26 26 26 F F F
21072024-02-03T17:10:23.974Z 21 420-439 15 18 18 24 33 33 F F F <---
21082024-02-03T17:10:23.974Z 22 440-459 8 8 8 15 15 15 F F F
21092024-02-03T17:10:23.974Z 23 460-479 14 14 14 23 23 23 F F F
21102024-02-03T17:10:23.974Z 24 480-499 10 10 10 17 17 17 F F F
21112024-02-03T17:10:23.974Z 25 500-519 17 17 17 28 28 28 F F F
21122024-02-03T17:10:23.974Z 26 520-539 17 17 17 28 28 28 F F F
21132024-02-03T17:10:23.975Z 27 540-559 16 16 16 26 26 26 F F F
21142024-02-03T17:10:23.975Z 28 560-579 16 16 16 26 26 26 F F F
21152024-02-03T17:10:23.975Z 29 580-599 15 18 18 24 33 33 T F F <---
21162024-02-03T17:10:23.975ZMax gen: 18, Max flush: 33
21172024-02-03T17:10:23.975ZError: Difference in extent metadata found!
21182024-02-03T17:10:23.978ZA Difference in extent metadata is expected here
21192024-02-03T17:10:23.978ZOn loop 9
21202024-02-03T17:10:23.978Z
21212024-02-03T17:10:23.978ZVerifying data now
21222024-02-03T17:10:28.757ZNo change after verify
21232024-02-03T17:10:28.757ZLoop: 9 Downstairs dump after verify (and repair):
21242024-02-03T17:10:28.780ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
21252024-02-03T17:10:28.780Z 0 000-019 13 13 13 20 20 20 F F F
21262024-02-03T17:10:28.781Z 1 020-039 16 16 16 25 25 25 F F F
21272024-02-03T17:10:28.781Z 2 040-059 16 16 16 25 25 25 F F F
21282024-02-03T17:10:28.781Z 3 060-079 6 6 6 12 12 12 F F F
21292024-02-03T17:10:28.784Z 4 080-099 19 19 19 34 34 34 F F F
21302024-02-03T17:10:28.784Z 5 100-119 18 18 18 31 31 31 F F F
21312024-02-03T17:10:28.784Z 6 120-139 16 16 16 25 25 25 F F F
21322024-02-03T17:10:28.785Z 7 140-159 15 15 15 24 24 24 F F F
21332024-02-03T17:10:28.785Z 8 160-179 18 18 18 33 33 33 F F F
21342024-02-03T17:10:28.785Z 9 180-199 12 12 12 18 18 18 F F F
21352024-02-03T17:10:28.785Z 10 200-219 19 19 19 34 34 34 F F F
21362024-02-03T17:10:28.785Z 11 220-239 16 16 16 27 27 27 F F F
21372024-02-03T17:10:28.785Z 12 240-259 14 14 14 21 21 21 F F F
21382024-02-03T17:10:28.785Z 13 260-279 16 16 16 26 26 26 F F F
21392024-02-03T17:10:28.785Z 14 280-299 18 18 18 30 30 30 F F F
21402024-02-03T17:10:28.785Z 15 300-319 18 18 18 30 30 30 F F F
21412024-02-03T17:10:28.785Z 16 320-339 19 19 19 34 34 34 F F F
21422024-02-03T17:10:28.785Z 17 340-359 19 19 19 34 34 34 F F F
21432024-02-03T17:10:28.785Z 18 360-379 19 19 19 34 34 34 F F F
21442024-02-03T17:10:28.786Z 19 380-399 14 14 14 21 21 21 F F F
21452024-02-03T17:10:28.786Z 20 400-419 16 16 16 26 26 26 F F F
21462024-02-03T17:10:28.786Z 21 420-439 18 18 18 33 33 33 F F F
21472024-02-03T17:10:28.786Z 22 440-459 8 8 8 15 15 15 F F F
21482024-02-03T17:10:28.786Z 23 460-479 14 14 14 23 23 23 F F F
21492024-02-03T17:10:28.786Z 24 480-499 10 10 10 17 17 17 F F F
21502024-02-03T17:10:28.786Z 25 500-519 17 17 17 28 28 28 F F F
21512024-02-03T17:10:28.786Z 26 520-539 17 17 17 28 28 28 F F F
21522024-02-03T17:10:28.786Z 27 540-559 16 16 16 26 26 26 F F F
21532024-02-03T17:10:28.786Z 28 560-579 16 16 16 26 26 26 F F F
21542024-02-03T17:10:28.786Z 29 580-599 18 18 18 33 33 33 F F F
21552024-02-03T17:10:28.786ZMax gen: 19, Max flush: 34
21562024-02-03T17:10:28.790Z
21572024-02-03T17:10:28.790ZBegin loop 10
21582024-02-03T17:10:28.790ZDownstairs to restart: 2
21592024-02-03T17:10:28.817Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.358989587Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209}
21602024-02-03T17:10:28.818Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.359527803Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209}
21612024-02-03T17:10:28.818Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.359560766Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209}
21622024-02-03T17:10:28.822Z{"msg":"Crucible a85a0b3e-100b-4809-8bb7-1fadde10435e has session id: 9f0c7d27-b0cf-4a87-b325-8d354f612564","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.359635943Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21632024-02-03T17:10:28.822Z{"msg":"Upstairs opts: Upstairs UUID: a85a0b3e-100b-4809-8bb7-1fadde10435e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.359730346Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21642024-02-03T17:10:28.822Z{"msg":"Crucible stats registered with UUID: a85a0b3e-100b-4809-8bb7-1fadde10435e","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.359813784Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21652024-02-03T17:10:28.822ZCrucible runtime is spawned
21662024-02-03T17:10:28.822Z{"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.360190823Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209}
21672024-02-03T17:10:28.822Z{"msg":"a85a0b3e-100b-4809-8bb7-1fadde10435e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.360249316Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21682024-02-03T17:10:28.822Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.360272905Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21692024-02-03T17:10:28.823Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.360332781Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21702024-02-03T17:10:28.823Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.360359158Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21712024-02-03T17:10:28.823Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.360414994Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21722024-02-03T17:10:28.823Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.360440183Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21732024-02-03T17:10:28.823Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.360467717Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21742024-02-03T17:10:28.823Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.360542588Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"":"io task","client":"0","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21752024-02-03T17:10:28.823Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.360590699Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"":"io task","client":"1","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21762024-02-03T17:10:28.823Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.360631793Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"":"io task","client":"2","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21772024-02-03T17:10:28.823Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.360890949Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"":"io task","client":"1","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21782024-02-03T17:10:28.823Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.361290518Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"":"io task","client":"2","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21792024-02-03T17:10:28.823Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.361375655Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"":"io task","client":"0","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21802024-02-03T17:10:28.823Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.362018804Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21812024-02-03T17:10:28.823Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.362112194Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21822024-02-03T17:10:28.824Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.362481005Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21832024-02-03T17:10:28.824Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.362546616Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21842024-02-03T17:10:28.824Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.362838085Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21852024-02-03T17:10:28.824Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.36308758Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21862024-02-03T17:10:28.829Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.363143223Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21872024-02-03T17:10:28.829Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.363605782Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21882024-02-03T17:10:28.829Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.364157287Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21892024-02-03T17:10:28.829Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.3642666Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21902024-02-03T17:10:28.829Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.36431593Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21912024-02-03T17:10:28.830Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.364343227Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21922024-02-03T17:10:28.830Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.364477843Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21932024-02-03T17:10:28.830Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.36453752Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21942024-02-03T17:10:28.830Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.364563829Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21952024-02-03T17:10:28.830Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.364852917Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21962024-02-03T17:10:28.830Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.364935923Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21972024-02-03T17:10:28.830Z{"msg":"[0]R flush_numbers[0..12]: [20, 25, 25, 12, 34, 31, 25, 24, 33, 18, 34, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.364982248Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21982024-02-03T17:10:28.830Z{"msg":"[0]R generation[0..12]: [13, 16, 16, 6, 19, 18, 16, 15, 18, 12, 19, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.365038109Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
21992024-02-03T17:10:28.830Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.365065022Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
22002024-02-03T17:10:28.830Z{"msg":"[1]R flush_numbers[0..12]: [20, 25, 25, 12, 34, 31, 25, 24, 33, 18, 34, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.365088414Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
22012024-02-03T17:10:28.830Z{"msg":"[1]R generation[0..12]: [13, 16, 16, 6, 19, 18, 16, 15, 18, 12, 19, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.365135651Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
22022024-02-03T17:10:28.831Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.365161657Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
22032024-02-03T17:10:28.831Z{"msg":"[2]R flush_numbers[0..12]: [20, 25, 25, 12, 34, 31, 25, 24, 33, 18, 34, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.365184716Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
22042024-02-03T17:10:28.831Z{"msg":"[2]R generation[0..12]: [13, 16, 16, 6, 19, 18, 16, 15, 18, 12, 19, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.365207178Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
22052024-02-03T17:10:28.831Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
22062024-02-03T17:10:28.831Z,"time":"2024-02-03T17:10:28.365230517Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
22072024-02-03T17:10:28.831Z{"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.365254936Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
22082024-02-03T17:10:28.831Z{"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.365302909Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
22092024-02-03T17:10:28.831Z{"msg":"Next flush: 35","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.36532662Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
22102024-02-03T17:10:28.831Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.365348338Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
22112024-02-03T17:10:28.831Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.365397861Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
22122024-02-03T17:10:28.831Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.365420271Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
22132024-02-03T17:10:28.831Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.36544199Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
22142024-02-03T17:10:28.831Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.365463792Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
22152024-02-03T17:10:28.832Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.365511064Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
22162024-02-03T17:10:28.832Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.365533897Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
22172024-02-03T17:10:28.832Z{"msg":"a85a0b3e-100b-4809-8bb7-1fadde10435e is now active with session: 9f0c7d27-b0cf-4a87-b325-8d354f612564","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.365555603Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
22182024-02-03T17:10:28.832Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.365602172Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209,"session_id":"9f0c7d27-b0cf-4a87-b325-8d354f612564"}
22192024-02-03T17:10:28.832Z{"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:28.365624714Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209}
22202024-02-03T17:10:28.832ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
22212024-02-03T17:10:28.832ZLoading write count information from file "/tmp/test_repair_verify.data"
22222024-02-03T17:10:28.832ZRun Repair workload
22232024-02-03T17:10:28.832Z01/30 Write block 364 len 40960 data: 3 4 4 5 6 6 6 3 4 4
22242024-02-03T17:10:28.837Z02/30 Write block 188 len 8192 data: 3 3
22252024-02-03T17:10:28.837Z03/30 Flush
22262024-02-03T17:10:28.837Z04/30 Write block 277 len 28672 data: 6 5 5 5 3 3 5
22272024-02-03T17:10:28.837Z05/30 Write block 44 len 36864 data: 2 2 2 3 3 3 4 2 2
22282024-02-03T17:10:28.838Z06/30 Read block 219 len 36864
22292024-02-03T17:10:28.841Z07/30 Read block 520 len 40960
22302024-02-03T17:10:28.841Z08/30 Read block 363 len 24576
22312024-02-03T17:10:28.844Z09/30 Read block 175 len 12288
22322024-02-03T17:10:28.844Z10/30 Read block 35 len 40960
22332024-02-03T17:10:28.849Z11/30 Flush
22342024-02-03T17:10:28.853Z12/30 Write block 258 len 4096 data: 3
22352024-02-03T17:10:28.853Z13/30 Read block 312 len 16384
22362024-02-03T17:10:28.853Z14/30 Write block 479 len 24576 data: 2 2 2 2 3 3
22372024-02-03T17:10:28.856Z15/30 Write block 205 len 40960 data: 3 3 3 3 3 4 3 3 4 3
22382024-02-03T17:10:28.856Z16/30 Read block 261 len 8192
22392024-02-03T17:10:28.873Z17/30 Write block 301 len 40960 data: 3 3 3 3 3 4 2 2 2 2
22402024-02-03T17:10:28.873Z18/30 Read block 265 len 40960
22412024-02-03T17:10:28.874Z19/30 Read block 177 len 32768
22422024-02-03T17:10:28.877Z20/30 Flush
22432024-02-03T17:10:28.877Z21/30 Write block 435 len 16384 data: 3 4 3 3
22442024-02-03T17:10:28.877Z22/30 Read block 116 len 16384
22452024-02-03T17:10:28.877Z23/30 Flush
22462024-02-03T17:10:28.877Z24/30 Flush
22472024-02-03T17:10:28.877Z25/30 Write block 504 len 24576 data: 3 4 4 4 4 4
22482024-02-03T17:10:28.877Z26/30 Write block 379 len 40960 data: 3 2 2 2 2 2 2 2 3 3
22492024-02-03T17:10:28.877Z27/30 Read block 561 len 36864
22502024-02-03T17:10:28.881Z28/30 Read block 349 len 28672
22512024-02-03T17:10:28.881Z29/30 Write block 104 len 16384 data: 2 2 2 2
22522024-02-03T17:10:28.881Z30/30 Read block 221 len 32768
22532024-02-03T17:10:28.884Z----------------------------------------------------------------
22542024-02-03T17:10:28.885Z Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30
22552024-02-03T17:10:28.885ZStates: Active Active Active
22562024-02-03T17:10:28.885ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
22572024-02-03T17:10:28.885Z 1 Acked 1000 Write 10 Done Done Done false
22582024-02-03T17:10:28.885Z 2 Acked 1001 Write 2 Done Done Sent false
22592024-02-03T17:10:28.885Z 3 Acked 1002 Flush 0 Done Done Sent false
22602024-02-03T17:10:28.885Z 4 Acked 1003 Write 7 Done Done Sent false
22612024-02-03T17:10:28.885Z 5 Acked 1004 Write 9 Done Done Sent false
22622024-02-03T17:10:28.885Z 6 Acked 1005 Read 9 Done Done Sent false
22632024-02-03T17:10:28.885Z 7 Acked 1006 Read 10 Done Done Sent false
22642024-02-03T17:10:28.885Z 8 Acked 1007 Read 6 Done Done Sent false
22652024-02-03T17:10:28.885Z 9 Acked 1008 Read 3 Done Done Sent false
22662024-02-03T17:10:28.885Z 10 Acked 1009 Read 10 Done Done Sent false
22672024-02-03T17:10:28.886Z 11 Acked 1010 Flush 0 Done Done Sent false
22682024-02-03T17:10:28.886Z 12 Acked 1011 Write 1 Done Done Sent false
22692024-02-03T17:10:28.886Z 13 Acked 1012 Read 4 Done Done Sent false
22702024-02-03T17:10:28.886Z 14 Acked 1013 Write 6 Done Done Sent false
22712024-02-03T17:10:28.886Z 15 Acked 1014 Write 10 Done Done Sent false
22722024-02-03T17:10:28.886Z 16 Acked 1015 Read 2 Done Done Sent false
22732024-02-03T17:10:28.886Z 17 Acked 1016 Write 10 Done Done Sent false
22742024-02-03T17:10:28.886Z 18 Acked 1017 Read 10 Done Done Sent false
22752024-02-03T17:10:28.886Z 19 Acked 1018 Read 8 Done Done Sent false
22762024-02-03T17:10:28.886Z 20 Acked 1019 Flush 0 Done Done Sent false
22772024-02-03T17:10:28.886Z 21 Acked 1020 Write 4 Done Done Sent false
22782024-02-03T17:10:28.886Z 22 Acked 1021 Read 4 Done Done Sent false
22792024-02-03T17:10:28.886Z 23 Acked 1022 Flush 0 Done Done Sent false
22802024-02-03T17:10:28.887Z 24 Acked 1023 Flush 0 Done Done Sent false
22812024-02-03T17:10:28.887Z 25 Acked 1024 Write 6 Done Done Sent false
22822024-02-03T17:10:28.887Z 26 Acked 1025 Write 10 Done Done Sent false
22832024-02-03T17:10:28.887Z 27 Acked 1026 Read 9 Done Done Sent false
22842024-02-03T17:10:28.887Z 28 Acked 1027 Read 7 Done Done Sent false
22852024-02-03T17:10:28.887Z 29 Acked 1028 Write 4 Done Done Sent false
22862024-02-03T17:10:28.887Z 30 Acked 1029 Read 8 Done Sent Sent false
22872024-02-03T17:10:28.887Z STATES DS:0 DS:1 DS:2 TOTAL
22882024-02-03T17:10:28.887Z New 0 0 0 0
22892024-02-03T17:10:28.887Z Sent 0 1 29 30
22902024-02-03T17:10:28.887Z Done 30 29 1 60
22912024-02-03T17:10:28.887Z Skipped 0 0 0 0
22922024-02-03T17:10:28.887Z Error 0 0 0 0
22932024-02-03T17:10:28.887ZLast Flush: 1023 1023 0
22942024-02-03T17:10:28.887ZDownstairs last five completed:
22952024-02-03T17:10:28.888ZUpstairs last five completed: 30 29 28 27 26
22962024-02-03T17:10:28.888Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:10:28.424009675Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1209}
22972024-02-03T17:10:28.888ZWrote out file "/tmp/test_repair_verify.data"
22982024-02-03T17:10:28.891Z
22992024-02-03T17:10:28.896ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
23002024-02-03T17:10:28.921ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23012024-02-03T17:10:28.921Z 0 000-019 13 13 13 20 20 20 F F F
23022024-02-03T17:10:28.921Z 1 020-039 16 16 16 25 25 25 F F F
23032024-02-03T17:10:28.922Z 2 040-059 20 20 16 36 36 25 F F F <---
23042024-02-03T17:10:28.926Z 3 060-079 6 6 6 12 12 12 F F F
23052024-02-03T17:10:28.926Z 4 080-099 19 19 19 34 34 34 F F F
23062024-02-03T17:10:28.926Z 5 100-119 18 18 18 31 31 31 T T F <---
23072024-02-03T17:10:28.927Z 6 120-139 16 16 16 25 25 25 F F F
23082024-02-03T17:10:28.927Z 7 140-159 15 15 15 24 24 24 F F F
23092024-02-03T17:10:28.927Z 8 160-179 18 18 18 33 33 33 F F F
23102024-02-03T17:10:28.927Z 9 180-199 20 20 12 35 35 18 F F F <---
23112024-02-03T17:10:28.927Z 10 200-219 20 20 19 37 37 34 F F F <---
23122024-02-03T17:10:28.927Z 11 220-239 16 16 16 27 27 27 F F F
23132024-02-03T17:10:28.927Z 12 240-259 20 20 14 37 37 21 F F F <---
23142024-02-03T17:10:28.927Z 13 260-279 20 20 16 36 36 26 F F F <---
23152024-02-03T17:10:28.927Z 14 280-299 20 20 18 36 36 30 F F F <---
23162024-02-03T17:10:28.927Z 15 300-319 20 20 18 37 37 30 F F F <---
23172024-02-03T17:10:28.927Z 16 320-339 19 19 19 34 34 34 F F F
23182024-02-03T17:10:28.927Z 17 340-359 19 19 19 34 34 34 F F F
23192024-02-03T17:10:28.927Z 18 360-379 20 20 19 35 35 34 T T T <---
23202024-02-03T17:10:28.927Z 19 380-399 14 14 14 21 21 21 T T F <---
23212024-02-03T17:10:28.927Z 20 400-419 16 16 16 26 26 26 F F F
23222024-02-03T17:10:28.927Z 21 420-439 20 20 18 38 38 33 F F F <---
23232024-02-03T17:10:28.927Z 22 440-459 8 8 8 15 15 15 F F F
23242024-02-03T17:10:28.928Z 23 460-479 20 20 14 37 37 23 F F F <---
23252024-02-03T17:10:28.928Z 24 480-499 20 20 10 37 37 17 F F F <---
23262024-02-03T17:10:28.928Z 25 500-519 17 17 17 28 28 28 T T F <---
23272024-02-03T17:10:28.928Z 26 520-539 17 17 17 28 28 28 F F F
23282024-02-03T17:10:28.928Z 27 540-559 16 16 16 26 26 26 F F F
23292024-02-03T17:10:28.928Z 28 560-579 16 16 16 26 26 26 F F F
23302024-02-03T17:10:28.928Z 29 580-599 18 18 18 33 33 33 F F F
23312024-02-03T17:10:28.928ZMax gen: 20, Max flush: 38
23322024-02-03T17:10:28.928ZError: Difference in extent metadata found!
23332024-02-03T17:10:28.928ZA Difference in extent metadata is expected here
23342024-02-03T17:10:28.928ZOn loop 10
23352024-02-03T17:10:28.928Z
23362024-02-03T17:10:28.931ZVerifying data now
23372024-02-03T17:10:34.142ZNo change after verify
23382024-02-03T17:10:34.142ZLoop: 10 Downstairs dump after verify (and repair):
23392024-02-03T17:10:34.165ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23402024-02-03T17:10:34.165Z 0 000-019 13 13 13 20 20 20 F F F
23412024-02-03T17:10:34.165Z 1 020-039 16 16 16 25 25 25 F F F
23422024-02-03T17:10:34.165Z 2 040-059 20 20 20 36 36 36 F F F
23432024-02-03T17:10:34.165Z 3 060-079 6 6 6 12 12 12 F F F
23442024-02-03T17:10:34.169Z 4 080-099 19 19 19 34 34 34 F F F
23452024-02-03T17:10:34.169Z 5 100-119 21 21 21 39 39 39 F F F
23462024-02-03T17:10:34.169Z 6 120-139 16 16 16 25 25 25 F F F
23472024-02-03T17:10:34.169Z 7 140-159 15 15 15 24 24 24 F F F
23482024-02-03T17:10:34.169Z 8 160-179 18 18 18 33 33 33 F F F
23492024-02-03T17:10:34.169Z 9 180-199 20 20 20 35 35 35 F F F
23502024-02-03T17:10:34.169Z 10 200-219 20 20 20 37 37 37 F F F
23512024-02-03T17:10:34.169Z 11 220-239 16 16 16 27 27 27 F F F
23522024-02-03T17:10:34.169Z 12 240-259 20 20 20 37 37 37 F F F
23532024-02-03T17:10:34.169Z 13 260-279 20 20 20 36 36 36 F F F
23542024-02-03T17:10:34.169Z 14 280-299 20 20 20 36 36 36 F F F
23552024-02-03T17:10:34.169Z 15 300-319 20 20 20 37 37 37 F F F
23562024-02-03T17:10:34.169Z 16 320-339 19 19 19 34 34 34 F F F
23572024-02-03T17:10:34.169Z 17 340-359 19 19 19 34 34 34 F F F
23582024-02-03T17:10:34.169Z 18 360-379 21 21 21 39 39 39 F F F
23592024-02-03T17:10:34.169Z 19 380-399 21 21 21 39 39 39 F F F
23602024-02-03T17:10:34.169Z 20 400-419 16 16 16 26 26 26 F F F
23612024-02-03T17:10:34.169Z 21 420-439 20 20 20 38 38 38 F F F
23622024-02-03T17:10:34.169Z 22 440-459 8 8 8 15 15 15 F F F
23632024-02-03T17:10:34.169Z 23 460-479 20 20 20 37 37 37 F F F
23642024-02-03T17:10:34.169Z 24 480-499 20 20 20 37 37 37 F F F
23652024-02-03T17:10:34.169Z 25 500-519 21 21 21 39 39 39 F F F
23662024-02-03T17:10:34.170Z 26 520-539 17 17 17 28 28 28 F F F
23672024-02-03T17:10:34.170Z 27 540-559 16 16 16 26 26 26 F F F
23682024-02-03T17:10:34.170Z 28 560-579 16 16 16 26 26 26 F F F
23692024-02-03T17:10:34.170Z 29 580-599 18 18 18 33 33 33 F F F
23702024-02-03T17:10:34.170ZMax gen: 21, Max flush: 39
23712024-02-03T17:10:34.173Z
23722024-02-03T17:10:34.173ZBegin loop 11
23732024-02-03T17:10:34.173ZDownstairs to restart: 2
23742024-02-03T17:10:34.200Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.74218819Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217}
23752024-02-03T17:10:34.200Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.742785266Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217}
23762024-02-03T17:10:34.205Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.742876632Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217}
23772024-02-03T17:10:34.205Z{"msg":"Crucible runtime is spawned
23782024-02-03T17:10:34.205ZCrucible 19ff8904-90b0-4839-940a-7d9c9fb0e6cb has session id: 6daffb05-631e-4e84-8e81-7ca1c55b908b","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.743134065Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
23792024-02-03T17:10:34.205Z{"msg":"Upstairs opts: Upstairs UUID: 19ff8904-90b0-4839-940a-7d9c9fb0e6cb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.743243764Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
23802024-02-03T17:10:34.205Z{"msg":"Crucible stats registered with UUID: 19ff8904-90b0-4839-940a-7d9c9fb0e6cb","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.743426171Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
23812024-02-03T17:10:34.205Z{"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.743509907Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217}
23822024-02-03T17:10:34.205Z{"msg":"19ff8904-90b0-4839-940a-7d9c9fb0e6cb active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.743552981Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
23832024-02-03T17:10:34.205Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.743630909Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
23842024-02-03T17:10:34.205Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.743719019Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
23852024-02-03T17:10:34.205Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.743878991Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
23862024-02-03T17:10:34.206Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.743938193Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
23872024-02-03T17:10:34.206Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.743983295Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
23882024-02-03T17:10:34.206Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.744024817Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
23892024-02-03T17:10:34.206Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.744065625Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"":"io task","client":"0","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
23902024-02-03T17:10:34.206Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.744146602Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"":"io task","client":"2","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
23912024-02-03T17:10:34.206Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.744188064Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"":"io task","client":"1","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
23922024-02-03T17:10:34.206Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.744265799Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"":"io task","client":"0","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
23932024-02-03T17:10:34.206Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.744308106Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"":"io task","client":"2","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
23942024-02-03T17:10:34.206Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.744388424Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"":"io task","client":"1","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
23952024-02-03T17:10:34.206Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.745478983Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
23962024-02-03T17:10:34.206Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.745578847Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
23972024-02-03T17:10:34.206Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.745666467Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
23982024-02-03T17:10:34.206Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.745714157Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
23992024-02-03T17:10:34.206Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.746457348Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24002024-02-03T17:10:34.206Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.746746092Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24012024-02-03T17:10:34.214Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.746945835Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24022024-02-03T17:10:34.214Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.747021476Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24032024-02-03T17:10:34.214Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.747394783Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24042024-02-03T17:10:34.214Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.747519338Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24052024-02-03T17:10:34.214Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.747570815Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24062024-02-03T17:10:34.214Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.747914071Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24072024-02-03T17:10:34.214Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.747976812Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24082024-02-03T17:10:34.214Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.748002979Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24092024-02-03T17:10:34.214Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.748032573Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24102024-02-03T17:10:34.214Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.748757937Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24112024-02-03T17:10:34.215Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.748844207Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24122024-02-03T17:10:34.215Z{"msg":"[0]R flush_numbers[0..12]: [20, 25, 36, 12, 34, 39, 25, 24, 33, 35, 37, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.748876331Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24132024-02-03T17:10:34.215Z{"msg":"[0]R generation[0..12]: [13, 16, 20, 6, 19, 21, 16, 15, 18, 20, 20, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.748926048Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24142024-02-03T17:10:34.215Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.748953075Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24152024-02-03T17:10:34.215Z{"msg":"[1]R flush_numbers[0..12]: [20, 25, 36, 12, 34, 39, 25, 24, 33, 35, 37, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.749001278Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24162024-02-03T17:10:34.215Z{"msg":"[1]R generation[0..12]: [13, 16, 20, 6, 19, 21, 16, 15, 18, 20, 20, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.749026642Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24172024-02-03T17:10:34.215Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.749088922Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24182024-02-03T17:10:34.215Z{"msg":"[2]R flush_numbers[0..12]: [20, 25, 36, 12, 34, 39, 25, 24, 33, 35, 37, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.749150246Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24192024-02-03T17:10:34.215Z{"msg":"[2]R generation[0..12]: [13, 16, 20, 6, 19, 21, 16, 15, 18, 20, 20, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.749194582Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24202024-02-03T17:10:34.215Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.749238191Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24212024-02-03T17:10:34.215Z{"msg":"Max found gen is 22","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
24222024-02-03T17:10:34.215Z,"time":"2024-02-03T17:10:33.749303739Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24232024-02-03T17:10:34.215Z{"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.749349477Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24242024-02-03T17:10:34.215Z{"msg":"Next flush: 40","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.749387476Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24252024-02-03T17:10:34.215Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.749427759Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24262024-02-03T17:10:34.215Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.749466436Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24272024-02-03T17:10:34.215Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.749501291Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24282024-02-03T17:10:34.215Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.749541523Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24292024-02-03T17:10:34.215Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.749606437Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24302024-02-03T17:10:34.215Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.749650291Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24312024-02-03T17:10:34.215Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.749756467Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24322024-02-03T17:10:34.215Z{"msg":"19ff8904-90b0-4839-940a-7d9c9fb0e6cb is now active with session: 6daffb05-631e-4e84-8e81-7ca1c55b908b","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.749802421Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24332024-02-03T17:10:34.215Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.749834281Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217,"session_id":"6daffb05-631e-4e84-8e81-7ca1c55b908b"}
24342024-02-03T17:10:34.215Z{"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:33.749856224Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217}
24352024-02-03T17:10:34.216ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
24362024-02-03T17:10:34.216ZLoading write count information from file "/tmp/test_repair_verify.data"
24372024-02-03T17:10:34.216ZRun Repair workload
24382024-02-03T17:10:34.216Z01/30 Write block 81 len 4096 data: 4
24392024-02-03T17:10:34.219Z02/30 Write block 399 len 12288 data: 3 3 3
24402024-02-03T17:10:34.219Z03/30 Write block 325 len 36864 data: 2 2 2 2 3 3 3 4 4
24412024-02-03T17:10:34.219Z04/30 Write block 93 len 36864 data: 3 3 2 2 2 2 2 2 2
24422024-02-03T17:10:34.219Z05/30 Flush
24432024-02-03T17:10:34.222Z06/30 Write block 385 len 12288 data: 3 3 4
24442024-02-03T17:10:34.222Z07/30 Flush
24452024-02-03T17:10:34.226Z08/30 Flush
24462024-02-03T17:10:34.226Z09/30 Write block 8 len 4096 data: 4
24472024-02-03T17:10:34.226Z10/30 Read block 101 len 16384
24482024-02-03T17:10:34.229Z11/30 Read block 422 len 4096
24492024-02-03T17:10:34.229Z12/30 Read block 110 len 32768
24502024-02-03T17:10:34.232Z13/30 Write block 241 len 36864 data: 3 3 3 3 3 3 2 4 4
24512024-02-03T17:10:34.232Z14/30 Read block 38 len 8192
24522024-02-03T17:10:34.235Z15/30 Write block 27 len 40960 data: 4 4 3 4 4 4 3 3 3 3
24532024-02-03T17:10:34.235Z16/30 Read block 13 len 4096
24542024-02-03T17:10:34.237Z17/30 Write block 523 len 32768 data: 2 2 3 3 3 3 3 3
24552024-02-03T17:10:34.238Z18/30 Read block 564 len 20480
24562024-02-03T17:10:34.240Z19/30 Write block 99 len 12288 data: 3 3 3
24572024-02-03T17:10:34.241Z20/30 Read block 135 len 4096
24582024-02-03T17:10:34.241Z21/30 Write block 553 len 12288 data: 3 3 3
24592024-02-03T17:10:34.244Z22/30 Read block 28 len 32768
24602024-02-03T17:10:34.244Z23/30 Read block 444 len 36864
24612024-02-03T17:10:34.247Z24/30 Read block 24 len 8192
24622024-02-03T17:10:34.247Z25/30 Write block 301 len 16384 data: 4 4 4 4
24632024-02-03T17:10:34.247Z26/30 Read block 472 len 12288
24642024-02-03T17:10:34.250Z27/30 Read block 321 len 8192
24652024-02-03T17:10:34.250Z28/30 Write block 334 len 28672 data: 4 3 3 3 3 2 2
24662024-02-03T17:10:34.253Z29/30 Read block 437 len 4096
24672024-02-03T17:10:34.253Z30/30 Read block 441 len 24576
24682024-02-03T17:10:34.256Z----------------------------------------------------------------
24692024-02-03T17:10:34.256Z Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:30
24702024-02-03T17:10:34.256ZStates: Active Active Active
24712024-02-03T17:10:34.256ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
24722024-02-03T17:10:34.256Z 1 Acked 1000 Write 1 Done Done Done false
24732024-02-03T17:10:34.256Z 2 Acked 1001 Write 3 Done Done Done false
24742024-02-03T17:10:34.256Z 3 Acked 1002 Write 9 Done Done Sent false
24752024-02-03T17:10:34.256Z 4 Acked 1003 Write 9 Done Done Sent false
24762024-02-03T17:10:34.256Z 5 Acked 1004 Flush 0 Done Done Sent false
24772024-02-03T17:10:34.256Z 6 Acked 1005 Write 3 Done Done Sent false
24782024-02-03T17:10:34.256Z 7 Acked 1006 Flush 0 Done Done Sent false
24792024-02-03T17:10:34.257Z 8 Acked 1007 Flush 0 Done Done Sent false
24802024-02-03T17:10:34.257Z 9 Acked 1008 Write 1 Done Done Sent false
24812024-02-03T17:10:34.257Z 10 Acked 1009 Read 4 Done Done Sent false
24822024-02-03T17:10:34.257Z 11 Acked 1010 Read 1 Done Done Sent false
24832024-02-03T17:10:34.257Z 12 Acked 1011 Read 8 Done Done Sent false
24842024-02-03T17:10:34.257Z 13 Acked 1012 Write 9 Done Done Sent false
24852024-02-03T17:10:34.257Z 14 Acked 1013 Read 2 Done Done Sent false
24862024-02-03T17:10:34.257Z 15 Acked 1014 Write 10 Done Done Sent false
24872024-02-03T17:10:34.257Z 16 Acked 1015 Read 1 Done Done Sent false
24882024-02-03T17:10:34.257Z 17 Acked 1016 Write 8 Done Done Sent false
24892024-02-03T17:10:34.257Z 18 Acked 1017 Read 5 Done Done Sent false
24902024-02-03T17:10:34.257Z 19 Acked 1018 Write 3 Done Done Sent false
24912024-02-03T17:10:34.257Z 20 Acked 1019 Read 1 Done Done Sent false
24922024-02-03T17:10:34.257Z 21 Acked 1020 Write 3 Done Done Sent false
24932024-02-03T17:10:34.257Z 22 Acked 1021 Read 8 Done Done Sent false
24942024-02-03T17:10:34.257Z 23 Acked 1022 Read 9 Done Done Sent false
24952024-02-03T17:10:34.257Z 24 Acked 1023 Read 2 Done Done Sent false
24962024-02-03T17:10:34.257Z 25 Acked 1024 Write 4 Done Done Sent false
24972024-02-03T17:10:34.257Z 26 Acked 1025 Read 3 Done Done Sent false
24982024-02-03T17:10:34.257Z 27 Acked 1026 Read 2 Done Done Sent false
24992024-02-03T17:10:34.257Z 28 Acked 1027 Write 7 Done Done Sent false
25002024-02-03T17:10:34.257Z 29 Acked 1028 Read 1 Done Done Sent false
25012024-02-03T17:10:34.257Z 30 Acked 1029 Read 6 Done Sent Sent false
25022024-02-03T17:10:34.257Z STATES DS:0 DS:1 DS:2 TOTAL
25032024-02-03T17:10:34.258Z New 0 0 0 0
25042024-02-03T17:10:34.258Z Sent 0 1 28 29
25052024-02-03T17:10:34.258Z Done 30 29 2 61
25062024-02-03T17:10:34.258Z Skipped 0 0 0 0
25072024-02-03T17:10:34.258Z Error 0 0 0 0
25082024-02-03T17:10:34.258ZLast Flush: 1007 1007 0
25092024-02-03T17:10:34.258ZDownstairs last five completed:
25102024-02-03T17:10:34.258ZUpstairs last five completed: 30 29 28 27 26
25112024-02-03T17:10:34.258Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:10:33.796823687Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1217}
25122024-02-03T17:10:34.258ZWrote out file "/tmp/test_repair_verify.data"
25132024-02-03T17:10:34.261Z
25142024-02-03T17:10:34.267ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
25152024-02-03T17:10:34.300ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25162024-02-03T17:10:34.300Z 0 000-019 13 13 13 20 20 20 T T F <---
25172024-02-03T17:10:34.303Z 1 020-039 16 16 16 25 25 25 T T F <---
25182024-02-03T17:10:34.303Z 2 040-059 20 20 20 36 36 36 F F F
25192024-02-03T17:10:34.303Z 3 060-079 6 6 6 12 12 12 F F F
25202024-02-03T17:10:34.303Z 4 080-099 22 22 19 40 40 34 T T T <---
25212024-02-03T17:10:34.303Z 5 100-119 22 22 21 40 40 39 T T F <---
25222024-02-03T17:10:34.304Z 6 120-139 16 16 16 25 25 25 F F F
25232024-02-03T17:10:34.304Z 7 140-159 15 15 15 24 24 24 F F F
25242024-02-03T17:10:34.304Z 8 160-179 18 18 18 33 33 33 F F F
25252024-02-03T17:10:34.304Z 9 180-199 20 20 20 35 35 35 F F F
25262024-02-03T17:10:34.304Z 10 200-219 20 20 20 37 37 37 F F F
25272024-02-03T17:10:34.304Z 11 220-239 16 16 16 27 27 27 F F F
25282024-02-03T17:10:34.304Z 12 240-259 20 20 20 37 37 37 T T F <---
25292024-02-03T17:10:34.304Z 13 260-279 20 20 20 36 36 36 F F F
25302024-02-03T17:10:34.304Z 14 280-299 20 20 20 36 36 36 F F F
25312024-02-03T17:10:34.304Z 15 300-319 20 20 20 37 37 37 T T F <---
25322024-02-03T17:10:34.304Z 16 320-339 22 22 19 40 40 34 T T F <---
25332024-02-03T17:10:34.304Z 17 340-359 19 19 19 34 34 34 T T F <---
25342024-02-03T17:10:34.304Z 18 360-379 21 21 21 39 39 39 F F F
25352024-02-03T17:10:34.304Z 19 380-399 22 22 21 41 41 39 F F T <---
25362024-02-03T17:10:34.304Z 20 400-419 22 22 16 40 40 26 F F T <---
25372024-02-03T17:10:34.304Z 21 420-439 20 20 20 38 38 38 F F F
25382024-02-03T17:10:34.304Z 22 440-459 8 8 8 15 15 15 F F F
25392024-02-03T17:10:34.304Z 23 460-479 20 20 20 37 37 37 F F F
25402024-02-03T17:10:34.304Z 24 480-499 20 20 20 37 37 37 F F F
25412024-02-03T17:10:34.304Z 25 500-519 21 21 21 39 39 39 F F F
25422024-02-03T17:10:34.304Z 26 520-539 17 17 17 28 28 28 T T F <---
25432024-02-03T17:10:34.304Z 27 540-559 16 16 16 26 26 26 T T F <---
25442024-02-03T17:10:34.304Z 28 560-579 16 16 16 26 26 26 F F F
25452024-02-03T17:10:34.304Z 29 580-599 18 18 18 33 33 33 F F F
25462024-02-03T17:10:34.305ZMax gen: 22, Max flush: 41
25472024-02-03T17:10:34.305ZError: Difference in extent metadata found!
25482024-02-03T17:10:34.305ZA Difference in extent metadata is expected here
25492024-02-03T17:10:34.305ZOn loop 11
25502024-02-03T17:10:34.305Z
25512024-02-03T17:10:34.308ZVerifying data now
25522024-02-03T17:10:40.383ZNo change after verify
25532024-02-03T17:10:40.383ZLoop: 11 Downstairs dump after verify (and repair):
25542024-02-03T17:10:40.406ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25552024-02-03T17:10:40.406Z 0 000-019 23 23 23 42 42 42 F F F
25562024-02-03T17:10:40.406Z 1 020-039 23 23 23 42 42 42 F F F
25572024-02-03T17:10:40.410Z 2 040-059 20 20 20 36 36 36 F F F
25582024-02-03T17:10:40.410Z 3 060-079 6 6 6 12 12 12 F F F
25592024-02-03T17:10:40.411Z 4 080-099 23 23 23 42 42 42 F F F
25602024-02-03T17:10:40.411Z 5 100-119 23 23 23 42 42 42 F F F
25612024-02-03T17:10:40.411Z 6 120-139 16 16 16 25 25 25 F F F
25622024-02-03T17:10:40.411Z 7 140-159 15 15 15 24 24 24 F F F
25632024-02-03T17:10:40.411Z 8 160-179 18 18 18 33 33 33 F F F
25642024-02-03T17:10:40.411Z 9 180-199 20 20 20 35 35 35 F F F
25652024-02-03T17:10:40.411Z 10 200-219 20 20 20 37 37 37 F F F
25662024-02-03T17:10:40.411Z 11 220-239 16 16 16 27 27 27 F F F
25672024-02-03T17:10:40.411Z 12 240-259 23 23 23 42 42 42 F F F
25682024-02-03T17:10:40.411Z 13 260-279 20 20 20 36 36 36 F F F
25692024-02-03T17:10:40.411Z 14 280-299 20 20 20 36 36 36 F F F
25702024-02-03T17:10:40.411Z 15 300-319 23 23 23 42 42 42 F F F
25712024-02-03T17:10:40.412Z 16 320-339 23 23 23 42 42 42 F F F
25722024-02-03T17:10:40.412Z 17 340-359 23 23 23 42 42 42 F F F
25732024-02-03T17:10:40.412Z 18 360-379 21 21 21 39 39 39 F F F
25742024-02-03T17:10:40.412Z 19 380-399 22 22 22 41 41 41 F F F
25752024-02-03T17:10:40.412Z 20 400-419 22 22 22 40 40 40 F F F
25762024-02-03T17:10:40.412Z 21 420-439 20 20 20 38 38 38 F F F
25772024-02-03T17:10:40.412Z 22 440-459 8 8 8 15 15 15 F F F
25782024-02-03T17:10:40.412Z 23 460-479 20 20 20 37 37 37 F F F
25792024-02-03T17:10:40.412Z 24 480-499 20 20 20 37 37 37 F F F
25802024-02-03T17:10:40.412Z 25 500-519 21 21 21 39 39 39 F F F
25812024-02-03T17:10:40.412Z 26 520-539 23 23 23 42 42 42 F F F
25822024-02-03T17:10:40.412Z 27 540-559 23 23 23 42 42 42 F F F
25832024-02-03T17:10:40.413Z 28 560-579 16 16 16 26 26 26 F F F
25842024-02-03T17:10:40.413Z 29 580-599 18 18 18 33 33 33 F F F
25852024-02-03T17:10:40.413ZMax gen: 23, Max flush: 42
25862024-02-03T17:10:40.418Z
25872024-02-03T17:10:40.418ZBegin loop 12
25882024-02-03T17:10:40.418ZDownstairs to restart: 0
25892024-02-03T17:10:40.443Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.985597033Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225}
25902024-02-03T17:10:40.443Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.986126156Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225}
25912024-02-03T17:10:40.443Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.98616056Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225}
25922024-02-03T17:10:40.449Z{"msg":"Crucible d03ceda9-fd42-4f03-83f2-2c6ec2ca0627 has session id: 2639f3f9-8724-44ff-ad69-7ef757db86f4","v":0,"name":"crucible","level":30Crucible runtime is spawned
25932024-02-03T17:10:40.449Z,"time":"2024-02-03T17:10:39.98623716Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
25942024-02-03T17:10:40.449Z{"msg":"Upstairs opts: Upstairs UUID: d03ceda9-fd42-4f03-83f2-2c6ec2ca0627, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.986348079Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
25952024-02-03T17:10:40.449Z{"msg":"Crucible stats registered with UUID: d03ceda9-fd42-4f03-83f2-2c6ec2ca0627","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.986423018Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
25962024-02-03T17:10:40.449Z{"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.98646342Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225}
25972024-02-03T17:10:40.449Z{"msg":"d03ceda9-fd42-4f03-83f2-2c6ec2ca0627 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.986518849Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
25982024-02-03T17:10:40.450Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.986542559Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
25992024-02-03T17:10:40.450Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.986565751Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26002024-02-03T17:10:40.450Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.986627775Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26012024-02-03T17:10:40.450Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.986673949Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26022024-02-03T17:10:40.450Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.98671352Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26032024-02-03T17:10:40.450Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.986751234Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26042024-02-03T17:10:40.450Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.98682479Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"":"io task","client":"2","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26052024-02-03T17:10:40.450Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.98688521Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"":"io task","client":"0","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26062024-02-03T17:10:40.450Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.987051298Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"":"io task","client":"1","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26072024-02-03T17:10:40.450Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.987141724Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"":"io task","client":"2","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26082024-02-03T17:10:40.450Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.987187841Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"":"io task","client":"1","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26092024-02-03T17:10:40.450Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.987358289Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"":"io task","client":"0","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26102024-02-03T17:10:40.451Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.988422996Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26112024-02-03T17:10:40.451Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.988519356Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26122024-02-03T17:10:40.451Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.988722903Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26132024-02-03T17:10:40.451Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.988827639Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26142024-02-03T17:10:40.451Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.989208049Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26152024-02-03T17:10:40.451Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.989260222Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26162024-02-03T17:10:40.451Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.98930196Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26172024-02-03T17:10:40.451Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.989815374Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26182024-02-03T17:10:40.461Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.990263124Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26192024-02-03T17:10:40.462Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.990403007Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26202024-02-03T17:10:40.462Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.990489835Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26212024-02-03T17:10:40.462Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.990536903Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26222024-02-03T17:10:40.462Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.990618152Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26232024-02-03T17:10:40.462Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.990648594Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26242024-02-03T17:10:40.462Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.990698017Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26252024-02-03T17:10:40.462Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.991277274Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26262024-02-03T17:10:40.462Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.991386638Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26272024-02-03T17:10:40.462Z{"msg":"[0]R flush_numbers[0..12]: [42, 42, 36, 12, 42, 42, 25, 24, 33, 35, 37, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.991461737Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26282024-02-03T17:10:40.462Z{"msg":"[0]R generation[0..12]: [23, 23, 20, 6, 23, 23, 16, 15, 18, 20, 20, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.991489548Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26292024-02-03T17:10:40.462Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.99154144Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26302024-02-03T17:10:40.462Z{"msg":"[1]R flush_numbers[0..12]: [42, 42, 36, 12, 42, 42, 25, 24, 33, 35, 37, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.991567748Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26312024-02-03T17:10:40.462Z{"msg":"[1]R generation[0..12]: [23, 23, 20, 6, 23, 23, 16, 15, 18, 20, 20, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.991591089Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26322024-02-03T17:10:40.462Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.991615213Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26332024-02-03T17:10:40.462Z{"msg":"[2]R flush_numbers[0..12]: [42, 42, 36, 12, 42, 42, 25, 24, 33, 35, 37, 27]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.9916383Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26342024-02-03T17:10:40.462Z{"msg":"Wait for a query_work_queue command to finish before sending IO
26352024-02-03T17:10:40.462Z[2]R generation[0..12]: [23, 23, 20, 6, 23, 23, 16, 15, 18, 20, 20, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.991692188Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26362024-02-03T17:10:40.462Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.99171857Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26372024-02-03T17:10:40.462Z{"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.991771036Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26382024-02-03T17:10:40.462Z{"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.99179576Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26392024-02-03T17:10:40.462Z{"msg":"Next flush: 43","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.991844968Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26402024-02-03T17:10:40.462Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.991869285Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26412024-02-03T17:10:40.463Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.991942453Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26422024-02-03T17:10:40.463Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.991964735Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26432024-02-03T17:10:40.463Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.991992862Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26442024-02-03T17:10:40.463Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.992073626Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26452024-02-03T17:10:40.463Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.992098934Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26462024-02-03T17:10:40.463Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.992119986Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26472024-02-03T17:10:40.463Z{"msg":"d03ceda9-fd42-4f03-83f2-2c6ec2ca0627 is now active with session: 2639f3f9-8724-44ff-ad69-7ef757db86f4","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.992141207Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26482024-02-03T17:10:40.463Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.992188465Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225,"session_id":"2639f3f9-8724-44ff-ad69-7ef757db86f4"}
26492024-02-03T17:10:40.463Z{"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:39.992211193Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225}
26502024-02-03T17:10:40.463ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
26512024-02-03T17:10:40.463ZLoading write count information from file "/tmp/test_repair_verify.data"
26522024-02-03T17:10:40.463ZRun Repair workload
26532024-02-03T17:10:40.463Z01/30 Write block 106 len 28672 data: 3 3 3 3 3 4 4
26542024-02-03T17:10:40.463Z02/30 Read block 121 len 4096
26552024-02-03T17:10:40.467Z03/30 Read block 447 len 16384
26562024-02-03T17:10:40.467Z04/30 Read block 536 len 8192
26572024-02-03T17:10:40.467Z05/30 Read block 48 len 24576
26582024-02-03T17:10:40.467Z06/30 Read block 517 len 32768
26592024-02-03T17:10:40.467Z07/30 Read block 435 len 16384
26602024-02-03T17:10:40.467Z08/30 Read block 434 len 28672
26612024-02-03T17:10:40.467Z09/30 Write block 109 len 40960 data: 4 4 5 5 3 4 4 4 6 5
26622024-02-03T17:10:40.467Z10/30 Read block 388 len 8192
26632024-02-03T17:10:40.471Z11/30 Write block 81 len 40960 data: 5 4 4 3 4 4 3 3 3 3
26642024-02-03T17:10:40.471Z12/30 Read block 360 len 40960
26652024-02-03T17:10:40.474Z13/30 Read block 175 len 28672
26662024-02-03T17:10:40.474Z14/30 Read block 193 len 4096
26672024-02-03T17:10:40.474Z15/30 Read block 352 len 40960
26682024-02-03T17:10:40.478Z16/30 Read block 247 len 24576
26692024-02-03T17:10:40.478Z17/30 Read block 541 len 12288
26702024-02-03T17:10:40.482Z18/30 Read block 540 len 4096
26712024-02-03T17:10:40.482Z19/30 Write block 509 len 28672 data: 5 5 5 4 4 3 3
26722024-02-03T17:10:40.482Z20/30 Read block 24 len 8192
26732024-02-03T17:10:40.486Z21/30 Write block 416 len 12288 data: 3 3 3
26742024-02-03T17:10:40.486Z22/30 Read block 339 len 20480
26752024-02-03T17:10:40.486Z23/30 Read block 252 len 8192
26762024-02-03T17:10:40.489Z24/30 Flush
26772024-02-03T17:10:40.493Z25/30 Write block 592 len 20480 data: 3 3 3 3 4
26782024-02-03T17:10:40.493Z26/30 Write block 221 len 12288 data: 2 2 2
26792024-02-03T17:10:40.493Z27/30 Write block 347 len 28672 data: 2 4 4 3 3 3 4
26802024-02-03T17:10:40.496Z28/30 Read block 494 len 28672
26812024-02-03T17:10:40.496Z29/30 Read block 501 len 12288
26822024-02-03T17:10:40.501Z30/30 Write block 206 len 12288 data: 4 4 4
26832024-02-03T17:10:40.501Z----------------------------------------------------------------
26842024-02-03T17:10:40.501Z Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:30
26852024-02-03T17:10:40.501ZStates: Active Active Active
26862024-02-03T17:10:40.501ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
26872024-02-03T17:10:40.501Z 1 Acked 1000 Write 7 Done Done Done false
26882024-02-03T17:10:40.501Z 2 Acked 1001 Read 1 Done Done Done false
26892024-02-03T17:10:40.502Z 3 Acked 1002 Read 4 Sent Done Done false
26902024-02-03T17:10:40.502Z 4 Acked 1003 Read 2 Sent Done Done false
26912024-02-03T17:10:40.502Z 5 Acked 1004 Read 6 Sent Done Done false
26922024-02-03T17:10:40.502Z 6 Acked 1005 Read 8 Sent Done Done false
26932024-02-03T17:10:40.502Z 7 Acked 1006 Read 4 Sent Done Done false
26942024-02-03T17:10:40.502Z 8 Acked 1007 Read 7 Sent Done Done false
26952024-02-03T17:10:40.502Z 9 Acked 1008 Write 10 Sent Done Done false
26962024-02-03T17:10:40.502Z 10 Acked 1009 Read 2 Sent Done Done false
26972024-02-03T17:10:40.502Z 11 Acked 1010 Write 10 Sent Done Done false
26982024-02-03T17:10:40.502Z 12 Acked 1011 Read 10 Sent Done Done false
26992024-02-03T17:10:40.502Z 13 Acked 1012 Read 7 Sent Done Done false
27002024-02-03T17:10:40.502Z 14 Acked 1013 Read 1 Sent Done Done false
27012024-02-03T17:10:40.502Z 15 Acked 1014 Read 10 Sent Done Done false
27022024-02-03T17:10:40.502Z 16 Acked 1015 Read 6 Sent Done Done false
27032024-02-03T17:10:40.502Z 17 Acked 1016 Read 3 Sent Done Done false
27042024-02-03T17:10:40.502Z 18 Acked 1017 Read 1 Sent Done Done false
27052024-02-03T17:10:40.502Z 19 Acked 1018 Write 7 Sent Done Done false
27062024-02-03T17:10:40.502Z 20 Acked 1019 Read 2 Sent Done Done false
27072024-02-03T17:10:40.502Z 21 Acked 1020 Write 3 Sent Done Done false
27082024-02-03T17:10:40.503Z 22 Acked 1021 Read 5 Sent Done Done false
27092024-02-03T17:10:40.503Z 23 Acked 1022 Read 2 Sent Done Done false
27102024-02-03T17:10:40.503Z 24 Acked 1023 Flush 0 Sent Done Done false
27112024-02-03T17:10:40.503Z 25 Acked 1024 Write 5 Sent Done Done false
27122024-02-03T17:10:40.503Z 26 Acked 1025 Write 3 Sent Done Done false
27132024-02-03T17:10:40.503Z 27 Acked 1026 Write 7 Sent Done Done false
27142024-02-03T17:10:40.503Z 28 Acked 1027 Read 7 Sent Done Done false
27152024-02-03T17:10:40.503Z 29 Acked 1028 Read 3 Sent Done Done false
27162024-02-03T17:10:40.503Z 30 Acked 1029 Write 3 Sent Sent Sent false
27172024-02-03T17:10:40.503Z STATES DS:0 DS:1 DS:2 TOTAL
27182024-02-03T17:10:40.503Z New 0 0 0 0
27192024-02-03T17:10:40.503Z Sent 28 1 1 30
27202024-02-03T17:10:40.503Z Done 2 29 29 60
27212024-02-03T17:10:40.503Z Skipped 0 0 0 0
27222024-02-03T17:10:40.503Z Error 0 0 0 0
27232024-02-03T17:10:40.503ZLast Flush: 0 1023 1023
27242024-02-03T17:10:40.503ZDownstairs last five completed:
27252024-02-03T17:10:40.503ZUpstairs last five completed: 30 29 28 27 26
27262024-02-03T17:10:40.503Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:10:40.041990039Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1225}
27272024-02-03T17:10:40.503ZWrote out file "/tmp/test_repair_verify.data"
27282024-02-03T17:10:40.508Z
27292024-02-03T17:10:40.513ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
27302024-02-03T17:10:40.539ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27312024-02-03T17:10:40.539Z 0 000-019 23 23 23 42 42 42 F F F
27322024-02-03T17:10:40.539Z 1 020-039 23 23 23 42 42 42 F F F
27332024-02-03T17:10:40.539Z 2 040-059 20 20 20 36 36 36 F F F
27342024-02-03T17:10:40.539Z 3 060-079 6 6 6 12 12 12 F F F
27352024-02-03T17:10:40.544Z 4 080-099 23 24 24 42 43 43 F F F <---
27362024-02-03T17:10:40.544Z 5 100-119 23 24 24 42 43 43 T F F <---
27372024-02-03T17:10:40.544Z 6 120-139 16 16 16 25 25 25 F F F
27382024-02-03T17:10:40.544Z 7 140-159 15 15 15 24 24 24 F F F
27392024-02-03T17:10:40.544Z 8 160-179 18 18 18 33 33 33 F F F
27402024-02-03T17:10:40.544Z 9 180-199 20 20 20 35 35 35 F F F
27412024-02-03T17:10:40.544Z 10 200-219 20 20 20 37 37 37 F T T <---
27422024-02-03T17:10:40.545Z 11 220-239 16 16 16 27 27 27 F T T <---
27432024-02-03T17:10:40.545Z 12 240-259 23 23 23 42 42 42 F F F
27442024-02-03T17:10:40.545Z 13 260-279 20 20 20 36 36 36 F F F
27452024-02-03T17:10:40.545Z 14 280-299 20 20 20 36 36 36 F F F
27462024-02-03T17:10:40.545Z 15 300-319 23 23 23 42 42 42 F F F
27472024-02-03T17:10:40.545Z 16 320-339 23 23 23 42 42 42 F F F
27482024-02-03T17:10:40.545Z 17 340-359 23 23 23 42 42 42 F T T <---
27492024-02-03T17:10:40.545Z 18 360-379 21 21 21 39 39 39 F F F
27502024-02-03T17:10:40.545Z 19 380-399 22 22 22 41 41 41 F F F
27512024-02-03T17:10:40.545Z 20 400-419 22 24 24 40 43 43 F F F <---
27522024-02-03T17:10:40.545Z 21 420-439 20 20 20 38 38 38 F F F
27532024-02-03T17:10:40.545Z 22 440-459 8 8 8 15 15 15 F F F
27542024-02-03T17:10:40.545Z 23 460-479 20 20 20 37 37 37 F F F
27552024-02-03T17:10:40.545Z 24 480-499 20 20 20 37 37 37 F F F
27562024-02-03T17:10:40.545Z 25 500-519 21 24 24 39 43 43 F F F <---
27572024-02-03T17:10:40.545Z 26 520-539 23 23 23 42 42 42 F F F
27582024-02-03T17:10:40.546Z 27 540-559 23 23 23 42 42 42 F F F
27592024-02-03T17:10:40.546Z 28 560-579 16 16 16 26 26 26 F F F
27602024-02-03T17:10:40.546Z 29 580-599 18 18 18 33 33 33 F T T <---
27612024-02-03T17:10:40.546ZMax gen: 24, Max flush: 43
27622024-02-03T17:10:40.546ZError: Difference in extent metadata found!
27632024-02-03T17:10:40.546ZA Difference in extent metadata is expected here
27642024-02-03T17:10:40.546ZOn loop 12
27652024-02-03T17:10:40.546Z
27662024-02-03T17:10:40.550ZVerifying data now
27672024-02-03T17:10:45.326ZNo change after verify
27682024-02-03T17:10:45.326ZLoop: 12 Downstairs dump after verify (and repair):
27692024-02-03T17:10:45.350ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27702024-02-03T17:10:45.350Z 0 000-019 23 23 23 42 42 42 F F F
27712024-02-03T17:10:45.350Z 1 020-039 23 23 23 42 42 42 F F F
27722024-02-03T17:10:45.353Z 2 040-059 20 20 20 36 36 36 F F F
27732024-02-03T17:10:45.353Z 3 060-079 6 6 6 12 12 12 F F F
27742024-02-03T17:10:45.353Z 4 080-099 24 24 24 43 43 43 F F F
27752024-02-03T17:10:45.353Z 5 100-119 24 24 24 43 43 43 F F F
27762024-02-03T17:10:45.353Z 6 120-139 16 16 16 25 25 25 F F F
27772024-02-03T17:10:45.353Z 7 140-159 15 15 15 24 24 24 F F F
27782024-02-03T17:10:45.353Z 8 160-179 18 18 18 33 33 33 F F F
27792024-02-03T17:10:45.353Z 9 180-199 20 20 20 35 35 35 F F F
27802024-02-03T17:10:45.353Z 10 200-219 25 25 25 44 44 44 F F F
27812024-02-03T17:10:45.353Z 11 220-239 25 25 25 44 44 44 F F F
27822024-02-03T17:10:45.353Z 12 240-259 23 23 23 42 42 42 F F F
27832024-02-03T17:10:45.353Z 13 260-279 20 20 20 36 36 36 F F F
27842024-02-03T17:10:45.353Z 14 280-299 20 20 20 36 36 36 F F F
27852024-02-03T17:10:45.353Z 15 300-319 23 23 23 42 42 42 F F F
27862024-02-03T17:10:45.354Z 16 320-339 23 23 23 42 42 42 F F F
27872024-02-03T17:10:45.354Z 17 340-359 25 25 25 44 44 44 F F F
27882024-02-03T17:10:45.354Z 18 360-379 21 21 21 39 39 39 F F F
27892024-02-03T17:10:45.354Z 19 380-399 22 22 22 41 41 41 F F F
27902024-02-03T17:10:45.354Z 20 400-419 24 24 24 43 43 43 F F F
27912024-02-03T17:10:45.354Z 21 420-439 20 20 20 38 38 38 F F F
27922024-02-03T17:10:45.354Z 22 440-459 8 8 8 15 15 15 F F F
27932024-02-03T17:10:45.354Z 23 460-479 20 20 20 37 37 37 F F F
27942024-02-03T17:10:45.354Z 24 480-499 20 20 20 37 37 37 F F F
27952024-02-03T17:10:45.354Z 25 500-519 24 24 24 43 43 43 F F F
27962024-02-03T17:10:45.354Z 26 520-539 23 23 23 42 42 42 F F F
27972024-02-03T17:10:45.354Z 27 540-559 23 23 23 42 42 42 F F F
27982024-02-03T17:10:45.354Z 28 560-579 16 16 16 26 26 26 F F F
27992024-02-03T17:10:45.354Z 29 580-599 25 25 25 44 44 44 F F F
28002024-02-03T17:10:45.354ZMax gen: 25, Max flush: 44
28012024-02-03T17:10:45.357Z
28022024-02-03T17:10:45.357ZBegin loop 13
28032024-02-03T17:10:45.357ZDownstairs to restart: 2
28042024-02-03T17:10:45.389Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.93139978Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233}
28052024-02-03T17:10:45.389Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.931850041Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233}
28062024-02-03T17:10:45.389Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
28072024-02-03T17:10:45.389Z,"time":"2024-02-03T17:10:44.931905667Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233}
28082024-02-03T17:10:45.389Z{"msg":"Crucible 849c4fdd-1790-446c-8412-38a1c1bdbe8c has session id: 3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.931958026Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28092024-02-03T17:10:45.394Z{"msg":"Upstairs opts: Upstairs UUID: 849c4fdd-1790-446c-8412-38a1c1bdbe8c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.932016825Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28102024-02-03T17:10:45.394Z{"msg":"Crucible stats registered with UUID: 849c4fdd-1790-446c-8412-38a1c1bdbe8c","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.932060715Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28112024-02-03T17:10:45.394Z{"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.932101853Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233}
28122024-02-03T17:10:45.394Z{"msg":"849c4fdd-1790-446c-8412-38a1c1bdbe8c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.932433114Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28132024-02-03T17:10:45.394Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.932504649Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28142024-02-03T17:10:45.394Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.932550734Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28152024-02-03T17:10:45.394Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.932658827Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28162024-02-03T17:10:45.394Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.932702622Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28172024-02-03T17:10:45.394Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.93274523Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28182024-02-03T17:10:45.394Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.932791043Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28192024-02-03T17:10:45.394Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.932876054Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"":"io task","client":"0","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28202024-02-03T17:10:45.394Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.932986421Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"":"io task","client":"2","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28212024-02-03T17:10:45.394Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.933066485Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"":"io task","client":"1","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28222024-02-03T17:10:45.394Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.933095236Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"":"io task","client":"2","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28232024-02-03T17:10:45.394Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.93311887Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"":"io task","client":"0","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28242024-02-03T17:10:45.394Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.933143353Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"":"io task","client":"1","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28252024-02-03T17:10:45.394Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.93475221Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28262024-02-03T17:10:45.394Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.934834952Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28272024-02-03T17:10:45.394Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.93488179Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28282024-02-03T17:10:45.394Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.934927977Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28292024-02-03T17:10:45.394Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.935606306Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28302024-02-03T17:10:45.394Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.935709356Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28312024-02-03T17:10:45.395Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.936086005Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28322024-02-03T17:10:45.395Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.936159166Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28332024-02-03T17:10:45.399Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.9370098Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28342024-02-03T17:10:45.400Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.937109504Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28352024-02-03T17:10:45.400Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.937142694Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28362024-02-03T17:10:45.400Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.937166413Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28372024-02-03T17:10:45.400Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.937216289Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28382024-02-03T17:10:45.400Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.937241431Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28392024-02-03T17:10:45.400Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.937263324Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28402024-02-03T17:10:45.400Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.938166767Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28412024-02-03T17:10:45.400Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.938267204Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28422024-02-03T17:10:45.400Z{"msg":"[0]R flush_numbers[0..12]: [42, 42, 36, 12, 43, 43, 25, 24, 33, 35, 44, 44]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.938304391Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28432024-02-03T17:10:45.400Z{"msg":"[0]R generation[0..12]: [23, 23, 20, 6, 24, 24, 16, 15, 18, 20, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.938357292Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28442024-02-03T17:10:45.400Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.938384696Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28452024-02-03T17:10:45.400Z{"msg":"[1]R flush_numbers[0..12]: [42, 42, 36, 12, 43, 43, 25, 24, 33, 35, 44, 44]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.93840863Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28462024-02-03T17:10:45.400Z{"msg":"[1]R generation[0..12]: [23, 23, 20, 6, 24, 24, 16, 15, 18, 20, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.938457133Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28472024-02-03T17:10:45.400Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","Wait for a query_work_queue command to finish before sending IO
28482024-02-03T17:10:45.400Zv":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.93848818Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28492024-02-03T17:10:45.400Z{"msg":"[2]R flush_numbers[0..12]: [42, 42, 36, 12, 43, 43, 25, 24, 33, 35, 44, 44]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.938539147Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28502024-02-03T17:10:45.400Z{"msg":"[2]R generation[0..12]: [23, 23, 20, 6, 24, 24, 16, 15, 18, 20, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.938564086Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28512024-02-03T17:10:45.400Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.938588624Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28522024-02-03T17:10:45.400Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.938637634Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28532024-02-03T17:10:45.400Z{"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.938661682Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28542024-02-03T17:10:45.400Z{"msg":"Next flush: 45","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.938709194Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28552024-02-03T17:10:45.400Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.938733102Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28562024-02-03T17:10:45.400Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.93877983Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28572024-02-03T17:10:45.401Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.938801935Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28582024-02-03T17:10:45.401Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.938848595Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28592024-02-03T17:10:45.401Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.938872304Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28602024-02-03T17:10:45.401Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.938919073Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28612024-02-03T17:10:45.401Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.938941549Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28622024-02-03T17:10:45.401Z{"msg":"849c4fdd-1790-446c-8412-38a1c1bdbe8c is now active with session: 3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.938964761Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28632024-02-03T17:10:45.401Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.938984563Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233,"session_id":"3e30d5f2-d6ad-4a72-99d7-93f77e46d1bb"}
28642024-02-03T17:10:45.401Z{"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:44.9390049Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233}
28652024-02-03T17:10:45.401ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
28662024-02-03T17:10:45.401ZLoading write count information from file "/tmp/test_repair_verify.data"
28672024-02-03T17:10:45.401ZRun Repair workload
28682024-02-03T17:10:45.401Z01/30 Write block 404 len 36864 data: 2 2 2 2 2 2 2 3 2
28692024-02-03T17:10:45.404Z02/30 Read block 576 len 8192
28702024-02-03T17:10:45.404Z03/30 Flush
28712024-02-03T17:10:45.405Z04/30 Write block 244 len 12288 data: 4 4 4
28722024-02-03T17:10:45.407Z05/30 Read block 250 len 24576
28732024-02-03T17:10:45.407Z06/30 Write block 338 len 32768 data: 4 3 3 2 2 2 2 2
28742024-02-03T17:10:45.410Z07/30 Write block 34 len 28672 data: 4 4 4 3 3 3 3
28752024-02-03T17:10:45.413Z08/30 Write block 185 len 32768 data: 3 2 3 4 4 4 4 4
28762024-02-03T17:10:45.413Z09/30 Flush
28772024-02-03T17:10:45.416Z10/30 Flush
28782024-02-03T17:10:45.416Z11/30 Write block 18 len 8192 data: 2 3
28792024-02-03T17:10:45.419Z12/30 Write block 37 len 16384 data: 4 4 4 4
28802024-02-03T17:10:45.419Z13/30 Read block 328 len 8192
28812024-02-03T17:10:45.422Z14/30 Write block 500 len 20480 data: 3 3 3 3 4
28822024-02-03T17:10:45.422Z15/30 Read block 519 len 16384
28832024-02-03T17:10:45.422Z16/30 Read block 291 len 28672
28842024-02-03T17:10:45.425Z17/30 Write block 190 len 24576 data: 5 5 5 3 3 3
28852024-02-03T17:10:45.425Z18/30 Read block 561 len 40960
28862024-02-03T17:10:45.428Z19/30 Read block 21 len 16384
28872024-02-03T17:10:45.428Z20/30 Flush
28882024-02-03T17:10:45.435Z21/30 Write block 356 len 24576 data: 3 3 3 4 4 3
28892024-02-03T17:10:45.438Z22/30 Read block 552 len 20480
28902024-02-03T17:10:45.438Z23/30 Write block 188 len 28672 data: 5 5 6 6 6 4 4
28912024-02-03T17:10:45.438Z24/30 Read block 104 len 32768
28922024-02-03T17:10:45.441Z25/30 Write block 289 len 36864 data: 3 3 3 2 2 2 2 2 2
28932024-02-03T17:10:45.441Z26/30 Write block 582 len 20480 data: 2 2 2 2 3
28942024-02-03T17:10:45.441Z27/30 Read block 383 len 12288
28952024-02-03T17:10:45.444Z28/30 Read block 494 len 12288
28962024-02-03T17:10:45.447Z29/30 Read block 19 len 32768
28972024-02-03T17:10:45.447Z30/30 Write block 73 len 8192 data: 2 3
28982024-02-03T17:10:45.447Z----------------------------------------------------------------
28992024-02-03T17:10:45.447Z Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:30
29002024-02-03T17:10:45.447ZStates: Active Active Active
29012024-02-03T17:10:45.447ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
29022024-02-03T17:10:45.447Z 1 Acked 1000 Write 9 Done Done Done false
29032024-02-03T17:10:45.447Z 2 Acked 1001 Read 2 Done Done Done false
29042024-02-03T17:10:45.447Z 3 Acked 1002 Flush 0 Done Done Sent false
29052024-02-03T17:10:45.447Z 4 Acked 1003 Write 3 Done Done Sent false
29062024-02-03T17:10:45.447Z 5 Acked 1004 Read 6 Done Done Sent false
29072024-02-03T17:10:45.448Z 6 Acked 1005 Write 8 Done Done Sent false
29082024-02-03T17:10:45.448Z 7 Acked 1006 Write 7 Done Done Sent false
29092024-02-03T17:10:45.448Z 8 Acked 1007 Write 8 Done Done Sent false
29102024-02-03T17:10:45.448Z 9 Acked 1008 Flush 0 Done Done Sent false
29112024-02-03T17:10:45.448Z 10 Acked 1009 Flush 0 Done Done Sent false
29122024-02-03T17:10:45.448Z 11 Acked 1010 Write 2 Done Done Sent false
29132024-02-03T17:10:45.448Z 12 Acked 1011 Write 4 Done Done Sent false
29142024-02-03T17:10:45.448Z 13 Acked 1012 Read 2 Done Done Sent false
29152024-02-03T17:10:45.448Z 14 Acked 1013 Write 5 Done Done Sent false
29162024-02-03T17:10:45.448Z 15 Acked 1014 Read 4 Done Done Sent false
29172024-02-03T17:10:45.448Z 16 Acked 1015 Read 7 Done Done Sent false
29182024-02-03T17:10:45.448Z 17 Acked 1016 Write 6 Done Done Sent false
29192024-02-03T17:10:45.448Z 18 Acked 1017 Read 10 Done Done Sent false
29202024-02-03T17:10:45.448Z 19 Acked 1018 Read 4 Done Done Sent false
29212024-02-03T17:10:45.448Z 20 Acked 1019 Flush 0 Done Done Sent false
29222024-02-03T17:10:45.448Z 21 Acked 1020 Write 6 Done Done Sent false
29232024-02-03T17:10:45.448Z 22 Acked 1021 Read 5 Done Done Sent false
29242024-02-03T17:10:45.448Z 23 Acked 1022 Write 7 Done Done Sent false
29252024-02-03T17:10:45.448Z 24 Acked 1023 Read 8 Done Done Sent false
29262024-02-03T17:10:45.452Z 25 Acked 1024 Write 9 Done Done Sent false
29272024-02-03T17:10:45.452Z 26 Acked 1025 Write 5 Done Done Sent false
29282024-02-03T17:10:45.452Z 27 Acked 1026 Read 3 Done Done Sent false
29292024-02-03T17:10:45.452Z 28 Acked 1027 Read 3 Done Done Sent false
29302024-02-03T17:10:45.452Z 29 Acked 1028 Read 8 Done Done Sent false
29312024-02-03T17:10:45.452Z 30 Acked 1029 Write 2 Sent Sent Sent false
29322024-02-03T17:10:45.452Z STATES DS:0 DS:1 DS:2 TOTAL
29332024-02-03T17:10:45.452Z New 0 0 0 0
29342024-02-03T17:10:45.452Z Sent 1 1 28 30
29352024-02-03T17:10:45.452Z Done 29 29 2 60
29362024-02-03T17:10:45.452Z Skipped 0 0 0 0
29372024-02-03T17:10:45.452Z Error 0 0 0 0
29382024-02-03T17:10:45.452ZLast Flush: 1019 1019 0
29392024-02-03T17:10:45.452ZDownstairs last five completed:
29402024-02-03T17:10:45.452ZUpstairs last five completed: 30 29 28 27 26
29412024-02-03T17:10:45.452Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:10:44.990850115Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1233}
29422024-02-03T17:10:45.452ZWrote out file "/tmp/test_repair_verify.data"
29432024-02-03T17:10:45.455Z
29442024-02-03T17:10:45.462ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
29452024-02-03T17:10:45.489ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
29462024-02-03T17:10:45.489Z 0 000-019 26 26 23 48 48 42 F F F <---
29472024-02-03T17:10:45.489Z 1 020-039 26 26 23 48 48 42 F F F <---
29482024-02-03T17:10:45.489Z 2 040-059 26 26 20 48 48 36 F F F <---
29492024-02-03T17:10:45.489Z 3 060-079 6 6 6 12 12 12 T T F <---
29502024-02-03T17:10:45.493Z 4 080-099 24 24 24 43 43 43 F F F
29512024-02-03T17:10:45.493Z 5 100-119 24 24 24 43 43 43 F F F
29522024-02-03T17:10:45.493Z 6 120-139 16 16 16 25 25 25 F F F
29532024-02-03T17:10:45.493Z 7 140-159 15 15 15 24 24 24 F F F
29542024-02-03T17:10:45.493Z 8 160-179 18 18 18 33 33 33 F F F
29552024-02-03T17:10:45.493Z 9 180-199 26 26 20 48 48 35 T T F <---
29562024-02-03T17:10:45.493Z 10 200-219 25 25 25 44 44 44 F F F
29572024-02-03T17:10:45.493Z 11 220-239 25 25 25 44 44 44 F F F
29582024-02-03T17:10:45.493Z 12 240-259 26 26 23 46 46 42 F F F <---
29592024-02-03T17:10:45.493Z 13 260-279 20 20 20 36 36 36 F F F
29602024-02-03T17:10:45.493Z 14 280-299 20 20 20 36 36 36 T T F <---
29612024-02-03T17:10:45.493Z 15 300-319 23 23 23 42 42 42 F F F
29622024-02-03T17:10:45.493Z 16 320-339 26 26 23 46 46 42 F F F <---
29632024-02-03T17:10:45.493Z 17 340-359 26 26 25 46 46 44 T T F <---
29642024-02-03T17:10:45.493Z 18 360-379 21 21 21 39 39 39 T T F <---
29652024-02-03T17:10:45.494Z 19 380-399 22 22 22 41 41 41 F F F
29662024-02-03T17:10:45.494Z 20 400-419 26 26 24 45 45 43 F F T <---
29672024-02-03T17:10:45.494Z 21 420-439 20 20 20 38 38 38 F F F
29682024-02-03T17:10:45.494Z 22 440-459 8 8 8 15 15 15 F F F
29692024-02-03T17:10:45.494Z 23 460-479 20 20 20 37 37 37 F F F
29702024-02-03T17:10:45.494Z 24 480-499 20 20 20 37 37 37 F F F
29712024-02-03T17:10:45.494Z 25 500-519 26 26 24 48 48 43 F F F <---
29722024-02-03T17:10:45.494Z 26 520-539 23 23 23 42 42 42 F F F
29732024-02-03T17:10:45.494Z 27 540-559 23 23 23 42 42 42 F F F
29742024-02-03T17:10:45.494Z 28 560-579 16 16 16 26 26 26 F F F
29752024-02-03T17:10:45.494Z 29 580-599 25 25 25 44 44 44 T T F <---
29762024-02-03T17:10:45.494ZMax gen: 26, Max flush: 48
29772024-02-03T17:10:45.494ZError: Difference in extent metadata found!
29782024-02-03T17:10:45.497ZA Difference in extent metadata is expected here
29792024-02-03T17:10:45.497ZOn loop 13
29802024-02-03T17:10:45.497Z
29812024-02-03T17:10:45.501ZVerifying data now
29822024-02-03T17:10:50.644ZNo change after verify
29832024-02-03T17:10:50.644ZLoop: 13 Downstairs dump after verify (and repair):
29842024-02-03T17:10:50.667ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
29852024-02-03T17:10:50.667Z 0 000-019 26 26 26 48 48 48 F F F
29862024-02-03T17:10:50.667Z 1 020-039 26 26 26 48 48 48 F F F
29872024-02-03T17:10:50.667Z 2 040-059 26 26 26 48 48 48 F F F
29882024-02-03T17:10:50.671Z 3 060-079 27 27 27 49 49 49 F F F
29892024-02-03T17:10:50.671Z 4 080-099 24 24 24 43 43 43 F F F
29902024-02-03T17:10:50.671Z 5 100-119 24 24 24 43 43 43 F F F
29912024-02-03T17:10:50.671Z 6 120-139 16 16 16 25 25 25 F F F
29922024-02-03T17:10:50.672Z 7 140-159 15 15 15 24 24 24 F F F
29932024-02-03T17:10:50.672Z 8 160-179 18 18 18 33 33 33 F F F
29942024-02-03T17:10:50.672Z 9 180-199 27 27 27 49 49 49 F F F
29952024-02-03T17:10:50.672Z 10 200-219 25 25 25 44 44 44 F F F
29962024-02-03T17:10:50.672Z 11 220-239 25 25 25 44 44 44 F F F
29972024-02-03T17:10:50.672Z 12 240-259 26 26 26 46 46 46 F F F
29982024-02-03T17:10:50.672Z 13 260-279 20 20 20 36 36 36 F F F
29992024-02-03T17:10:50.672Z 14 280-299 27 27 27 49 49 49 F F F
30002024-02-03T17:10:50.672Z 15 300-319 23 23 23 42 42 42 F F F
30012024-02-03T17:10:50.672Z 16 320-339 26 26 26 46 46 46 F F F
30022024-02-03T17:10:50.672Z 17 340-359 27 27 27 49 49 49 F F F
30032024-02-03T17:10:50.672Z 18 360-379 27 27 27 49 49 49 F F F
30042024-02-03T17:10:50.673Z 19 380-399 22 22 22 41 41 41 F F F
30052024-02-03T17:10:50.673Z 20 400-419 26 26 26 45 45 45 F F F
30062024-02-03T17:10:50.673Z 21 420-439 20 20 20 38 38 38 F F F
30072024-02-03T17:10:50.673Z 22 440-459 8 8 8 15 15 15 F F F
30082024-02-03T17:10:50.673Z 23 460-479 20 20 20 37 37 37 F F F
30092024-02-03T17:10:50.673Z 24 480-499 20 20 20 37 37 37 F F F
30102024-02-03T17:10:50.673Z 25 500-519 26 26 26 48 48 48 F F F
30112024-02-03T17:10:50.673Z 26 520-539 23 23 23 42 42 42 F F F
30122024-02-03T17:10:50.673Z 27 540-559 23 23 23 42 42 42 F F F
30132024-02-03T17:10:50.673Z 28 560-579 16 16 16 26 26 26 F F F
30142024-02-03T17:10:50.673Z 29 580-599 27 27 27 49 49 49 F F F
30152024-02-03T17:10:50.673ZMax gen: 27, Max flush: 49
30162024-02-03T17:10:50.677Z
30172024-02-03T17:10:50.677ZBegin loop 14
30182024-02-03T17:10:50.678ZDownstairs to restart: 2
30192024-02-03T17:10:50.703Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.24608567Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241}
30202024-02-03T17:10:50.708Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.246569121Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241}
30212024-02-03T17:10:50.708ZCrucible runtime is spawned
30222024-02-03T17:10:50.708Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.246865001Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241}
30232024-02-03T17:10:50.708Z{"msg":"Crucible 77215241-d3f2-43ef-903d-19f081c6d467 has session id: f14b5bdf-7207-4ff3-befa-ca09f0f90a44","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.247072845Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30242024-02-03T17:10:50.708Z{"msg":"Upstairs opts: Upstairs UUID: 77215241-d3f2-43ef-903d-19f081c6d467, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.247160336Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30252024-02-03T17:10:50.708Z{"msg":"Crucible stats registered with UUID: 77215241-d3f2-43ef-903d-19f081c6d467","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.247214076Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30262024-02-03T17:10:50.708Z{"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.247237254Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241}
30272024-02-03T17:10:50.708Z{"msg":"77215241-d3f2-43ef-903d-19f081c6d467 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.247269313Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30282024-02-03T17:10:50.708Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.247302832Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30292024-02-03T17:10:50.708Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.24732744Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30302024-02-03T17:10:50.709Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.247420933Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30312024-02-03T17:10:50.709Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.247464975Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30322024-02-03T17:10:50.709Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.247503076Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30332024-02-03T17:10:50.709Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.24757463Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30342024-02-03T17:10:50.709Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.247615829Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"":"io task","client":"0","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30352024-02-03T17:10:50.709Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.247696275Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"":"io task","client":"1","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30362024-02-03T17:10:50.709Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.247783961Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30372024-02-03T17:10:50.709Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.247827895Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"":"io task","client":"0","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30382024-02-03T17:10:50.709Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.24793459Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"":"io task","client":"1","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30392024-02-03T17:10:50.709Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.247980512Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30402024-02-03T17:10:50.709Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.249297149Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30412024-02-03T17:10:50.709Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.249453349Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30422024-02-03T17:10:50.710Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.249567768Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30432024-02-03T17:10:50.710Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.249615999Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30442024-02-03T17:10:50.716Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.250421937Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30452024-02-03T17:10:50.716Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.250643882Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30462024-02-03T17:10:50.716Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.250885157Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30472024-02-03T17:10:50.716Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.250970304Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30482024-02-03T17:10:50.716Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.251318637Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30492024-02-03T17:10:50.716Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.251414532Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30502024-02-03T17:10:50.716Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.251463119Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30512024-02-03T17:10:50.716Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.251585541Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30522024-02-03T17:10:50.716Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.251646267Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30532024-02-03T17:10:50.716Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.251671872Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30542024-02-03T17:10:50.716Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.251827605Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30552024-02-03T17:10:50.716Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.252562475Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30562024-02-03T17:10:50.717Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.252662328Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30572024-02-03T17:10:50.717Z{"msg":"[0]R flush_numbers[0..12]: [48, 48, 48, 49, 43, 43, 25, 24, 33, 49, 44, 44]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.252690834Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30582024-02-03T17:10:50.717Z{"msg":"[0]R generation[0..12]: [26, 26, 26, 27, 24, 24, 16, 15, 18, 27, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.252715124Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30592024-02-03T17:10:50.717Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.252739472Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30602024-02-03T17:10:50.717Z{"msg":"[1]R flush_numbers[0..12]: [48, 48, 48, 49, 43, 43, 25, 24, 33, 49, 44, 44]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.252762441Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30612024-02-03T17:10:50.717Z{"msg":"[1]R generation[0..12]: [26, 26, 26, 27, 24, 24, 16, 15, 18, 27, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.252785227Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30622024-02-03T17:10:50.717Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.252834558Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30632024-02-03T17:10:50.717Z{"msg":"[2]R flush_numbers[0..12]: [48, 48, 48, 49, 43, 43, 25, 24, 33, 49, 44, 44]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.252860382Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30642024-02-03T17:10:50.717ZWait for a query_work_queue command to finish before sending IO
30652024-02-03T17:10:50.717Z{"msg":"[2]R generation[0..12]: [26, 26, 26, 27, 24, 24, 16, 15, 18, 27, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.252912562Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30662024-02-03T17:10:50.717Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.25293896Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30672024-02-03T17:10:50.717Z{"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.252988611Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30682024-02-03T17:10:50.717Z{"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.253013291Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30692024-02-03T17:10:50.718Z{"msg":"Next flush: 50","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.253059371Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30702024-02-03T17:10:50.718Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.253083119Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30712024-02-03T17:10:50.718Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.253106761Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30722024-02-03T17:10:50.718Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.253126923Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30732024-02-03T17:10:50.718Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.253149061Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30742024-02-03T17:10:50.718Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.253197614Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30752024-02-03T17:10:50.718Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.25322199Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30762024-02-03T17:10:50.718Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.253242854Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30772024-02-03T17:10:50.718Z{"msg":"77215241-d3f2-43ef-903d-19f081c6d467 is now active with session: f14b5bdf-7207-4ff3-befa-ca09f0f90a44","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.253289738Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30782024-02-03T17:10:50.718Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.253312062Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241,"session_id":"f14b5bdf-7207-4ff3-befa-ca09f0f90a44"}
30792024-02-03T17:10:50.718Z{"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:50.25333255Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241}
30802024-02-03T17:10:50.718ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
30812024-02-03T17:10:50.718ZLoading write count information from file "/tmp/test_repair_verify.data"
30822024-02-03T17:10:50.719ZRun Repair workload
30832024-02-03T17:10:50.719Z01/30 Write block 571 len 4096 data: 3
30842024-02-03T17:10:50.719Z02/30 Read block 168 len 36864
30852024-02-03T17:10:50.722Z03/30 Read block 132 len 40960
30862024-02-03T17:10:50.723Z04/30 Read block 205 len 28672
30872024-02-03T17:10:50.723Z05/30 Write block 354 len 24576 data: 4 4 4 4 4 5
30882024-02-03T17:10:50.723Z06/30 Write block 70 len 12288 data: 3 3 2
30892024-02-03T17:10:50.723Z07/30 Write block 370 len 40960 data: 7 4 5 5 4 3 3 4 3 4
30902024-02-03T17:10:50.726Z08/30 Write block 158 len 40960 data: 2 2 2 2 3 3 3 3 3 3
30912024-02-03T17:10:50.726Z09/30 Read block 212 len 8192
30922024-02-03T17:10:50.730Z10/30 Write block 141 len 20480 data: 2 2 2 2 2
30932024-02-03T17:10:50.730Z11/30 Flush
30942024-02-03T17:10:50.734Z12/30 Flush
30952024-02-03T17:10:50.734Z13/30 Write block 340 len 16384 data: 4 3 3 3
30962024-02-03T17:10:50.734Z14/30 Write block 236 len 8192 data: 2 2
30972024-02-03T17:10:50.738Z15/30 Write block 326 len 16384 data: 3 3 3 4
30982024-02-03T17:10:50.738Z16/30 Read block 531 len 20480
30992024-02-03T17:10:50.738Z17/30 Write block 319 len 8192 data: 4 4
31002024-02-03T17:10:50.738Z18/30 Write block 360 len 20480 data: 5 4 2 2 4
31012024-02-03T17:10:50.742Z19/30 Flush
31022024-02-03T17:10:50.745Z20/30 Write block 177 len 4096 data: 4
31032024-02-03T17:10:50.745Z21/30 Read block 76 len 24576
31042024-02-03T17:10:50.745Z22/30 Read block 231 len 8192
31052024-02-03T17:10:50.748Z23/30 Read block 471 len 16384
31062024-02-03T17:10:50.749Z24/30 Read block 424 len 12288
31072024-02-03T17:10:50.752Z25/30 Write block 423 len 40960 data: 3 3 3 3 3 3 4 4 3 2
31082024-02-03T17:10:50.752Z26/30 Read block 185 len 16384
31092024-02-03T17:10:50.752Z27/30 Read block 233 len 24576
31102024-02-03T17:10:50.756Z28/30 Write block 102 len 24576 data: 2 2 3 3 4 4
31112024-02-03T17:10:50.756Z29/30 Write block 215 len 12288 data: 3 3 3
31122024-02-03T17:10:50.757Z30/30 Read block 512 len 36864
31132024-02-03T17:10:50.772Z----------------------------------------------------------------
31142024-02-03T17:10:50.772Z Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:30
31152024-02-03T17:10:50.772ZStates: Active Active Active
31162024-02-03T17:10:50.772ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
31172024-02-03T17:10:50.772Z 1 Acked 1000 Write 1 Done Done Sent false
31182024-02-03T17:10:50.772Z 2 Acked 1001 Read 9 Done Done Sent false
31192024-02-03T17:10:50.772Z 3 Acked 1002 Read 10 Done Done Sent false
31202024-02-03T17:10:50.772Z 4 Acked 1003 Read 7 Done Done Sent false
31212024-02-03T17:10:50.773Z 5 Acked 1004 Write 6 Done Done Sent false
31222024-02-03T17:10:50.773Z 6 Acked 1005 Write 3 Done Done Sent false
31232024-02-03T17:10:50.773Z 7 Acked 1006 Write 10 Done Done Sent false
31242024-02-03T17:10:50.773Z 8 Acked 1007 Write 10 Done Done Sent false
31252024-02-03T17:10:50.773Z 9 Acked 1008 Read 2 Done Done Sent false
31262024-02-03T17:10:50.773Z 10 Acked 1009 Write 5 Done Done Sent false
31272024-02-03T17:10:50.773Z 11 Acked 1010 Flush 0 Done Done Sent false
31282024-02-03T17:10:50.773Z 12 Acked 1011 Flush 0 Done Done Sent false
31292024-02-03T17:10:50.773Z 13 Acked 1012 Write 4 Done Done Sent false
31302024-02-03T17:10:50.773Z 14 Acked 1013 Write 2 Done Done Sent false
31312024-02-03T17:10:50.773Z 15 Acked 1014 Write 4 Done Done Sent false
31322024-02-03T17:10:50.773Z 16 Acked 1015 Read 5 Done Done Sent false
31332024-02-03T17:10:50.773Z 17 Acked 1016 Write 2 Done Done Sent false
31342024-02-03T17:10:50.773Z 18 Acked 1017 Write 5 Done Done Sent false
31352024-02-03T17:10:50.773Z 19 Acked 1018 Flush 0 Done Done Sent false
31362024-02-03T17:10:50.773Z 20 Acked 1019 Write 1 Done Done Sent false
31372024-02-03T17:10:50.773Z 21 Acked 1020 Read 6 Done Done Sent false
31382024-02-03T17:10:50.773Z 22 Acked 1021 Read 2 Done Done Sent false
31392024-02-03T17:10:50.774Z 23 Acked 1022 Read 4 Done Done Sent false
31402024-02-03T17:10:50.774Z 24 Acked 1023 Read 3 Done Done Sent false
31412024-02-03T17:10:50.774Z 25 Acked 1024 Write 10 Done Done Sent false
31422024-02-03T17:10:50.774Z 26 Acked 1025 Read 4 Done Done Sent false
31432024-02-03T17:10:50.774Z 27 Acked 1026 Read 6 Done Done Sent false
31442024-02-03T17:10:50.774Z 28 Acked 1027 Write 6 Done Done Sent false
31452024-02-03T17:10:50.774Z 29 Acked 1028 Write 3 Done Done Sent false
31462024-02-03T17:10:50.774Z 30 Acked 1029 Read 9 Sent Done Sent false
31472024-02-03T17:10:50.774Z STATES DS:0 DS:1 DS:2 TOTAL
31482024-02-03T17:10:50.774Z New 0 0 0 0
31492024-02-03T17:10:50.774Z Sent 1 0 30 31
31502024-02-03T17:10:50.774Z Done 29 30 0 59
31512024-02-03T17:10:50.774Z Skipped 0 0 0 0
31522024-02-03T17:10:50.774Z Error 0 0 0 0
31532024-02-03T17:10:50.774ZLast Flush: 1018 1018 0
31542024-02-03T17:10:50.774ZDownstairs last five completed:
31552024-02-03T17:10:50.774ZUpstairs last five completed: 30 29 28 27 26
31562024-02-03T17:10:50.774Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:10:50.301572463Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1241}
31572024-02-03T17:10:50.774ZWrote out file "/tmp/test_repair_verify.data"
31582024-02-03T17:10:50.779Z
31592024-02-03T17:10:50.779ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
31602024-02-03T17:10:50.796ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
31612024-02-03T17:10:50.796Z 0 000-019 26 26 26 48 48 48 F F F
31622024-02-03T17:10:50.797Z 1 020-039 26 26 26 48 48 48 F F F
31632024-02-03T17:10:50.797Z 2 040-059 26 26 26 48 48 48 F F F
31642024-02-03T17:10:50.800Z 3 060-079 28 28 27 50 50 49 F F F <---
31652024-02-03T17:10:50.800Z 4 080-099 24 24 24 43 43 43 F F F
31662024-02-03T17:10:50.800Z 5 100-119 24 24 24 43 43 43 T T F <---
31672024-02-03T17:10:50.801Z 6 120-139 16 16 16 25 25 25 F F F
31682024-02-03T17:10:50.801Z 7 140-159 28 28 15 50 50 24 F F F <---
31692024-02-03T17:10:50.801Z 8 160-179 28 28 18 50 50 33 T T F <---
31702024-02-03T17:10:50.801Z 9 180-199 27 27 27 49 49 49 F F F
31712024-02-03T17:10:50.801Z 10 200-219 25 25 25 44 44 44 T T F <---
31722024-02-03T17:10:50.801Z 11 220-239 28 28 25 52 52 44 F F F <---
31732024-02-03T17:10:50.801Z 12 240-259 26 26 26 46 46 46 F F F
31742024-02-03T17:10:50.801Z 13 260-279 20 20 20 36 36 36 F F F
31752024-02-03T17:10:50.801Z 14 280-299 27 27 27 49 49 49 F F F
31762024-02-03T17:10:50.801Z 15 300-319 28 28 23 52 52 42 F F F <---
31772024-02-03T17:10:50.801Z 16 320-339 28 28 26 52 52 46 F F F <---
31782024-02-03T17:10:50.801Z 17 340-359 28 28 27 52 52 49 F F F <---
31792024-02-03T17:10:50.801Z 18 360-379 28 28 27 52 52 49 F F F <---
31802024-02-03T17:10:50.802Z 19 380-399 22 22 22 41 41 41 F F F
31812024-02-03T17:10:50.802Z 20 400-419 26 26 26 45 45 45 F F F
31822024-02-03T17:10:50.802Z 21 420-439 20 20 20 38 38 38 T T F <---
31832024-02-03T17:10:50.802Z 22 440-459 8 8 8 15 15 15 F F F
31842024-02-03T17:10:50.802Z 23 460-479 20 20 20 37 37 37 F F F
31852024-02-03T17:10:50.802Z 24 480-499 20 20 20 37 37 37 F F F
31862024-02-03T17:10:50.802Z 25 500-519 26 26 26 48 48 48 F F F
31872024-02-03T17:10:50.802Z 26 520-539 23 23 23 42 42 42 F F F
31882024-02-03T17:10:50.802Z 27 540-559 23 23 23 42 42 42 F F F
31892024-02-03T17:10:50.802Z 28 560-579 28 28 16 50 50 26 F F F <---
31902024-02-03T17:10:50.802Z 29 580-599 27 27 27 49 49 49 F F F
31912024-02-03T17:10:50.802ZMax gen: 28, Max flush: 52
31922024-02-03T17:10:50.803ZError: Difference in extent metadata found!
31932024-02-03T17:10:50.811ZA Difference in extent metadata is expected here
31942024-02-03T17:10:50.811ZOn loop 14
31952024-02-03T17:10:50.811Z
31962024-02-03T17:10:50.814ZVerifying data now
31972024-02-03T17:10:55.864ZNo change after verify
31982024-02-03T17:10:55.864ZLoop: 14 Downstairs dump after verify (and repair):
31992024-02-03T17:10:55.887ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
32002024-02-03T17:10:55.887Z 0 000-019 26 26 26 48 48 48 F F F
32012024-02-03T17:10:55.887Z 1 020-039 26 26 26 48 48 48 F F F
32022024-02-03T17:10:55.887Z 2 040-059 26 26 26 48 48 48 F F F
32032024-02-03T17:10:55.890Z 3 060-079 28 28 28 50 50 50 F F F
32042024-02-03T17:10:55.890Z 4 080-099 24 24 24 43 43 43 F F F
32052024-02-03T17:10:55.891Z 5 100-119 29 29 29 53 53 53 F F F
32062024-02-03T17:10:55.891Z 6 120-139 16 16 16 25 25 25 F F F
32072024-02-03T17:10:55.891Z 7 140-159 28 28 28 50 50 50 F F F
32082024-02-03T17:10:55.891Z 8 160-179 29 29 29 53 53 53 F F F
32092024-02-03T17:10:55.891Z 9 180-199 27 27 27 49 49 49 F F F
32102024-02-03T17:10:55.891Z 10 200-219 29 29 29 53 53 53 F F F
32112024-02-03T17:10:55.891Z 11 220-239 28 28 28 52 52 52 F F F
32122024-02-03T17:10:55.891Z 12 240-259 26 26 26 46 46 46 F F F
32132024-02-03T17:10:55.891Z 13 260-279 20 20 20 36 36 36 F F F
32142024-02-03T17:10:55.891Z 14 280-299 27 27 27 49 49 49 F F F
32152024-02-03T17:10:55.891Z 15 300-319 28 28 28 52 52 52 F F F
32162024-02-03T17:10:55.891Z 16 320-339 28 28 28 52 52 52 F F F
32172024-02-03T17:10:55.891Z 17 340-359 28 28 28 52 52 52 F F F
32182024-02-03T17:10:55.891Z 18 360-379 28 28 28 52 52 52 F F F
32192024-02-03T17:10:55.891Z 19 380-399 22 22 22 41 41 41 F F F
32202024-02-03T17:10:55.891Z 20 400-419 26 26 26 45 45 45 F F F
32212024-02-03T17:10:55.891Z 21 420-439 29 29 29 53 53 53 F F F
32222024-02-03T17:10:55.891Z 22 440-459 8 8 8 15 15 15 F F F
32232024-02-03T17:10:55.891Z 23 460-479 20 20 20 37 37 37 F F F
32242024-02-03T17:10:55.891Z 24 480-499 20 20 20 37 37 37 F F F
32252024-02-03T17:10:55.891Z 25 500-519 26 26 26 48 48 48 F F F
32262024-02-03T17:10:55.891Z 26 520-539 23 23 23 42 42 42 F F F
32272024-02-03T17:10:55.891Z 27 540-559 23 23 23 42 42 42 F F F
32282024-02-03T17:10:55.891Z 28 560-579 28 28 28 50 50 50 F F F
32292024-02-03T17:10:55.892Z 29 580-599 27 27 27 49 49 49 F F F
32302024-02-03T17:10:55.892ZMax gen: 29, Max flush: 53
32312024-02-03T17:10:55.894Z
32322024-02-03T17:10:55.895ZBegin loop 15
32332024-02-03T17:10:55.895ZDownstairs to restart: 2
32342024-02-03T17:10:55.923Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.466573432Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249}
32352024-02-03T17:10:55.923Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.467073108Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249}
32362024-02-03T17:10:55.926Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.4671551Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249}
32372024-02-03T17:10:55.926Z{"msg":"Crucible ca241d50-d237-4c31-b1f7-ada7672bb4f2 has session id: f0798cfe-45b8-432e-bb8c-1ec856394c4e","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.467263786Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32382024-02-03T17:10:55.926Z{"msg":"Upstairs opts: Upstairs UUID: ca241d50-d237-4c31-b1f7-ada7672bb4f2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
32392024-02-03T17:10:55.926Z,"time":"2024-02-03T17:10:55.467340173Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32402024-02-03T17:10:55.926Z{"msg":"Crucible stats registered with UUID: ca241d50-d237-4c31-b1f7-ada7672bb4f2","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.467374683Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32412024-02-03T17:10:55.926Z{"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.467516065Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249}
32422024-02-03T17:10:55.926Z{"msg":"ca241d50-d237-4c31-b1f7-ada7672bb4f2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.467714382Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32432024-02-03T17:10:55.926Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.467773236Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32442024-02-03T17:10:55.926Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.467799614Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32452024-02-03T17:10:55.927Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.467821831Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32462024-02-03T17:10:55.927Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.468009065Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32472024-02-03T17:10:55.927Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.468143534Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32482024-02-03T17:10:55.927Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.468281003Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32492024-02-03T17:10:55.927Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.468407572Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"":"io task","client":"0","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32502024-02-03T17:10:55.927Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.468456515Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"":"io task","client":"2","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32512024-02-03T17:10:55.927Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.468536642Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"":"io task","client":"1","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32522024-02-03T17:10:55.927Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.468569485Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"":"io task","client":"0","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32532024-02-03T17:10:55.927Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.468625854Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"":"io task","client":"2","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32542024-02-03T17:10:55.927Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.468666827Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"":"io task","client":"1","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32552024-02-03T17:10:55.927Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.469526724Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32562024-02-03T17:10:55.927Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.46974642Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32572024-02-03T17:10:55.927Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.469828024Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32582024-02-03T17:10:55.927Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.469872829Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32592024-02-03T17:10:55.931Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.47067362Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32602024-02-03T17:10:55.931Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.470859461Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32612024-02-03T17:10:55.931Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.471183675Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32622024-02-03T17:10:55.932Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.4712738Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32632024-02-03T17:10:55.932Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.471831065Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32642024-02-03T17:10:55.932Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.471955779Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32652024-02-03T17:10:55.932Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.472003851Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32662024-02-03T17:10:55.932Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.472136625Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32672024-02-03T17:10:55.932Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.47220907Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32682024-02-03T17:10:55.932Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.472236247Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32692024-02-03T17:10:55.932Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.472617158Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32702024-02-03T17:10:55.932Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.473375306Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32712024-02-03T17:10:55.932Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.473462874Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32722024-02-03T17:10:55.932Z{"msg":"[0]R flush_numbers[0..12]: [48, 48, 48, 50, 43, 53, 25, 50, 53, 49, 53, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.473490839Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32732024-02-03T17:10:55.932Z{"msg":"[0]R generation[0..12]: [26, 26, 26, 28, 24, 29, 16, 28, 29, 27, 29, 28]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.473514184Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32742024-02-03T17:10:55.932Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.473563573Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32752024-02-03T17:10:55.932Z{"msg":"[1]R flush_numbers[0..12]: [48, 48, 48, 50, 43, 53, 25, 50, 53, 49, 53, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.473590159Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32762024-02-03T17:10:55.932Z{"msg":"[1]R generation[0..12]: [26, 26, 26, 28, 24, 29, 16, 28, 29, 27, 29, 28]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.473637699Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32772024-02-03T17:10:55.932Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.473663563Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32782024-02-03T17:10:55.932Z{"msg":"[2]R flush_numbers[0..12]: [48, 48, 48, 50, 43, 53, 25, 50, 53, 49, 53, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.47368664Z","hostname"Wait for a query_work_queue command to finish before sending IO
32792024-02-03T17:10:55.932Z:"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32802024-02-03T17:10:55.932Z{"msg":"[2]R generation[0..12]: [26, 26, 26, 28, 24, 29, 16, 28, 29, 27, 29, 28]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.473817178Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32812024-02-03T17:10:55.932Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.473848982Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32822024-02-03T17:10:55.932Z{"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.473896871Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32832024-02-03T17:10:55.932Z{"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.473921741Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32842024-02-03T17:10:55.932Z{"msg":"Next flush: 54","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.473944893Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32852024-02-03T17:10:55.932Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.473967019Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32862024-02-03T17:10:55.933Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.473989152Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32872024-02-03T17:10:55.933Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.474008738Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32882024-02-03T17:10:55.933Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.474030612Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32892024-02-03T17:10:55.933Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.474077394Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32902024-02-03T17:10:55.933Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.474102385Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32912024-02-03T17:10:55.933Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.474123061Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32922024-02-03T17:10:55.933Z{"msg":"ca241d50-d237-4c31-b1f7-ada7672bb4f2 is now active with session: f0798cfe-45b8-432e-bb8c-1ec856394c4e","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.474169152Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32932024-02-03T17:10:55.933Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.474191419Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249,"session_id":"f0798cfe-45b8-432e-bb8c-1ec856394c4e"}
32942024-02-03T17:10:55.933Z{"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:10:55.474212261Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249}
32952024-02-03T17:10:55.933ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
32962024-02-03T17:10:55.933ZLoading write count information from file "/tmp/test_repair_verify.data"
32972024-02-03T17:10:55.933ZRun Repair workload
32982024-02-03T17:10:55.936Z01/30 Write block 306 len 40960 data: 5 3 3 3 3 2 3 4 4 4
32992024-02-03T17:10:55.936Z02/30 Read block 397 len 4096
33002024-02-03T17:10:55.936Z03/30 Read block 305 len 4096
33012024-02-03T17:10:55.939Z04/30 Write block 8 len 20480 data: 5 5 5 4 3
33022024-02-03T17:10:55.939Z05/30 Read block 454 len 8192
33032024-02-03T17:10:55.939Z06/30 Write block 477 len 20480 data: 2 2 3 3 3
33042024-02-03T17:10:55.942Z07/30 Read block 549 len 4096
33052024-02-03T17:10:55.942Z08/30 Read block 370 len 8192
33062024-02-03T17:10:55.942Z09/30 Read block 307 len 16384
33072024-02-03T17:10:55.946Z10/30 Read block 411 len 12288
33082024-02-03T17:10:55.946Z11/30 Write block 287 len 32768 data: 4 3 4 4 4 3 3 3
33092024-02-03T17:10:55.952Z12/30 Write block 520 len 4096 data: 3
33102024-02-03T17:10:55.952Z13/30 Write block 545 len 36864 data: 2 3 4 3 3 3 2 2 4
33112024-02-03T17:10:55.955Z14/30 Read block 215 len 40960
33122024-02-03T17:10:55.955Z15/30 Write block 438 len 20480 data: 4 4 4 4 4
33132024-02-03T17:10:55.955Z16/30 Write block 36 len 28672 data: 5 5 5 5 5 3 3
33142024-02-03T17:10:55.955Z17/30 Write block 184 len 36864 data: 3 4 3 4 6 6 7 7 7
33152024-02-03T17:10:55.958Z18/30 Flush
33162024-02-03T17:10:55.962Z19/30 Write block 148 len 4096 data: 2
33172024-02-03T17:10:55.965Z20/30 Read block 503 len 32768
33182024-02-03T17:10:55.965Z21/30 Read block 541 len 16384
33192024-02-03T17:10:55.968Z22/30 Read block 224 len 40960
33202024-02-03T17:10:55.971Z23/30 Write block 119 len 4096 data: 4
33212024-02-03T17:10:55.971Z24/30 Read block 96 len 16384
33222024-02-03T17:10:55.971Z25/30 Read block 149 len 16384
33232024-02-03T17:10:55.973Z26/30 Write block 84 len 4096 data: 4
33242024-02-03T17:10:55.974Z27/30 Read block 295 len 24576
33252024-02-03T17:10:55.976Z28/30 Read block 426 len 40960
33262024-02-03T17:10:55.977Z29/30 Read block 572 len 12288
33272024-02-03T17:10:55.980Z30/30 Write block 446 len 40960 data: 2 2 2 2 2 2 2 2 2 2
33282024-02-03T17:10:55.980Z----------------------------------------------------------------
33292024-02-03T17:10:55.980Z Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:30
33302024-02-03T17:10:55.980ZStates: Active Active Active
33312024-02-03T17:10:55.980ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
33322024-02-03T17:10:55.980Z 1 Acked 1000 Write 10 Done Done Done false
33332024-02-03T17:10:55.980Z 2 Acked 1001 Read 1 Done Done Done false
33342024-02-03T17:10:55.980Z 3 Acked 1002 Read 1 Done Done Done false
33352024-02-03T17:10:55.980Z 4 Acked 1003 Write 5 Done Done Done false
33362024-02-03T17:10:55.980Z 5 Acked 1004 Read 2 Done Done Done false
33372024-02-03T17:10:55.981Z 6 Acked 1005 Write 5 Done Done Done false
33382024-02-03T17:10:55.981Z 7 Acked 1006 Read 1 Done Done Done false
33392024-02-03T17:10:55.981Z 8 Acked 1007 Read 2 Done Done Sent false
33402024-02-03T17:10:55.981Z 9 Acked 1008 Read 4 Done Done Sent false
33412024-02-03T17:10:55.981Z 10 Acked 1009 Read 3 Done Done Sent false
33422024-02-03T17:10:55.981Z 11 Acked 1010 Write 8 Done Done Sent false
33432024-02-03T17:10:55.981Z 12 Acked 1011 Write 1 Done Done Sent false
33442024-02-03T17:10:55.981Z 13 Acked 1012 Write 9 Done Done Sent false
33452024-02-03T17:10:55.981Z 14 Acked 1013 Read 10 Done Done Sent false
33462024-02-03T17:10:55.981Z 15 Acked 1014 Write 5 Done Done Sent false
33472024-02-03T17:10:55.981Z 16 Acked 1015 Write 7 Done Done Sent false
33482024-02-03T17:10:55.981Z 17 Acked 1016 Write 9 Done Done Sent false
33492024-02-03T17:10:55.981Z 18 Acked 1017 Flush 0 Done Done Sent false
33502024-02-03T17:10:55.981Z 19 Acked 1018 Write 1 Done Done Sent false
33512024-02-03T17:10:55.981Z 20 Acked 1019 Read 8 Done Done Sent false
33522024-02-03T17:10:55.981Z 21 Acked 1020 Read 4 Done Done Sent false
33532024-02-03T17:10:55.981Z 22 Acked 1021 Read 10 Done Done Sent false
33542024-02-03T17:10:55.981Z 23 Acked 1022 Write 1 Done Done Sent false
33552024-02-03T17:10:55.981Z 24 Acked 1023 Read 4 Done Done Sent false
33562024-02-03T17:10:55.981Z 25 Acked 1024 Read 4 Done Done Sent false
33572024-02-03T17:10:55.981Z 26 Acked 1025 Write 1 Done Done Sent false
33582024-02-03T17:10:55.981Z 27 Acked 1026 Read 6 Done Done Sent false
33592024-02-03T17:10:55.981Z 28 Acked 1027 Read 10 Done Done Sent false
33602024-02-03T17:10:55.981Z 29 Acked 1028 Read 3 Done Done Sent false
33612024-02-03T17:10:55.981Z 30 Acked 1029 Write 10 Sent Sent Sent false
33622024-02-03T17:10:55.982Z STATES DS:0 DS:1 DS:2 TOTAL
33632024-02-03T17:10:55.982Z New 0 0 0 0
33642024-02-03T17:10:55.982Z Sent 1 1 23 25
33652024-02-03T17:10:55.982Z Done 29 29 7 65
33662024-02-03T17:10:55.982Z Skipped 0 0 0 0
33672024-02-03T17:10:55.982Z Error 0 0 0 0
33682024-02-03T17:10:55.982ZLast Flush: 1017 1017 0
33692024-02-03T17:10:55.982ZDownstairs last five completed:
33702024-02-03T17:10:55.982ZUpstairs last five completed: 30 29 28 27 26
33712024-02-03T17:10:55.982Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:10:55.524269806Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1249}
33722024-02-03T17:10:55.985ZWrote out file "/tmp/test_repair_verify.data"
33732024-02-03T17:10:55.988Z
33742024-02-03T17:10:55.994ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
33752024-02-03T17:10:56.020ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
33762024-02-03T17:10:56.020Z 0 000-019 30 30 26 54 54 48 F F T <---
33772024-02-03T17:10:56.026Z 1 020-039 30 30 26 54 54 48 F F F <---
33782024-02-03T17:10:56.027Z 2 040-059 30 30 26 54 54 48 F F F <---
33792024-02-03T17:10:56.027Z 3 060-079 28 28 28 50 50 50 F F F
33802024-02-03T17:10:56.027Z 4 080-099 24 24 24 43 43 43 T T F <---
33812024-02-03T17:10:56.027Z 5 100-119 29 29 29 53 53 53 T T F <---
33822024-02-03T17:10:56.027Z 6 120-139 16 16 16 25 25 25 F F F
33832024-02-03T17:10:56.027Z 7 140-159 28 28 28 50 50 50 T T F <---
33842024-02-03T17:10:56.027Z 8 160-179 29 29 29 53 53 53 F F F
33852024-02-03T17:10:56.027Z 9 180-199 30 30 27 54 54 49 F F F <---
33862024-02-03T17:10:56.027Z 10 200-219 29 29 29 53 53 53 F F F
33872024-02-03T17:10:56.027Z 11 220-239 28 28 28 52 52 52 F F F
33882024-02-03T17:10:56.027Z 12 240-259 26 26 26 46 46 46 F F F
33892024-02-03T17:10:56.027Z 13 260-279 20 20 20 36 36 36 F F F
33902024-02-03T17:10:56.027Z 14 280-299 30 30 27 54 54 49 F F F <---
33912024-02-03T17:10:56.027Z 15 300-319 30 30 28 54 54 52 F F T <---
33922024-02-03T17:10:56.027Z 16 320-339 28 28 28 52 52 52 F F F
33932024-02-03T17:10:56.027Z 17 340-359 28 28 28 52 52 52 F F F
33942024-02-03T17:10:56.027Z 18 360-379 28 28 28 52 52 52 F F F
33952024-02-03T17:10:56.027Z 19 380-399 22 22 22 41 41 41 F F F
33962024-02-03T17:10:56.027Z 20 400-419 26 26 26 45 45 45 F F F
33972024-02-03T17:10:56.027Z 21 420-439 30 30 29 54 54 53 F F F <---
33982024-02-03T17:10:56.027Z 22 440-459 30 30 8 54 54 15 T T F <---
33992024-02-03T17:10:56.027Z 23 460-479 30 30 20 54 54 37 F F T <---
34002024-02-03T17:10:56.027Z 24 480-499 30 30 20 54 54 37 F F T <---
34012024-02-03T17:10:56.028Z 25 500-519 26 26 26 48 48 48 F F F
34022024-02-03T17:10:56.028Z 26 520-539 30 30 23 54 54 42 F F F <---
34032024-02-03T17:10:56.028Z 27 540-559 30 30 23 54 54 42 F F F <---
34042024-02-03T17:10:56.028Z 28 560-579 28 28 28 50 50 50 F F F
34052024-02-03T17:10:56.028Z 29 580-599 27 27 27 49 49 49 F F F
34062024-02-03T17:10:56.028ZMax gen: 30, Max flush: 54
34072024-02-03T17:10:56.028ZError: Difference in extent metadata found!
34082024-02-03T17:10:56.031ZA Difference in extent metadata is expected here
34092024-02-03T17:10:56.031ZOn loop 15
34102024-02-03T17:10:56.031Z
34112024-02-03T17:10:56.031ZVerifying data now
34122024-02-03T17:11:02.307ZNo change after verify
34132024-02-03T17:11:02.307ZLoop: 15 Downstairs dump after verify (and repair):
34142024-02-03T17:11:02.331ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
34152024-02-03T17:11:02.331Z 0 000-019 30 30 30 54 54 54 F F F
34162024-02-03T17:11:02.332Z 1 020-039 30 30 30 54 54 54 F F F
34172024-02-03T17:11:02.332Z 2 040-059 30 30 30 54 54 54 F F F
34182024-02-03T17:11:02.332Z 3 060-079 28 28 28 50 50 50 F F F
34192024-02-03T17:11:02.332Z 4 080-099 31 31 31 55 55 55 F F F
34202024-02-03T17:11:02.332Z 5 100-119 31 31 31 55 55 55 F F F
34212024-02-03T17:11:02.332Z 6 120-139 16 16 16 25 25 25 F F F
34222024-02-03T17:11:02.332Z 7 140-159 31 31 31 55 55 55 F F F
34232024-02-03T17:11:02.338Z 8 160-179 29 29 29 53 53 53 F F F
34242024-02-03T17:11:02.338Z 9 180-199 30 30 30 54 54 54 F F F
34252024-02-03T17:11:02.339Z 10 200-219 29 29 29 53 53 53 F F F
34262024-02-03T17:11:02.339Z 11 220-239 28 28 28 52 52 52 F F F
34272024-02-03T17:11:02.339Z 12 240-259 26 26 26 46 46 46 F F F
34282024-02-03T17:11:02.339Z 13 260-279 20 20 20 36 36 36 F F F
34292024-02-03T17:11:02.339Z 14 280-299 30 30 30 54 54 54 F F F
34302024-02-03T17:11:02.339Z 15 300-319 30 30 30 54 54 54 F F F
34312024-02-03T17:11:02.339Z 16 320-339 28 28 28 52 52 52 F F F
34322024-02-03T17:11:02.339Z 17 340-359 28 28 28 52 52 52 F F F
34332024-02-03T17:11:02.339Z 18 360-379 28 28 28 52 52 52 F F F
34342024-02-03T17:11:02.339Z 19 380-399 22 22 22 41 41 41 F F F
34352024-02-03T17:11:02.339Z 20 400-419 26 26 26 45 45 45 F F F
34362024-02-03T17:11:02.339Z 21 420-439 30 30 30 54 54 54 F F F
34372024-02-03T17:11:02.339Z 22 440-459 31 31 31 55 55 55 F F F
34382024-02-03T17:11:02.339Z 23 460-479 30 30 30 54 54 54 F F F
34392024-02-03T17:11:02.340Z 24 480-499 30 30 30 54 54 54 F F F
34402024-02-03T17:11:02.340Z 25 500-519 26 26 26 48 48 48 F F F
34412024-02-03T17:11:02.340Z 26 520-539 30 30 30 54 54 54 F F F
34422024-02-03T17:11:02.340Z 27 540-559 30 30 30 54 54 54 F F F
34432024-02-03T17:11:02.340Z 28 560-579 28 28 28 50 50 50 F F F
34442024-02-03T17:11:02.340Z 29 580-599 27 27 27 49 49 49 F F F
34452024-02-03T17:11:02.340ZMax gen: 31, Max flush: 55
34462024-02-03T17:11:02.344Z
34472024-02-03T17:11:02.345ZBegin loop 16
34482024-02-03T17:11:02.345ZDownstairs to restart: 0
34492024-02-03T17:11:02.369Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.913513777Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257}
34502024-02-03T17:11:02.370Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.914052324Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257}
34512024-02-03T17:11:02.370Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.914084849Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257}
34522024-02-03T17:11:02.374Z{"msg":"Crucible 991e02cc-ee4b-460b-b436-202fd8f26e09 has session id: a7f6bcf7-4a9b-451e-b632-114c5ce4736a","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.914117044Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34532024-02-03T17:11:02.374Z{"msg":"Crucible runtime is spawned
34542024-02-03T17:11:02.374ZUpstairs opts: Upstairs UUID: 991e02cc-ee4b-460b-b436-202fd8f26e09, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.914429552Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34552024-02-03T17:11:02.374Z{"msg":"Crucible stats registered with UUID: 991e02cc-ee4b-460b-b436-202fd8f26e09","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.914487412Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34562024-02-03T17:11:02.374Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.914526097Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257}
34572024-02-03T17:11:02.374Z{"msg":"991e02cc-ee4b-460b-b436-202fd8f26e09 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.914808744Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34582024-02-03T17:11:02.374Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.914869135Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34592024-02-03T17:11:02.374Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.914923129Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34602024-02-03T17:11:02.374Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.914947868Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34612024-02-03T17:11:02.374Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.914971197Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34622024-02-03T17:11:02.374Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.914993482Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34632024-02-03T17:11:02.375Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.915022596Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34642024-02-03T17:11:02.375Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.91505969Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"":"io task","client":"2","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34652024-02-03T17:11:02.375Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.915099256Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"":"io task","client":"0","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34662024-02-03T17:11:02.375Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.915215828Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"":"io task","client":"1","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34672024-02-03T17:11:02.375Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.915303183Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"":"io task","client":"0","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34682024-02-03T17:11:02.375Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.915339917Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"":"io task","client":"2","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34692024-02-03T17:11:02.375Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.915583262Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"":"io task","client":"1","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34702024-02-03T17:11:02.375Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.916520657Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34712024-02-03T17:11:02.375Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.916621175Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34722024-02-03T17:11:02.375Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.916794493Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34732024-02-03T17:11:02.375Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.916922961Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34742024-02-03T17:11:02.375Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.917538948Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34752024-02-03T17:11:02.375Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.917784025Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34762024-02-03T17:11:02.375Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.918123353Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34772024-02-03T17:11:02.375Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.918257055Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34782024-02-03T17:11:02.381Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.918648397Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34792024-02-03T17:11:02.381Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.918755124Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34802024-02-03T17:11:02.381Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.918784082Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34812024-02-03T17:11:02.381Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.91880515Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34822024-02-03T17:11:02.381Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.918829016Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34832024-02-03T17:11:02.381Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.918898002Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34842024-02-03T17:11:02.381Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.919160772Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34852024-02-03T17:11:02.381Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.919959747Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34862024-02-03T17:11:02.381Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.920092173Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34872024-02-03T17:11:02.381Z{"msg":"[0]R flush_numbers[0..12]: [54, 54, 54, 50, 55, 55, 25, 55, 53, 54, 53, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.920119744Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34882024-02-03T17:11:02.381Z{"msg":"[0]R generation[0..12]: [30, 30, 30, 28, 31, 31, 16, 31, 29, 30, 29, 28]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.920143431Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34892024-02-03T17:11:02.381Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.920167267Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34902024-02-03T17:11:02.382Z{"msg":"[1]R flush_numbers[0..12]: [54, 54, 54, 50, 55, 55, 25, 55, 53, 54, 53, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.920217029Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34912024-02-03T17:11:02.382Z{"msg":"[1]R generation[0..12]: [30, 30, 30, 28, 31, 31, 16, 31, 29, 30, 29, 28]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.920242878Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34922024-02-03T17:11:02.382Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.920266655Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34932024-02-03T17:11:02.382Z{"msg":"[2]R flush_numbers[0..12]: [54, 54, 54, 50, 55, 55, 25, 55, 53, 54, 53, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.920315514Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"":"downstairs"Wait for a query_work_queue command to finish before sending IO
34942024-02-03T17:11:02.382Z,"":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34952024-02-03T17:11:02.382Z{"msg":"[2]R generation[0..12]: [30, 30, 30, 28, 31, 31, 16, 31, 29, 30, 29, 28]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.920344745Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34962024-02-03T17:11:02.382Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.920368239Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34972024-02-03T17:11:02.382Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.920414993Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34982024-02-03T17:11:02.382Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.920439461Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
34992024-02-03T17:11:02.382Z{"msg":"Next flush: 56","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.920505464Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
35002024-02-03T17:11:02.382Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.920529517Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
35012024-02-03T17:11:02.382Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.920551693Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
35022024-02-03T17:11:02.382Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.920571781Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
35032024-02-03T17:11:02.382Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.920593469Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
35042024-02-03T17:11:02.382Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.920616662Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
35052024-02-03T17:11:02.382Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.920639044Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
35062024-02-03T17:11:02.382Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.920659443Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
35072024-02-03T17:11:02.383Z{"msg":"991e02cc-ee4b-460b-b436-202fd8f26e09 is now active with session: a7f6bcf7-4a9b-451e-b632-114c5ce4736a","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.920687472Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
35082024-02-03T17:11:02.383Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.92071222Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257,"session_id":"a7f6bcf7-4a9b-451e-b632-114c5ce4736a"}
35092024-02-03T17:11:02.383Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:01.920733415Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257}
35102024-02-03T17:11:02.383ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
35112024-02-03T17:11:02.383ZLoading write count information from file "/tmp/test_repair_verify.data"
35122024-02-03T17:11:02.383ZRun Repair workload
35132024-02-03T17:11:02.383Z01/30 Write block 293 len 32768 data: 4 4 3 3 3 2 3 3
35142024-02-03T17:11:02.388Z02/30 Write block 141 len 36864 data: 3 3 3 3 3 3 2 3 2
35152024-02-03T17:11:02.388Z03/30 Write block 474 len 40960 data: 2 2 2 3 3 4 4 4 3 4
35162024-02-03T17:11:02.388Z04/30 Read block 173 len 40960
35172024-02-03T17:11:02.388Z05/30 Read block 87 len 24576
35182024-02-03T17:11:02.392Z06/30 Read block 176 len 8192
35192024-02-03T17:11:02.392Z07/30 Read block 326 len 4096
35202024-02-03T17:11:02.392Z08/30 Write block 176 len 40960 data: 4 5 3 2 3 3 3 4 4 5
35212024-02-03T17:11:02.408Z09/30 Write block 97 len 20480 data: 3 3 4 4 4
35222024-02-03T17:11:02.408Z10/30 Write block 281 len 28672 data: 4 4 6 5 5 4 5
35232024-02-03T17:11:02.408Z11/30 Write block 290 len 28672 data: 5 5 4 5 5 4 4
35242024-02-03T17:11:02.412Z12/30 Read block 102 len 28672
35252024-02-03T17:11:02.412Z13/30 Write block 190 len 32768 data: 8 8 8 5 5 4 2 2
35262024-02-03T17:11:02.412Z14/30 Write block 265 len 24576 data: 5 5 6 6 4 4
35272024-02-03T17:11:02.412Z15/30 Read block 244 len 20480
35282024-02-03T17:11:02.412Z16/30 Write block 155 len 24576 data: 3 3 2 3 3 3
35292024-02-03T17:11:02.412Z17/30 Write block 291 len 16384 data: 6 5 6 6
35302024-02-03T17:11:02.412Z18/30 Read block 307 len 36864
35312024-02-03T17:11:02.412Z19/30 Read block 265 len 20480
35322024-02-03T17:11:02.412Z20/30 Read block 206 len 28672
35332024-02-03T17:11:02.417Z21/30 Write block 246 len 20480 data: 5 3 5 5 4
35342024-02-03T17:11:02.417Z22/30 Write block 279 len 20480 data: 6 6 5 5 7
35352024-02-03T17:11:02.417Z23/30 Flush
35362024-02-03T17:11:02.422Z24/30 Write block 99 len 12288 data: 5 5 5
35372024-02-03T17:11:02.428Z25/30 Read block 487 len 32768
35382024-02-03T17:11:02.428Z26/30 Read block 108 len 16384
35392024-02-03T17:11:02.432Z27/30 Write block 58 len 8192 data: 2 2
35402024-02-03T17:11:02.432Z28/30 Read block 11 len 24576
35412024-02-03T17:11:02.432Z29/30 Read block 91 len 40960
35422024-02-03T17:11:02.432Z30/30 Write block 348 len 32768 data: 5 5 4 4 4 5 5 5
35432024-02-03T17:11:02.436Z----------------------------------------------------------------
35442024-02-03T17:11:02.436Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30
35452024-02-03T17:11:02.436ZStates: Active Active Active
35462024-02-03T17:11:02.436ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
35472024-02-03T17:11:02.436Z 1 Acked 1000 Write 8 Sent Done Done false
35482024-02-03T17:11:02.436Z 2 Acked 1001 Write 9 Sent Done Done false
35492024-02-03T17:11:02.436Z 3 Acked 1002 Write 10 Sent Done Done false
35502024-02-03T17:11:02.436Z 4 Acked 1003 Read 10 Sent Done Done false
35512024-02-03T17:11:02.436Z 5 Acked 1004 Read 6 Sent Done Done false
35522024-02-03T17:11:02.436Z 6 Acked 1005 Read 2 Sent Done Done false
35532024-02-03T17:11:02.436Z 7 Acked 1006 Read 1 Sent Done Done false
35542024-02-03T17:11:02.436Z 8 Acked 1007 Write 10 Sent Done Done false
35552024-02-03T17:11:02.437Z 9 Acked 1008 Write 5 Sent Done Done false
35562024-02-03T17:11:02.437Z 10 Acked 1009 Write 7 Sent Done Done false
35572024-02-03T17:11:02.437Z 11 Acked 1010 Write 7 Sent Done Done false
35582024-02-03T17:11:02.437Z 12 Acked 1011 Read 7 Sent Done Done false
35592024-02-03T17:11:02.437Z 13 Acked 1012 Write 8 Sent Done Done false
35602024-02-03T17:11:02.437Z 14 Acked 1013 Write 6 Sent Done Done false
35612024-02-03T17:11:02.437Z 15 Acked 1014 Read 5 Sent Done Done false
35622024-02-03T17:11:02.437Z 16 Acked 1015 Write 6 Sent Done Done false
35632024-02-03T17:11:02.437Z 17 Acked 1016 Write 4 Sent Done Done false
35642024-02-03T17:11:02.437Z 18 Acked 1017 Read 9 Sent Done Done false
35652024-02-03T17:11:02.437Z 19 Acked 1018 Read 5 Sent Done Done false
35662024-02-03T17:11:02.437Z 20 Acked 1019 Read 7 Sent Done Done false
35672024-02-03T17:11:02.437Z 21 Acked 1020 Write 5 Sent Done Done false
35682024-02-03T17:11:02.437Z 22 Acked 1021 Write 5 Sent Done Done false
35692024-02-03T17:11:02.437Z 23 Acked 1022 Flush 0 Sent Done Done false
35702024-02-03T17:11:02.437Z 24 Acked 1023 Write 3 Sent Done Done false
35712024-02-03T17:11:02.437Z 25 Acked 1024 Read 8 Sent Done Done false
35722024-02-03T17:11:02.437Z 26 Acked 1025 Read 4 Sent Done Done false
35732024-02-03T17:11:02.437Z 27 Acked 1026 Write 2 Sent Done Done false
35742024-02-03T17:11:02.437Z 28 Acked 1027 Read 6 Sent Done Done false
35752024-02-03T17:11:02.437Z 29 Acked 1028 Read 10 Sent Done Done false
35762024-02-03T17:11:02.437Z 30 Acked 1029 Write 8 Sent Sent Sent false
35772024-02-03T17:11:02.437Z STATES DS:0 DS:1 DS:2 TOTAL
35782024-02-03T17:11:02.437Z New 0 0 0 0
35792024-02-03T17:11:02.437Z Sent 30 1 1 32
35802024-02-03T17:11:02.438Z Done 0 29 29 58
35812024-02-03T17:11:02.438Z Skipped 0 0 0 0
35822024-02-03T17:11:02.438Z Error 0 0 0 0
35832024-02-03T17:11:02.438ZLast Flush: 0 1022 1022
35842024-02-03T17:11:02.438ZDownstairs last five completed:
35852024-02-03T17:11:02.438ZUpstairs last five completed: 30 29 28 27 26
35862024-02-03T17:11:02.438Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:11:01.97775335Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1257}
35872024-02-03T17:11:02.438ZWrote out file "/tmp/test_repair_verify.data"
35882024-02-03T17:11:02.442Z
35892024-02-03T17:11:02.447ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
35902024-02-03T17:11:02.473ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
35912024-02-03T17:11:02.474Z 0 000-019 30 30 30 54 54 54 F F F
35922024-02-03T17:11:02.474Z 1 020-039 30 30 30 54 54 54 F F F
35932024-02-03T17:11:02.474Z 2 040-059 30 30 30 54 54 54 F T T <---
35942024-02-03T17:11:02.474Z 3 060-079 28 28 28 50 50 50 F F F
35952024-02-03T17:11:02.474Z 4 080-099 31 32 32 55 56 56 F T T <---
35962024-02-03T17:11:02.474Z 5 100-119 31 32 32 55 56 56 F T T <---
35972024-02-03T17:11:02.474Z 6 120-139 16 16 16 25 25 25 F F F
35982024-02-03T17:11:02.474Z 7 140-159 31 32 32 55 56 56 F F F <---
35992024-02-03T17:11:02.474Z 8 160-179 29 32 32 53 56 56 F F F <---
36002024-02-03T17:11:02.474Z 9 180-199 30 32 32 54 56 56 F F F <---
36012024-02-03T17:11:02.478Z 10 200-219 29 29 29 53 53 53 F F F
36022024-02-03T17:11:02.478Z 11 220-239 28 28 28 52 52 52 F F F
36032024-02-03T17:11:02.478Z 12 240-259 26 32 32 46 56 56 F F F <---
36042024-02-03T17:11:02.478Z 13 260-279 20 32 32 36 56 56 F F F <---
36052024-02-03T17:11:02.479Z 14 280-299 30 32 32 54 56 56 F F F <---
36062024-02-03T17:11:02.479Z 15 300-319 30 32 32 54 56 56 F F F <---
36072024-02-03T17:11:02.479Z 16 320-339 28 28 28 52 52 52 F F F
36082024-02-03T17:11:02.479Z 17 340-359 28 28 28 52 52 52 F T T <---
36092024-02-03T17:11:02.479Z 18 360-379 28 28 28 52 52 52 F F F
36102024-02-03T17:11:02.479Z 19 380-399 22 22 22 41 41 41 F F F
36112024-02-03T17:11:02.479Z 20 400-419 26 26 26 45 45 45 F F F
36122024-02-03T17:11:02.479Z 21 420-439 30 30 30 54 54 54 F F F
36132024-02-03T17:11:02.479Z 22 440-459 31 31 31 55 55 55 F F F
36142024-02-03T17:11:02.479Z 23 460-479 30 32 32 54 56 56 F F F <---
36152024-02-03T17:11:02.479Z 24 480-499 30 32 32 54 56 56 F F F <---
36162024-02-03T17:11:02.479Z 25 500-519 26 26 26 48 48 48 F F F
36172024-02-03T17:11:02.480Z 26 520-539 30 30 30 54 54 54 F F F
36182024-02-03T17:11:02.480Z 27 540-559 30 30 30 54 54 54 F F F
36192024-02-03T17:11:02.480Z 28 560-579 28 28 28 50 50 50 F F F
36202024-02-03T17:11:02.480Z 29 580-599 27 27 27 49 49 49 F F F
36212024-02-03T17:11:02.480ZMax gen: 32, Max flush: 56
36222024-02-03T17:11:02.480ZError: Difference in extent metadata found!
36232024-02-03T17:11:02.484ZA Difference in extent metadata is expected here
36242024-02-03T17:11:02.484ZOn loop 16
36252024-02-03T17:11:02.484Z
36262024-02-03T17:11:02.484ZVerifying data now
36272024-02-03T17:11:07.642ZNo change after verify
36282024-02-03T17:11:07.642ZLoop: 16 Downstairs dump after verify (and repair):
36292024-02-03T17:11:07.665ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
36302024-02-03T17:11:07.665Z 0 000-019 30 30 30 54 54 54 F F F
36312024-02-03T17:11:07.665Z 1 020-039 30 30 30 54 54 54 F F F
36322024-02-03T17:11:07.665Z 2 040-059 33 33 33 57 57 57 F F F
36332024-02-03T17:11:07.665Z 3 060-079 28 28 28 50 50 50 F F F
36342024-02-03T17:11:07.669Z 4 080-099 33 33 33 57 57 57 F F F
36352024-02-03T17:11:07.669Z 5 100-119 33 33 33 57 57 57 F F F
36362024-02-03T17:11:07.669Z 6 120-139 16 16 16 25 25 25 F F F
36372024-02-03T17:11:07.669Z 7 140-159 32 32 32 56 56 56 F F F
36382024-02-03T17:11:07.669Z 8 160-179 32 32 32 56 56 56 F F F
36392024-02-03T17:11:07.669Z 9 180-199 32 32 32 56 56 56 F F F
36402024-02-03T17:11:07.669Z 10 200-219 29 29 29 53 53 53 F F F
36412024-02-03T17:11:07.669Z 11 220-239 28 28 28 52 52 52 F F F
36422024-02-03T17:11:07.669Z 12 240-259 32 32 32 56 56 56 F F F
36432024-02-03T17:11:07.669Z 13 260-279 32 32 32 56 56 56 F F F
36442024-02-03T17:11:07.669Z 14 280-299 32 32 32 56 56 56 F F F
36452024-02-03T17:11:07.669Z 15 300-319 32 32 32 56 56 56 F F F
36462024-02-03T17:11:07.669Z 16 320-339 28 28 28 52 52 52 F F F
36472024-02-03T17:11:07.669Z 17 340-359 33 33 33 57 57 57 F F F
36482024-02-03T17:11:07.669Z 18 360-379 28 28 28 52 52 52 F F F
36492024-02-03T17:11:07.669Z 19 380-399 22 22 22 41 41 41 F F F
36502024-02-03T17:11:07.669Z 20 400-419 26 26 26 45 45 45 F F F
36512024-02-03T17:11:07.669Z 21 420-439 30 30 30 54 54 54 F F F
36522024-02-03T17:11:07.669Z 22 440-459 31 31 31 55 55 55 F F F
36532024-02-03T17:11:07.669Z 23 460-479 32 32 32 56 56 56 F F F
36542024-02-03T17:11:07.669Z 24 480-499 32 32 32 56 56 56 F F F
36552024-02-03T17:11:07.670Z 25 500-519 26 26 26 48 48 48 F F F
36562024-02-03T17:11:07.670Z 26 520-539 30 30 30 54 54 54 F F F
36572024-02-03T17:11:07.670Z 27 540-559 30 30 30 54 54 54 F F F
36582024-02-03T17:11:07.670Z 28 560-579 28 28 28 50 50 50 F F F
36592024-02-03T17:11:07.670Z 29 580-599 27 27 27 49 49 49 F F F
36602024-02-03T17:11:07.670ZMax gen: 33, Max flush: 57
36612024-02-03T17:11:07.675Z
36622024-02-03T17:11:07.675ZBegin loop 17
36632024-02-03T17:11:07.675ZDownstairs to restart: 0
36642024-02-03T17:11:07.700Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.245398744Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265}
36652024-02-03T17:11:07.700Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.245886236Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265}
36662024-02-03T17:11:07.704Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.245970587Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265}
36672024-02-03T17:11:07.704Z{"msg":"Crucible 43ea179f-b790-45a2-957b-609e3dc772d0 has session id: c3cbc022-16a4-45f4-8709-fa358fbfc0e5","v":0,"name":"crucible","level":Crucible runtime is spawned
36682024-02-03T17:11:07.704Z30,"time":"2024-02-03T17:11:07.246237671Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36692024-02-03T17:11:07.704Z{"msg":"Upstairs opts: Upstairs UUID: 43ea179f-b790-45a2-957b-609e3dc772d0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.246340736Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36702024-02-03T17:11:07.704Z{"msg":"Crucible stats registered with UUID: 43ea179f-b790-45a2-957b-609e3dc772d0","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.246372316Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36712024-02-03T17:11:07.704Z{"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.246421059Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265}
36722024-02-03T17:11:07.704Z{"msg":"43ea179f-b790-45a2-957b-609e3dc772d0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.246442357Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36732024-02-03T17:11:07.704Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.246462608Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36742024-02-03T17:11:07.704Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.246495505Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36752024-02-03T17:11:07.704Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.246563758Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36762024-02-03T17:11:07.704Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.246606525Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36772024-02-03T17:11:07.704Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.246715452Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36782024-02-03T17:11:07.704Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.246762412Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36792024-02-03T17:11:07.705Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.246803989Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"":"io task","client":"2","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36802024-02-03T17:11:07.705Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.246898913Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"":"io task","client":"0","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36812024-02-03T17:11:07.705Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.246947572Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"":"io task","client":"1","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36822024-02-03T17:11:07.705Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.2471339Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"":"io task","client":"0","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36832024-02-03T17:11:07.705Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.247239133Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"":"io task","client":"2","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36842024-02-03T17:11:07.705Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.247269889Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"":"io task","client":"1","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36852024-02-03T17:11:07.705Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.248767451Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36862024-02-03T17:11:07.705Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.248840896Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36872024-02-03T17:11:07.705Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.248882288Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36882024-02-03T17:11:07.705Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.24892612Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36892024-02-03T17:11:07.705Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.248969418Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36902024-02-03T17:11:07.705Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.249000611Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36912024-02-03T17:11:07.709Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.249753201Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36922024-02-03T17:11:07.709Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.249950402Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36932024-02-03T17:11:07.709Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.250417107Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36942024-02-03T17:11:07.709Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.250898117Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36952024-02-03T17:11:07.710Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.251026737Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36962024-02-03T17:11:07.710Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.251077647Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36972024-02-03T17:11:07.710Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.251161974Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36982024-02-03T17:11:07.710Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.251200617Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
36992024-02-03T17:11:07.710Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.251253577Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
37002024-02-03T17:11:07.710Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.251457326Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
37012024-02-03T17:11:07.710Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.25151098Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
37022024-02-03T17:11:07.710Z{"msg":"[0]R flush_numbers[0..12]: [54, 54, 57, 50, 57, 57, 25, 56, 56, 56, 53, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.251554973Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
37032024-02-03T17:11:07.710Z{"msg":"[0]R generation[0..12]: [30, 30, 33, 28, 33, 33, 16, 32, 32, 32, 29, 28]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.251637726Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
37042024-02-03T17:11:07.710Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.251719931Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
37052024-02-03T17:11:07.710Z{"msg":"[1]R flush_numbers[0..12]: [54, 54, 57, 50, 57, 57, 25, 56, 56, 56, 53, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.251801381Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
37062024-02-03T17:11:07.710ZWait for a query_work_queue command to finish before sending IO
37072024-02-03T17:11:07.710Z{"msg":"[1]R generation[0..12]: [30, 30, 33, 28, 33, 33, 16, 32, 32, 32, 29, 28]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.251887955Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
37082024-02-03T17:11:07.710Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.251939611Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
37092024-02-03T17:11:07.710Z{"msg":"[2]R flush_numbers[0..12]: [54, 54, 57, 50, 57, 57, 25, 56, 56, 56, 53, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.252018251Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
37102024-02-03T17:11:07.710Z{"msg":"[2]R generation[0..12]: [30, 30, 33, 28, 33, 33, 16, 32, 32, 32, 29, 28]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.252055581Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
37112024-02-03T17:11:07.710Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.252080959Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
37122024-02-03T17:11:07.710Z{"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.252102652Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
37132024-02-03T17:11:07.710Z{"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.252151125Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
37142024-02-03T17:11:07.710Z{"msg":"Next flush: 58","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.252174846Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
37152024-02-03T17:11:07.710Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.252196382Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
37162024-02-03T17:11:07.711Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.252249506Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
37172024-02-03T17:11:07.711Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.252274041Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
37182024-02-03T17:11:07.711Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.252324235Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
37192024-02-03T17:11:07.711Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.252348112Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
37202024-02-03T17:11:07.711Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.252370135Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
37212024-02-03T17:11:07.711Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.252392135Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
37222024-02-03T17:11:07.711Z{"msg":"43ea179f-b790-45a2-957b-609e3dc772d0 is now active with session: c3cbc022-16a4-45f4-8709-fa358fbfc0e5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.25241358Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
37232024-02-03T17:11:07.711Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.252433196Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265,"session_id":"c3cbc022-16a4-45f4-8709-fa358fbfc0e5"}
37242024-02-03T17:11:07.711Z{"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:07.252453215Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265}
37252024-02-03T17:11:07.711ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
37262024-02-03T17:11:07.711ZLoading write count information from file "/tmp/test_repair_verify.data"
37272024-02-03T17:11:07.711ZRun Repair workload
37282024-02-03T17:11:07.714Z01/30 Write block 304 len 40960 data: 5 4 6 4 4 4 4 3 4 5
37292024-02-03T17:11:07.715Z02/30 Write block 355 len 32768 data: 6 5 5 5 6 6 5 3
37302024-02-03T17:11:07.715Z03/30 Write block 441 len 24576 data: 5 5 3 2 2 3
37312024-02-03T17:11:07.718Z04/30 Write block 183 len 32768 data: 5 5 6 4 5 7 7 9
37322024-02-03T17:11:07.718Z05/30 Write block 75 len 32768 data: 3 4 4 4 4 4 6 5
37332024-02-03T17:11:07.720Z06/30 Write block 379 len 40960 data: 5 3 3 3 3 3 4 4 5 4
37342024-02-03T17:11:07.720Z07/30 Flush
37352024-02-03T17:11:07.724Z08/30 Write block 511 len 36864 data: 6 5 5 4 4 4 4 3 3
37362024-02-03T17:11:07.727Z09/30 Read block 113 len 12288
37372024-02-03T17:11:07.727Z10/30 Read block 322 len 8192
37382024-02-03T17:11:07.732Z11/30 Read block 73 len 12288
37392024-02-03T17:11:07.732Z12/30 Read block 353 len 4096
37402024-02-03T17:11:07.733Z13/30 Write block 474 len 28672 data: 3 3 3 4 4 5 5
37412024-02-03T17:11:07.736Z14/30 Write block 286 len 4096 data: 5
37422024-02-03T17:11:07.736Z15/30 Write block 573 len 36864 data: 4 4 4 4 3 3 3 3 3
37432024-02-03T17:11:07.736Z16/30 Write block 248 len 20480 data: 6 6 5 4 4
37442024-02-03T17:11:07.737Z17/30 Read block 416 len 8192
37452024-02-03T17:11:07.737Z18/30 Read block 26 len 28672
37462024-02-03T17:11:07.752Z19/30 Read block 86 len 28672
37472024-02-03T17:11:07.752Z20/30 Read block 195 len 24576
37482024-02-03T17:11:07.755Z21/30 Read block 103 len 40960
37492024-02-03T17:11:07.756Z22/30 Write block 93 len 20480 data: 4 4 3 3 4
37502024-02-03T17:11:07.756Z23/30 Read block 347 len 40960
37512024-02-03T17:11:07.756Z24/30 Flush
37522024-02-03T17:11:07.756Z25/30 Write block 234 len 16384 data: 3 2 3 3
37532024-02-03T17:11:07.756Z26/30 Flush
37542024-02-03T17:11:07.759Z27/30 Write block 266 len 20480 data: 6 7 7 5 5
37552024-02-03T17:11:07.759Z28/30 Read block 6 len 32768
37562024-02-03T17:11:07.759Z29/30 Read block 401 len 40960
37572024-02-03T17:11:07.766Z30/30 Read block 586 len 12288
37582024-02-03T17:11:07.767Z----------------------------------------------------------------
37592024-02-03T17:11:07.767Z Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30
37602024-02-03T17:11:07.767ZStates: Active Active Active
37612024-02-03T17:11:07.767ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
37622024-02-03T17:11:07.767Z 1 Acked 1000 Write 10 Sent Done Done false
37632024-02-03T17:11:07.767Z 2 Acked 1001 Write 8 Sent Done Done false
37642024-02-03T17:11:07.767Z 3 Acked 1002 Write 6 Sent Done Done false
37652024-02-03T17:11:07.767Z 4 Acked 1003 Write 8 Sent Done Done false
37662024-02-03T17:11:07.767Z 5 Acked 1004 Write 8 Sent Done Done false
37672024-02-03T17:11:07.767Z 6 Acked 1005 Write 10 Sent Done Done false
37682024-02-03T17:11:07.767Z 7 Acked 1006 Flush 0 Sent Done Done false
37692024-02-03T17:11:07.767Z 8 Acked 1007 Write 9 Sent Done Done false
37702024-02-03T17:11:07.767Z 9 Acked 1008 Read 3 Sent Done Done false
37712024-02-03T17:11:07.768Z 10 Acked 1009 Read 2 Sent Done Done false
37722024-02-03T17:11:07.768Z 11 Acked 1010 Read 3 Sent Done Done false
37732024-02-03T17:11:07.768Z 12 Acked 1011 Read 1 Sent Done Done false
37742024-02-03T17:11:07.768Z 13 Acked 1012 Write 7 Sent Done Done false
37752024-02-03T17:11:07.768Z 14 Acked 1013 Write 1 Sent Done Done false
37762024-02-03T17:11:07.768Z 15 Acked 1014 Write 9 Sent Done Done false
37772024-02-03T17:11:07.768Z 16 Acked 1015 Write 5 Sent Done Done false
37782024-02-03T17:11:07.768Z 17 Acked 1016 Read 2 Sent Done Done false
37792024-02-03T17:11:07.768Z 18 Acked 1017 Read 7 Sent Done Done false
37802024-02-03T17:11:07.768Z 19 Acked 1018 Read 7 Sent Done Done false
37812024-02-03T17:11:07.768Z 20 Acked 1019 Read 6 Sent Done Done false
37822024-02-03T17:11:07.768Z 21 Acked 1020 Read 10 Sent Done Done false
37832024-02-03T17:11:07.768Z 22 Acked 1021 Write 5 Sent Done Done false
37842024-02-03T17:11:07.769Z 23 Acked 1022 Read 10 Sent Done Done false
37852024-02-03T17:11:07.769Z 24 Acked 1023 Flush 0 Sent Done Done false
37862024-02-03T17:11:07.769Z 25 Acked 1024 Write 4 Sent Done Done false
37872024-02-03T17:11:07.769Z 26 Acked 1025 Flush 0 Sent Done Done false
37882024-02-03T17:11:07.769Z 27 Acked 1026 Write 5 Sent Done Done false
37892024-02-03T17:11:07.769Z 28 Acked 1027 Read 8 Sent Done Done false
37902024-02-03T17:11:07.769Z 29 Acked 1028 Read 10 Sent Done Done false
37912024-02-03T17:11:07.769Z 30 Acked 1029 Read 3 Sent Sent Done false
37922024-02-03T17:11:07.769Z STATES DS:0 DS:1 DS:2 TOTAL
37932024-02-03T17:11:07.769Z New 0 0 0 0
37942024-02-03T17:11:07.769Z Sent 30 1 0 31
37952024-02-03T17:11:07.769Z Done 0 29 30 59
37962024-02-03T17:11:07.769Z Skipped 0 0 0 0
37972024-02-03T17:11:07.770Z Error 0 0 0 0
37982024-02-03T17:11:07.770ZLast Flush: 0 1025 1025
37992024-02-03T17:11:07.770ZDownstairs last five completed:
38002024-02-03T17:11:07.770ZUpstairs last five completed: 30 29 28 27 26
38012024-02-03T17:11:07.770Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:11:07.307688855Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1265}
38022024-02-03T17:11:07.770ZWrote out file "/tmp/test_repair_verify.data"
38032024-02-03T17:11:07.774Z
38042024-02-03T17:11:07.777ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
38052024-02-03T17:11:07.799ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38062024-02-03T17:11:07.799Z 0 000-019 30 30 30 54 54 54 F F F
38072024-02-03T17:11:07.799Z 1 020-039 30 30 30 54 54 54 F F F
38082024-02-03T17:11:07.799Z 2 040-059 33 33 33 57 57 57 F F F
38092024-02-03T17:11:07.799Z 3 060-079 28 34 34 50 58 58 F F F <---
38102024-02-03T17:11:07.803Z 4 080-099 33 34 34 57 59 59 F F F <---
38112024-02-03T17:11:07.803Z 5 100-119 33 33 33 57 57 57 F F F
38122024-02-03T17:11:07.803Z 6 120-139 16 16 16 25 25 25 F F F
38132024-02-03T17:11:07.803Z 7 140-159 32 32 32 56 56 56 F F F
38142024-02-03T17:11:07.803Z 8 160-179 32 32 32 56 56 56 F F F
38152024-02-03T17:11:07.803Z 9 180-199 32 34 34 56 58 58 F F F <---
38162024-02-03T17:11:07.803Z 10 200-219 29 29 29 53 53 53 F F F
38172024-02-03T17:11:07.803Z 11 220-239 28 34 34 52 60 60 F F F <---
38182024-02-03T17:11:07.803Z 12 240-259 32 34 34 56 59 59 F F F <---
38192024-02-03T17:11:07.803Z 13 260-279 32 32 32 56 56 56 F T T <---
38202024-02-03T17:11:07.803Z 14 280-299 32 34 34 56 59 59 F F F <---
38212024-02-03T17:11:07.803Z 15 300-319 32 34 34 56 58 58 F F F <---
38222024-02-03T17:11:07.803Z 16 320-339 28 28 28 52 52 52 F F F
38232024-02-03T17:11:07.803Z 17 340-359 33 34 34 57 58 58 F F F <---
38242024-02-03T17:11:07.803Z 18 360-379 28 34 34 52 58 58 F F F <---
38252024-02-03T17:11:07.803Z 19 380-399 22 34 34 41 58 58 F F F <---
38262024-02-03T17:11:07.803Z 20 400-419 26 26 26 45 45 45 F F F
38272024-02-03T17:11:07.803Z 21 420-439 30 30 30 54 54 54 F F F
38282024-02-03T17:11:07.804Z 22 440-459 31 34 34 55 58 58 F F F <---
38292024-02-03T17:11:07.804Z 23 460-479 32 34 34 56 59 59 F F F <---
38302024-02-03T17:11:07.804Z 24 480-499 32 34 34 56 59 59 F F F <---
38312024-02-03T17:11:07.804Z 25 500-519 26 34 34 48 59 59 F F F <---
38322024-02-03T17:11:07.804Z 26 520-539 30 30 30 54 54 54 F F F
38332024-02-03T17:11:07.804Z 27 540-559 30 30 30 54 54 54 F F F
38342024-02-03T17:11:07.804Z 28 560-579 28 34 34 50 59 59 F F F <---
38352024-02-03T17:11:07.804Z 29 580-599 27 34 34 49 59 59 F F F <---
38362024-02-03T17:11:07.804ZMax gen: 34, Max flush: 60
38372024-02-03T17:11:07.804ZError: Difference in extent metadata found!
38382024-02-03T17:11:07.810ZA Difference in extent metadata is expected here
38392024-02-03T17:11:07.810ZOn loop 17
38402024-02-03T17:11:07.810Z
38412024-02-03T17:11:07.810ZVerifying data now
38422024-02-03T17:11:13.222ZNo change after verify
38432024-02-03T17:11:13.223ZLoop: 17 Downstairs dump after verify (and repair):
38442024-02-03T17:11:13.246ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38452024-02-03T17:11:13.246Z 0 000-019 30 30 30 54 54 54 F F F
38462024-02-03T17:11:13.246Z 1 020-039 30 30 30 54 54 54 F F F
38472024-02-03T17:11:13.246Z 2 040-059 33 33 33 57 57 57 F F F
38482024-02-03T17:11:13.249Z 3 060-079 34 34 34 58 58 58 F F F
38492024-02-03T17:11:13.250Z 4 080-099 34 34 34 59 59 59 F F F
38502024-02-03T17:11:13.250Z 5 100-119 33 33 33 57 57 57 F F F
38512024-02-03T17:11:13.250Z 6 120-139 16 16 16 25 25 25 F F F
38522024-02-03T17:11:13.250Z 7 140-159 32 32 32 56 56 56 F F F
38532024-02-03T17:11:13.250Z 8 160-179 32 32 32 56 56 56 F F F
38542024-02-03T17:11:13.250Z 9 180-199 34 34 34 58 58 58 F F F
38552024-02-03T17:11:13.250Z 10 200-219 29 29 29 53 53 53 F F F
38562024-02-03T17:11:13.250Z 11 220-239 34 34 34 60 60 60 F F F
38572024-02-03T17:11:13.250Z 12 240-259 34 34 34 59 59 59 F F F
38582024-02-03T17:11:13.250Z 13 260-279 35 35 35 61 61 61 F F F
38592024-02-03T17:11:13.250Z 14 280-299 34 34 34 59 59 59 F F F
38602024-02-03T17:11:13.250Z 15 300-319 34 34 34 58 58 58 F F F
38612024-02-03T17:11:13.250Z 16 320-339 28 28 28 52 52 52 F F F
38622024-02-03T17:11:13.250Z 17 340-359 34 34 34 58 58 58 F F F
38632024-02-03T17:11:13.250Z 18 360-379 34 34 34 58 58 58 F F F
38642024-02-03T17:11:13.250Z 19 380-399 34 34 34 58 58 58 F F F
38652024-02-03T17:11:13.250Z 20 400-419 26 26 26 45 45 45 F F F
38662024-02-03T17:11:13.250Z 21 420-439 30 30 30 54 54 54 F F F
38672024-02-03T17:11:13.250Z 22 440-459 34 34 34 58 58 58 F F F
38682024-02-03T17:11:13.250Z 23 460-479 34 34 34 59 59 59 F F F
38692024-02-03T17:11:13.250Z 24 480-499 34 34 34 59 59 59 F F F
38702024-02-03T17:11:13.250Z 25 500-519 34 34 34 59 59 59 F F F
38712024-02-03T17:11:13.251Z 26 520-539 30 30 30 54 54 54 F F F
38722024-02-03T17:11:13.251Z 27 540-559 30 30 30 54 54 54 F F F
38732024-02-03T17:11:13.251Z 28 560-579 34 34 34 59 59 59 F F F
38742024-02-03T17:11:13.251Z 29 580-599 34 34 34 59 59 59 F F F
38752024-02-03T17:11:13.251ZMax gen: 35, Max flush: 61
38762024-02-03T17:11:13.251Z
38772024-02-03T17:11:13.251ZBegin loop 18
38782024-02-03T17:11:13.251ZDownstairs to restart: 2
38792024-02-03T17:11:13.282Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.825877983Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273}
38802024-02-03T17:11:13.287Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
38812024-02-03T17:11:13.287Z,"time":"2024-02-03T17:11:12.826360197Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273}
38822024-02-03T17:11:13.287Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.826516078Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273}
38832024-02-03T17:11:13.287Z{"msg":"Crucible 40998746-803a-42f8-bfe9-6aa38161ad46 has session id: b5202dba-4504-44b9-92d2-571f0258f8cd","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.82655308Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
38842024-02-03T17:11:13.287Z{"msg":"Upstairs opts: Upstairs UUID: 40998746-803a-42f8-bfe9-6aa38161ad46, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.826638201Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
38852024-02-03T17:11:13.287Z{"msg":"Crucible stats registered with UUID: 40998746-803a-42f8-bfe9-6aa38161ad46","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.826667544Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
38862024-02-03T17:11:13.287Z{"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.826689794Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273}
38872024-02-03T17:11:13.287Z{"msg":"40998746-803a-42f8-bfe9-6aa38161ad46 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.8267343Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
38882024-02-03T17:11:13.287Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.82675732Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
38892024-02-03T17:11:13.288Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.826780519Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
38902024-02-03T17:11:13.288Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.826853913Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
38912024-02-03T17:11:13.288Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.826899344Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
38922024-02-03T17:11:13.288Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.827031627Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
38932024-02-03T17:11:13.288Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.827066203Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
38942024-02-03T17:11:13.288Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.82712237Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"":"io task","client":"0","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
38952024-02-03T17:11:13.288Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.827146751Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"":"io task","client":"1","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
38962024-02-03T17:11:13.288Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.827222204Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"":"io task","client":"2","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
38972024-02-03T17:11:13.288Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.827472532Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"":"io task","client":"0","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
38982024-02-03T17:11:13.288Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.827727872Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"":"io task","client":"1","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
38992024-02-03T17:11:13.288Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.827800318Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"":"io task","client":"2","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39002024-02-03T17:11:13.288Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.829038834Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39012024-02-03T17:11:13.288Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.829179901Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39022024-02-03T17:11:13.288Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.829289853Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39032024-02-03T17:11:13.288Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.829321107Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39042024-02-03T17:11:13.288Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.829505132Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39052024-02-03T17:11:13.289Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.829563147Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39062024-02-03T17:11:13.289Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.830088626Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39072024-02-03T17:11:13.289Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.830448474Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39082024-02-03T17:11:13.289Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.830802417Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39092024-02-03T17:11:13.289Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.830974896Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39102024-02-03T17:11:13.289Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.831089993Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39112024-02-03T17:11:13.289Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.831133108Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39122024-02-03T17:11:13.289Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.831165737Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39132024-02-03T17:11:13.289Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.831205202Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39142024-02-03T17:11:13.294Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.831274126Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39152024-02-03T17:11:13.295Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.83174574Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39162024-02-03T17:11:13.295Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.831843758Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39172024-02-03T17:11:13.295Z{"msg":"[0]R flush_numbers[0..12]: [54, 54, 57, 58, 59, 57, 25, 56, 56, 58, 53, 60]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.831891795Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39182024-02-03T17:11:13.295Z{"msg":"[0]R generation[0..12]: [30, 30, 33, 34, 34, 33, 16, 32, 32, 34, 29, 34]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.831933213Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39192024-02-03T17:11:13.295Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.831976886Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39202024-02-03T17:11:13.295Z{"msg":"[1]R flush_numbers[0..12]: [54, 54, 57, 58, 59, 57, 25, 56, 56, 58, 53, 60]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
39212024-02-03T17:11:13.295Z,"time":"2024-02-03T17:11:12.832055686Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39222024-02-03T17:11:13.295Z{"msg":"[1]R generation[0..12]: [30, 30, 33, 34, 34, 33, 16, 32, 32, 34, 29, 34]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.832149003Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39232024-02-03T17:11:13.295Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.83223493Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39242024-02-03T17:11:13.295Z{"msg":"[2]R flush_numbers[0..12]: [54, 54, 57, 58, 59, 57, 25, 56, 56, 58, 53, 60]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.832320982Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39252024-02-03T17:11:13.295Z{"msg":"[2]R generation[0..12]: [30, 30, 33, 34, 34, 33, 16, 32, 32, 34, 29, 34]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.832422873Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39262024-02-03T17:11:13.295Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.832505864Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39272024-02-03T17:11:13.296Z{"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.83259058Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39282024-02-03T17:11:13.296Z{"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.832679288Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39292024-02-03T17:11:13.296Z{"msg":"Next flush: 62","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.832760554Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39302024-02-03T17:11:13.296Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.832841179Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39312024-02-03T17:11:13.296Z{"msg":"No downstairs reconciliation required","v":0,"name":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
39322024-02-03T17:11:13.296Zcrucible","level":30,"time":"2024-02-03T17:11:12.833025084Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39332024-02-03T17:11:13.296Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.833087907Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39342024-02-03T17:11:13.296Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.833114357Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39352024-02-03T17:11:13.296Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.833162286Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39362024-02-03T17:11:13.296Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.833187015Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39372024-02-03T17:11:13.296Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.833208065Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39382024-02-03T17:11:13.296Z{"msg":"40998746-803a-42f8-bfe9-6aa38161ad46 is now active with session: b5202dba-4504-44b9-92d2-571f0258f8cd","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.833254856Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39392024-02-03T17:11:13.297Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.833277267Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273,"session_id":"b5202dba-4504-44b9-92d2-571f0258f8cd"}
39402024-02-03T17:11:13.297Z{"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:12.8333232Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273}
39412024-02-03T17:11:13.297ZLoading write count information from file "/tmp/test_repair_verify.data"
39422024-02-03T17:11:13.297ZRun Repair workload
39432024-02-03T17:11:13.297Z01/30 Flush
39442024-02-03T17:11:13.297Z02/30 Write block 161 len 32768 data: 3 4 4 4 4 4 4 3
39452024-02-03T17:11:13.297Z03/30 Read block 166 len 12288
39462024-02-03T17:11:13.301Z04/30 Read block 207 len 32768
39472024-02-03T17:11:13.301Z05/30 Read block 22 len 20480
39482024-02-03T17:11:13.301Z06/30 Read block 188 len 12288
39492024-02-03T17:11:13.301Z07/30 Write block 228 len 8192 data: 2 2
39502024-02-03T17:11:13.301Z08/30 Read block 423 len 12288
39512024-02-03T17:11:13.304Z09/30 Write block 319 len 28672 data: 5 5 5 5 6 3 3
39522024-02-03T17:11:13.304Z10/30 Write block 319 len 24576 data: 6 6 6 6 7 4
39532024-02-03T17:11:13.304Z11/30 Read block 164 len 12288
39542024-02-03T17:11:13.308Z12/30 Write block 116 len 20480 data: 5 7 6 5 4
39552024-02-03T17:11:13.308Z13/30 Read block 561 len 12288
39562024-02-03T17:11:13.311Z14/30 Write block 380 len 8192 data: 4 4
39572024-02-03T17:11:13.311Z15/30 Read block 127 len 20480
39582024-02-03T17:11:13.338Z16/30 Write block 294 len 24576 data: 7 5 5 4 3 4
39592024-02-03T17:11:13.338Z17/30 Write block 568 len 32768 data: 5 5 5 4 4 5 5 5
39602024-02-03T17:11:13.338Z18/30 Write block 501 len 16384 data: 4 4 4 5
39612024-02-03T17:11:13.338Z19/30 Read block 286 len 24576
39622024-02-03T17:11:13.338Z20/30 Read block 345 len 28672
39632024-02-03T17:11:13.338Z21/30 Read block 453 len 40960
39642024-02-03T17:11:13.338Z22/30 Write block 14 len 40960 data: 3 2 2 2 3 4 3 3 4 4
39652024-02-03T17:11:13.338Z23/30 Write block 559 len 40960 data: 3 3 4 4 4 4 3 3 3 6
39662024-02-03T17:11:13.338Z24/30 Read block 240 len 8192
39672024-02-03T17:11:13.338Z25/30 Flush
39682024-02-03T17:11:13.364Z26/30 Write block 564 len 12288 data: 5 4 4
39692024-02-03T17:11:13.368Z27/30 Read block 314 len 4096
39702024-02-03T17:11:13.368Z28/30 Read block 296 len 4096
39712024-02-03T17:11:13.368Z29/30 Write block 547 len 32768 data: 5 4 4 4 3 3 5 4
39722024-02-03T17:11:13.368Z30/30 Write block 105 len 12288 data: 4 5 5
39732024-02-03T17:11:13.368Z----------------------------------------------------------------
39742024-02-03T17:11:13.368Z Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:29
39752024-02-03T17:11:13.368ZStates: Active Active Active
39762024-02-03T17:11:13.368ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
39772024-02-03T17:11:13.368Z 2 Acked 1001 Write 8 Done Done Done false
39782024-02-03T17:11:13.369Z 3 Acked 1002 Read 3 Done Done Sent false
39792024-02-03T17:11:13.369Z 4 Acked 1003 Read 8 Done Done Sent false
39802024-02-03T17:11:13.369Z 5 Acked 1004 Read 5 Done Done Sent false
39812024-02-03T17:11:13.369Z 6 Acked 1005 Read 3 Done Done Sent false
39822024-02-03T17:11:13.369Z 7 Acked 1006 Write 2 Done Done Sent false
39832024-02-03T17:11:13.369Z 8 Acked 1007 Read 3 Done Done Sent false
39842024-02-03T17:11:13.369Z 9 Acked 1008 Write 7 Done Done Sent false
39852024-02-03T17:11:13.369Z 10 Acked 1009 Write 6 Done Done Sent false
39862024-02-03T17:11:13.369Z 11 Acked 1010 Read 3 Done Done Sent false
39872024-02-03T17:11:13.369Z 12 Acked 1011 Write 5 Done Done Sent false
39882024-02-03T17:11:13.369Z 13 Acked 1012 Read 3 Done Done Sent false
39892024-02-03T17:11:13.369Z 14 Acked 1013 Write 2 Done Done Sent false
39902024-02-03T17:11:13.369Z 15 Acked 1014 Read 5 Done Done Sent false
39912024-02-03T17:11:13.369Z 16 Acked 1015 Write 6 Done Done Sent false
39922024-02-03T17:11:13.370Z 17 Acked 1016 Write 8 Done Done Sent false
39932024-02-03T17:11:13.370Z 18 Acked 1017 Write 4 Done Done Sent false
39942024-02-03T17:11:13.370Z 19 Acked 1018 Read 6 Done Done Sent false
39952024-02-03T17:11:13.370Z 20 Acked 1019 Read 7 Done Done Sent false
39962024-02-03T17:11:13.370Z 21 Acked 1020 Read 10 Done Done Sent false
39972024-02-03T17:11:13.370Z 22 Acked 1021 Write 10 Done Done Sent false
39982024-02-03T17:11:13.370Z 23 Acked 1022 Write 10 Done Done Sent false
39992024-02-03T17:11:13.370Z 24 Acked 1023 Read 2 Done Done Sent false
40002024-02-03T17:11:13.370Z 25 Acked 1024 Flush 0 Done Done Sent false
40012024-02-03T17:11:13.370Z 26 Acked 1025 Write 3 Done Done Sent false
40022024-02-03T17:11:13.370Z 27 Acked 1026 Read 1 Done Done Sent false
40032024-02-03T17:11:13.370Z 28 Acked 1027 Read 1 Done Done Sent false
40042024-02-03T17:11:13.370Z 29 Acked 1028 Write 8 Done Sent Sent false
40052024-02-03T17:11:13.371Z 30 Acked 1029 Write 3 Sent Sent Sent false
40062024-02-03T17:11:13.371Z STATES DS:0 DS:1 DS:2 TOTAL
40072024-02-03T17:11:13.371Z New 0 0 0 0
40082024-02-03T17:11:13.371Z Sent 1 2 28 31
40092024-02-03T17:11:13.371Z Done 28 27 1 56
40102024-02-03T17:11:13.371Z Skipped 0 0 0 0
40112024-02-03T17:11:13.371Z Error 0 0 0 0
40122024-02-03T17:11:13.371ZLast Flush: 1024 1024 1000
40132024-02-03T17:11:13.371ZDownstairs last five completed: 1000
40142024-02-03T17:11:13.371ZUpstairs last five completed: 30 29 28 27 26
40152024-02-03T17:11:13.371Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:11:12.903304285Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1273}
40162024-02-03T17:11:13.371ZWrote out file "/tmp/test_repair_verify.data"
40172024-02-03T17:11:13.371Z
40182024-02-03T17:11:13.375ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
40192024-02-03T17:11:13.397ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
40202024-02-03T17:11:13.397Z 0 000-019 36 36 30 63 63 54 F F F <---
40212024-02-03T17:11:13.397Z 1 020-039 36 36 30 63 63 54 F F F <---
40222024-02-03T17:11:13.397Z 2 040-059 33 33 33 57 57 57 F F F
40232024-02-03T17:11:13.397Z 3 060-079 34 34 34 58 58 58 F F F
40242024-02-03T17:11:13.397Z 4 080-099 34 34 34 59 59 59 F F F
40252024-02-03T17:11:13.400Z 5 100-119 36 36 33 63 63 57 T T F <---
40262024-02-03T17:11:13.401Z 6 120-139 36 36 16 63 63 25 F F F <---
40272024-02-03T17:11:13.401Z 7 140-159 32 32 32 56 56 56 F F F
40282024-02-03T17:11:13.401Z 8 160-179 36 36 32 63 63 56 F F T <---
40292024-02-03T17:11:13.401Z 9 180-199 34 34 34 58 58 58 F F F
40302024-02-03T17:11:13.401Z 10 200-219 29 29 29 53 53 53 F F F
40312024-02-03T17:11:13.401Z 11 220-239 36 36 34 63 63 60 F F F <---
40322024-02-03T17:11:13.401Z 12 240-259 34 34 34 59 59 59 F F F
40332024-02-03T17:11:13.401Z 13 260-279 35 35 35 61 61 61 F F F
40342024-02-03T17:11:13.401Z 14 280-299 36 36 34 63 63 59 F F F <---
40352024-02-03T17:11:13.401Z 15 300-319 36 36 34 63 63 58 F F F <---
40362024-02-03T17:11:13.401Z 16 320-339 36 36 28 63 63 52 F F F <---
40372024-02-03T17:11:13.401Z 17 340-359 34 34 34 58 58 58 F F F
40382024-02-03T17:11:13.402Z 18 360-379 34 34 34 58 58 58 F F F
40392024-02-03T17:11:13.402Z 19 380-399 36 36 34 63 63 58 F F F <---
40402024-02-03T17:11:13.402Z 20 400-419 26 26 26 45 45 45 F F F
40412024-02-03T17:11:13.402Z 21 420-439 30 30 30 54 54 54 F F F
40422024-02-03T17:11:13.402Z 22 440-459 34 34 34 58 58 58 F F F
40432024-02-03T17:11:13.402Z 23 460-479 34 34 34 59 59 59 F F F
40442024-02-03T17:11:13.402Z 24 480-499 34 34 34 59 59 59 F F F
40452024-02-03T17:11:13.402Z 25 500-519 36 36 34 63 63 59 F F F <---
40462024-02-03T17:11:13.402Z 26 520-539 30 30 30 54 54 54 F F F
40472024-02-03T17:11:13.402Z 27 540-559 36 36 30 63 63 54 T T F <---
40482024-02-03T17:11:13.402Z 28 560-579 36 36 34 63 63 59 T T F <---
40492024-02-03T17:11:13.402Z 29 580-599 34 34 34 59 59 59 F F F
40502024-02-03T17:11:13.402ZMax gen: 36, Max flush: 63
40512024-02-03T17:11:13.403ZError: Difference in extent metadata found!
40522024-02-03T17:11:13.406ZA Difference in extent metadata is expected here
40532024-02-03T17:11:13.406ZOn loop 18
40542024-02-03T17:11:13.406Z
40552024-02-03T17:11:13.406ZVerifying data now
40562024-02-03T17:11:18.532ZNo change after verify
40572024-02-03T17:11:18.532ZLoop: 18 Downstairs dump after verify (and repair):
40582024-02-03T17:11:18.557ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
40592024-02-03T17:11:18.557Z 0 000-019 36 36 36 63 63 63 F F F
40602024-02-03T17:11:18.558Z 1 020-039 36 36 36 63 63 63 F F F
40612024-02-03T17:11:18.561Z 2 040-059 33 33 33 57 57 57 F F F
40622024-02-03T17:11:18.562Z 3 060-079 34 34 34 58 58 58 F F F
40632024-02-03T17:11:18.562Z 4 080-099 34 34 34 59 59 59 F F F
40642024-02-03T17:11:18.562Z 5 100-119 37 37 37 64 64 64 F F F
40652024-02-03T17:11:18.562Z 6 120-139 36 36 36 63 63 63 F F F
40662024-02-03T17:11:18.562Z 7 140-159 32 32 32 56 56 56 F F F
40672024-02-03T17:11:18.562Z 8 160-179 36 36 36 63 63 63 F F F
40682024-02-03T17:11:18.562Z 9 180-199 34 34 34 58 58 58 F F F
40692024-02-03T17:11:18.562Z 10 200-219 29 29 29 53 53 53 F F F
40702024-02-03T17:11:18.562Z 11 220-239 36 36 36 63 63 63 F F F
40712024-02-03T17:11:18.562Z 12 240-259 34 34 34 59 59 59 F F F
40722024-02-03T17:11:18.562Z 13 260-279 35 35 35 61 61 61 F F F
40732024-02-03T17:11:18.562Z 14 280-299 36 36 36 63 63 63 F F F
40742024-02-03T17:11:18.562Z 15 300-319 36 36 36 63 63 63 F F F
40752024-02-03T17:11:18.562Z 16 320-339 36 36 36 63 63 63 F F F
40762024-02-03T17:11:18.562Z 17 340-359 34 34 34 58 58 58 F F F
40772024-02-03T17:11:18.562Z 18 360-379 34 34 34 58 58 58 F F F
40782024-02-03T17:11:18.562Z 19 380-399 36 36 36 63 63 63 F F F
40792024-02-03T17:11:18.562Z 20 400-419 26 26 26 45 45 45 F F F
40802024-02-03T17:11:18.562Z 21 420-439 30 30 30 54 54 54 F F F
40812024-02-03T17:11:18.562Z 22 440-459 34 34 34 58 58 58 F F F
40822024-02-03T17:11:18.562Z 23 460-479 34 34 34 59 59 59 F F F
40832024-02-03T17:11:18.562Z 24 480-499 34 34 34 59 59 59 F F F
40842024-02-03T17:11:18.562Z 25 500-519 36 36 36 63 63 63 F F F
40852024-02-03T17:11:18.562Z 26 520-539 30 30 30 54 54 54 F F F
40862024-02-03T17:11:18.563Z 27 540-559 37 37 37 64 64 64 F F F
40872024-02-03T17:11:18.563Z 28 560-579 37 37 37 64 64 64 F F F
40882024-02-03T17:11:18.563Z 29 580-599 34 34 34 59 59 59 F F F
40892024-02-03T17:11:18.563ZMax gen: 37, Max flush: 64
40902024-02-03T17:11:18.563Z
40912024-02-03T17:11:18.563ZBegin loop 19
40922024-02-03T17:11:18.563ZDownstairs to restart: 2
40932024-02-03T17:11:18.591Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.136900983Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281}
40942024-02-03T17:11:18.602Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.137373515Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281}
40952024-02-03T17:11:18.602Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.137457356Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281}
40962024-02-03T17:11:18.603ZCrucible runtime is spawned
40972024-02-03T17:11:18.603Z{"msg":"Crucible 0469a27f-ac02-41a1-ab58-c83dcdf5e3f0 has session id: f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.137658355Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
40982024-02-03T17:11:18.603Z{"msg":"Upstairs opts: Upstairs UUID: 0469a27f-ac02-41a1-ab58-c83dcdf5e3f0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.137756692Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
40992024-02-03T17:11:18.603Z{"msg":"Crucible stats registered with UUID: 0469a27f-ac02-41a1-ab58-c83dcdf5e3f0","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.13778755Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41002024-02-03T17:11:18.603Z{"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.137853588Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281}
41012024-02-03T17:11:18.603Z{"msg":"0469a27f-ac02-41a1-ab58-c83dcdf5e3f0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.137878832Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41022024-02-03T17:11:18.603Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.13790042Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41032024-02-03T17:11:18.603Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.137989156Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41042024-02-03T17:11:18.603Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.138097123Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41052024-02-03T17:11:18.603Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.138211738Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41062024-02-03T17:11:18.603Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.138328041Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41072024-02-03T17:11:18.603Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.138412832Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41082024-02-03T17:11:18.603Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.138460834Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"":"io task","client":"2","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41092024-02-03T17:11:18.603Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.13854735Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"":"io task","client":"0","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41102024-02-03T17:11:18.603Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.13862096Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"":"io task","client":"1","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41112024-02-03T17:11:18.603Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.138667514Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"":"io task","client":"1","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41122024-02-03T17:11:18.603Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.138740905Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"":"io task","client":"0","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41132024-02-03T17:11:18.603Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.1387884Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"":"io task","client":"2","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41142024-02-03T17:11:18.603Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.139952737Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41152024-02-03T17:11:18.603Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.140028408Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41162024-02-03T17:11:18.603Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.140055113Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41172024-02-03T17:11:18.603Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.140077261Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41182024-02-03T17:11:18.604Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.140959507Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41192024-02-03T17:11:18.604Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.141103036Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41202024-02-03T17:11:18.604Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.141134266Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41212024-02-03T17:11:18.604Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.141164412Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41222024-02-03T17:11:18.604Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.142026092Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41232024-02-03T17:11:18.604Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.142126718Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41242024-02-03T17:11:18.604Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.142176785Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41252024-02-03T17:11:18.604Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.142262978Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41262024-02-03T17:11:18.604Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.1423458Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41272024-02-03T17:11:18.604Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.142427174Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41282024-02-03T17:11:18.604Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.142472743Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41292024-02-03T17:11:18.604Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.143195481Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41302024-02-03T17:11:18.604Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.143271529Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41312024-02-03T17:11:18.604Z{"msg":"[0]R flush_numbers[0..12]: [63, 63, 57, 58, 59, 64, 63, 56, 63, 58, 53, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.14329984Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41322024-02-03T17:11:18.604Z{"msg":"[0]R generation[0..12]: [36, 36, 33, 34, 34, 37, 36, 32, 36, 34, 29, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.143323511Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41332024-02-03T17:11:18.604Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.143390286Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41342024-02-03T17:11:18.604Z{"msg":"[1]R flush_numbers[0..12]: [63, 63, 57, 58, 59, 64, 63, 56, 63, 58, 53, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.143436122Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41352024-02-03T17:11:18.604Z{"msg":"[1]R generation[0..12]: [36, 36, 33, 34, 34, 37, 36, 32, 36, 34, 29, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.143504087Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41362024-02-03T17:11:18.604Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
41372024-02-03T17:11:18.604Z2024-02-03T17:11:18.143550959Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41382024-02-03T17:11:18.604Z{"msg":"[2]R flush_numbers[0..12]: [63, 63, 57, 58, 59, 64, 63, 56, 63, 58, 53, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.143627542Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41392024-02-03T17:11:18.604Z{"msg":"[2]R generation[0..12]: [36, 36, 33, 34, 34, 37, 36, 32, 36, 34, 29, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.143671908Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41402024-02-03T17:11:18.604Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.143739678Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41412024-02-03T17:11:18.604Z{"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.143782301Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41422024-02-03T17:11:18.605Z{"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.143848005Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41432024-02-03T17:11:18.605Z{"msg":"Next flush: 65","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.143889866Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41442024-02-03T17:11:18.605Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.143958187Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41452024-02-03T17:11:18.605Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.144001353Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41462024-02-03T17:11:18.605Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.144075642Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41472024-02-03T17:11:18.605Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.144117054Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41482024-02-03T17:11:18.605Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.144185356Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
41492024-02-03T17:11:18.605Zf9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41502024-02-03T17:11:18.605Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.144233596Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41512024-02-03T17:11:18.605Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.144297224Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41522024-02-03T17:11:18.605Z{"msg":"0469a27f-ac02-41a1-ab58-c83dcdf5e3f0 is now active with session: f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.144339069Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41532024-02-03T17:11:18.605Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.14440147Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281,"session_id":"f9e7cd44-9646-4ac3-96d2-1c68d5a0b2aa"}
41542024-02-03T17:11:18.605Z{"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:18.144442116Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281}
41552024-02-03T17:11:18.605ZLoading write count information from file "/tmp/test_repair_verify.data"
41562024-02-03T17:11:18.605ZRun Repair workload
41572024-02-03T17:11:18.605Z01/30 Write block 391 len 12288 data: 4 5 5
41582024-02-03T17:11:18.609Z02/30 Write block 541 len 36864 data: 2 2 2 2 3 4 6 5 5
41592024-02-03T17:11:18.609Z03/30 Write block 331 len 16384 data: 4 5 5 5
41602024-02-03T17:11:18.609Z04/30 Read block 415 len 12288
41612024-02-03T17:11:18.609Z05/30 Flush
41622024-02-03T17:11:18.612Z06/30 Write block 429 len 40960 data: 5 5 4 3 3 3 4 5 4 5
41632024-02-03T17:11:18.613Z07/30 Read block 251 len 12288
41642024-02-03T17:11:18.613Z08/30 Read block 50 len 32768
41652024-02-03T17:11:18.616Z09/30 Write block 258 len 16384 data: 4 3 3 3
41662024-02-03T17:11:18.616Z10/30 Write block 42 len 8192 data: 4 2
41672024-02-03T17:11:18.619Z11/30 Read block 586 len 16384
41682024-02-03T17:11:18.619Z12/30 Read block 183 len 12288
41692024-02-03T17:11:18.622Z13/30 Read block 524 len 8192
41702024-02-03T17:11:18.622Z14/30 Write block 73 len 28672 data: 3 4 4 5 5 5 5
41712024-02-03T17:11:18.622Z15/30 Read block 445 len 32768
41722024-02-03T17:11:18.625Z16/30 Read block 111 len 36864
41732024-02-03T17:11:18.628Z17/30 Write block 2 len 24576 data: 3 4 4 4 5 4
41742024-02-03T17:11:18.628Z18/30 Write block 523 len 32768 data: 3 3 4 4 4 4 4 4
41752024-02-03T17:11:18.631Z19/30 Write block 413 len 40960 data: 3 4 4 4 4 4 3 3 3 3
41762024-02-03T17:11:18.631Z20/30 Read block 147 len 40960
41772024-02-03T17:11:18.634Z21/30 Write block 312 len 24576 data: 5 6 5 5 4 5
41782024-02-03T17:11:18.634Z22/30 Read block 259 len 12288
41792024-02-03T17:11:18.637Z23/30 Write block 309 len 12288 data: 5 5 4
41802024-02-03T17:11:18.637Z24/30 Write block 417 len 8192 data: 5 5
41812024-02-03T17:11:18.637Z25/30 Flush
41822024-02-03T17:11:18.642Z26/30 Flush
41832024-02-03T17:11:18.645Z27/30 Write block 586 len 16384 data: 4 3 3 3
41842024-02-03T17:11:18.645Z28/30 Read block 222 len 40960
41852024-02-03T17:11:18.650Z29/30 Write block 443 len 24576 data: 4 3 3 4 3 3
41862024-02-03T17:11:18.650Z30/30 Write block 17 len 12288 data: 3 4 5
41872024-02-03T17:11:18.654Z----------------------------------------------------------------
41882024-02-03T17:11:18.654Z Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:30
41892024-02-03T17:11:18.654ZStates: Active Active Active
41902024-02-03T17:11:18.654ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
41912024-02-03T17:11:18.655Z 1 Acked 1000 Write 3 Done Done Done false
41922024-02-03T17:11:18.655Z 2 Acked 1001 Write 9 Done Done Done false
41932024-02-03T17:11:18.655Z 3 Acked 1002 Write 4 Done Done Sent false
41942024-02-03T17:11:18.655Z 4 Acked 1003 Read 3 Done Done Sent false
41952024-02-03T17:11:18.655Z 5 Acked 1004 Flush 0 Done Done Sent false
41962024-02-03T17:11:18.655Z 6 Acked 1005 Write 10 Done Done Sent false
41972024-02-03T17:11:18.655Z 7 Acked 1006 Read 3 Done Done Sent false
41982024-02-03T17:11:18.655Z 8 Acked 1007 Read 8 Done Done Sent false
41992024-02-03T17:11:18.655Z 9 Acked 1008 Write 4 Done Done Sent false
42002024-02-03T17:11:18.655Z 10 Acked 1009 Write 2 Done Done Sent false
42012024-02-03T17:11:18.655Z 11 Acked 1010 Read 4 Done Done Sent false
42022024-02-03T17:11:18.655Z 12 Acked 1011 Read 3 Done Done Sent false
42032024-02-03T17:11:18.655Z 13 Acked 1012 Read 2 Done Done Sent false
42042024-02-03T17:11:18.655Z 14 Acked 1013 Write 7 Done Done Sent false
42052024-02-03T17:11:18.655Z 15 Acked 1014 Read 8 Done Done Sent false
42062024-02-03T17:11:18.655Z 16 Acked 1015 Read 9 Done Done Sent false
42072024-02-03T17:11:18.655Z 17 Acked 1016 Write 6 Done Done Sent false
42082024-02-03T17:11:18.655Z 18 Acked 1017 Write 8 Done Done Sent false
42092024-02-03T17:11:18.655Z 19 Acked 1018 Write 10 Done Done Sent false
42102024-02-03T17:11:18.655Z 20 Acked 1019 Read 10 Done Done Sent false
42112024-02-03T17:11:18.655Z 21 Acked 1020 Write 6 Done Done Sent false
42122024-02-03T17:11:18.655Z 22 Acked 1021 Read 3 Done Done Sent false
42132024-02-03T17:11:18.655Z 23 Acked 1022 Write 3 Done Done Sent false
42142024-02-03T17:11:18.655Z 24 Acked 1023 Write 2 Done Done Sent false
42152024-02-03T17:11:18.656Z 25 Acked 1024 Flush 0 Done Done Sent false
42162024-02-03T17:11:18.656Z 26 Acked 1025 Flush 0 Done Done Sent false
42172024-02-03T17:11:18.656Z 27 Acked 1026 Write 4 Done Done Sent false
42182024-02-03T17:11:18.656Z 28 Acked 1027 Read 10 Done Done Sent false
42192024-02-03T17:11:18.656Z 29 Acked 1028 Write 6 Sent Sent Sent false
42202024-02-03T17:11:18.656Z 30 Acked 1029 Write 3 Sent Sent Sent false
42212024-02-03T17:11:18.656Z STATES DS:0 DS:1 DS:2 TOTAL
42222024-02-03T17:11:18.656Z New 0 0 0 0
42232024-02-03T17:11:18.656Z Sent 2 2 28 32
42242024-02-03T17:11:18.656Z Done 28 28 2 58
42252024-02-03T17:11:18.656Z Skipped 0 0 0 0
42262024-02-03T17:11:18.656Z Error 0 0 0 0
42272024-02-03T17:11:18.656ZLast Flush: 1025 1025 0
42282024-02-03T17:11:18.656ZDownstairs last five completed:
42292024-02-03T17:11:18.656ZUpstairs last five completed: 30 29 28 27 26
42302024-02-03T17:11:18.656Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:11:18.194875715Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1281}
42312024-02-03T17:11:18.656ZWrote out file "/tmp/test_repair_verify.data"
42322024-02-03T17:11:18.659Z
42332024-02-03T17:11:18.663ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
42342024-02-03T17:11:18.686ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
42352024-02-03T17:11:18.686Z 0 000-019 38 38 36 66 66 63 T T F <---
42362024-02-03T17:11:18.689Z 1 020-039 36 36 36 63 63 63 F F F
42372024-02-03T17:11:18.689Z 2 040-059 38 38 33 66 66 57 F F F <---
42382024-02-03T17:11:18.689Z 3 060-079 38 38 34 66 66 58 F F F <---
42392024-02-03T17:11:18.689Z 4 080-099 34 34 34 59 59 59 F F F
42402024-02-03T17:11:18.689Z 5 100-119 37 37 37 64 64 64 F F F
42412024-02-03T17:11:18.689Z 6 120-139 36 36 36 63 63 63 F F F
42422024-02-03T17:11:18.689Z 7 140-159 32 32 32 56 56 56 F F F
42432024-02-03T17:11:18.689Z 8 160-179 36 36 36 63 63 63 F F F
42442024-02-03T17:11:18.689Z 9 180-199 34 34 34 58 58 58 F F F
42452024-02-03T17:11:18.689Z 10 200-219 29 29 29 53 53 53 F F F
42462024-02-03T17:11:18.690Z 11 220-239 36 36 36 63 63 63 F F F
42472024-02-03T17:11:18.690Z 12 240-259 38 38 34 66 66 59 F F F <---
42482024-02-03T17:11:18.690Z 13 260-279 38 38 35 66 66 61 F F F <---
42492024-02-03T17:11:18.690Z 14 280-299 36 36 36 63 63 63 F F F
42502024-02-03T17:11:18.690Z 15 300-319 38 38 36 66 66 63 F F F <---
42512024-02-03T17:11:18.690Z 16 320-339 38 38 36 65 65 63 F F F <---
42522024-02-03T17:11:18.690Z 17 340-359 34 34 34 58 58 58 F F F
42532024-02-03T17:11:18.690Z 18 360-379 34 34 34 58 58 58 F F F
42542024-02-03T17:11:18.690Z 19 380-399 38 38 36 65 65 63 F F T <---
42552024-02-03T17:11:18.690Z 20 400-419 38 38 26 66 66 45 F F F <---
42562024-02-03T17:11:18.690Z 21 420-439 38 38 30 66 66 54 F F F <---
42572024-02-03T17:11:18.690Z 22 440-459 34 34 34 58 58 58 T T F <---
42582024-02-03T17:11:18.690Z 23 460-479 34 34 34 59 59 59 F F F
42592024-02-03T17:11:18.690Z 24 480-499 34 34 34 59 59 59 F F F
42602024-02-03T17:11:18.690Z 25 500-519 36 36 36 63 63 63 F F F
42612024-02-03T17:11:18.690Z 26 520-539 38 38 30 66 66 54 F F F <---
42622024-02-03T17:11:18.690Z 27 540-559 38 38 37 65 65 64 F F T <---
42632024-02-03T17:11:18.690Z 28 560-579 37 37 37 64 64 64 F F F
42642024-02-03T17:11:18.690Z 29 580-599 34 34 34 59 59 59 T T F <---
42652024-02-03T17:11:18.690ZMax gen: 38, Max flush: 66
42662024-02-03T17:11:18.690ZError: Difference in extent metadata found!
42672024-02-03T17:11:18.694ZA Difference in extent metadata is expected here
42682024-02-03T17:11:18.694ZOn loop 19
42692024-02-03T17:11:18.695Z
42702024-02-03T17:11:18.697ZVerifying data now
42712024-02-03T17:11:23.901ZNo change after verify
42722024-02-03T17:11:23.901ZLoop: 19 Downstairs dump after verify (and repair):
42732024-02-03T17:11:23.925ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
42742024-02-03T17:11:23.925Z 0 000-019 39 39 39 67 67 67 F F F
42752024-02-03T17:11:23.926Z 1 020-039 36 36 36 63 63 63 F F F
42762024-02-03T17:11:23.926Z 2 040-059 38 38 38 66 66 66 F F F
42772024-02-03T17:11:23.926Z 3 060-079 38 38 38 66 66 66 F F F
42782024-02-03T17:11:23.929Z 4 080-099 34 34 34 59 59 59 F F F
42792024-02-03T17:11:23.929Z 5 100-119 37 37 37 64 64 64 F F F
42802024-02-03T17:11:23.929Z 6 120-139 36 36 36 63 63 63 F F F
42812024-02-03T17:11:23.929Z 7 140-159 32 32 32 56 56 56 F F F
42822024-02-03T17:11:23.929Z 8 160-179 36 36 36 63 63 63 F F F
42832024-02-03T17:11:23.929Z 9 180-199 34 34 34 58 58 58 F F F
42842024-02-03T17:11:23.929Z 10 200-219 29 29 29 53 53 53 F F F
42852024-02-03T17:11:23.929Z 11 220-239 36 36 36 63 63 63 F F F
42862024-02-03T17:11:23.930Z 12 240-259 38 38 38 66 66 66 F F F
42872024-02-03T17:11:23.930Z 13 260-279 38 38 38 66 66 66 F F F
42882024-02-03T17:11:23.930Z 14 280-299 36 36 36 63 63 63 F F F
42892024-02-03T17:11:23.930Z 15 300-319 38 38 38 66 66 66 F F F
42902024-02-03T17:11:23.930Z 16 320-339 38 38 38 65 65 65 F F F
42912024-02-03T17:11:23.930Z 17 340-359 34 34 34 58 58 58 F F F
42922024-02-03T17:11:23.930Z 18 360-379 34 34 34 58 58 58 F F F
42932024-02-03T17:11:23.930Z 19 380-399 38 38 38 65 65 65 F F F
42942024-02-03T17:11:23.930Z 20 400-419 38 38 38 66 66 66 F F F
42952024-02-03T17:11:23.930Z 21 420-439 38 38 38 66 66 66 F F F
42962024-02-03T17:11:23.930Z 22 440-459 39 39 39 67 67 67 F F F
42972024-02-03T17:11:23.930Z 23 460-479 34 34 34 59 59 59 F F F
42982024-02-03T17:11:23.930Z 24 480-499 34 34 34 59 59 59 F F F
42992024-02-03T17:11:23.930Z 25 500-519 36 36 36 63 63 63 F F F
43002024-02-03T17:11:23.930Z 26 520-539 38 38 38 66 66 66 F F F
43012024-02-03T17:11:23.931Z 27 540-559 38 38 38 65 65 65 F F F
43022024-02-03T17:11:23.931Z 28 560-579 37 37 37 64 64 64 F F F
43032024-02-03T17:11:23.931Z 29 580-599 39 39 39 67 67 67 F F F
43042024-02-03T17:11:23.931ZMax gen: 39, Max flush: 67
43052024-02-03T17:11:23.931Z
43062024-02-03T17:11:23.931ZBegin loop 20
43072024-02-03T17:11:23.931ZDownstairs to restart: 1
43082024-02-03T17:11:23.964Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.508026601Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289}
43092024-02-03T17:11:23.964Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.508481681Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289}
43102024-02-03T17:11:23.969Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.508510375Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289}
43112024-02-03T17:11:23.969Z{"msg":"Crucible 1292f6bc-a7fb-49ac-be7a-6db67f8aa657 has session id: feb0855a-159d-470d-a95f-96cc7bde1466","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.508572007Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43122024-02-03T17:11:23.969Z{"msg":"Upstairs opts: Upstairs UUID: 1292f6bc-a7fb-49ac-be7a-6db67f8aa657, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
43132024-02-03T17:11:23.970Z,"time":"2024-02-03T17:11:23.508633418Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43142024-02-03T17:11:23.970Z{"msg":"Crucible stats registered with UUID: 1292f6bc-a7fb-49ac-be7a-6db67f8aa657","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.508937053Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43152024-02-03T17:11:23.970Z{"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.509116776Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289}
43162024-02-03T17:11:23.970Z{"msg":"1292f6bc-a7fb-49ac-be7a-6db67f8aa657 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.509231866Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43172024-02-03T17:11:23.970Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.50939862Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43182024-02-03T17:11:23.970Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.509483448Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43192024-02-03T17:11:23.970Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.509531887Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43202024-02-03T17:11:23.970Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.509652197Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43212024-02-03T17:11:23.970Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.509718266Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43222024-02-03T17:11:23.970Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.509757707Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43232024-02-03T17:11:23.970Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.509823045Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"":"io task","client":"2","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43242024-02-03T17:11:23.970Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.509854125Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"":"io task","client":"0","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43252024-02-03T17:11:23.971Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.509921223Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"":"io task","client":"1","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43262024-02-03T17:11:23.971Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.509952482Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"":"io task","client":"0","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43272024-02-03T17:11:23.971Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.51013229Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"":"io task","client":"1","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43282024-02-03T17:11:23.971Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.510194676Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"":"io task","client":"2","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43292024-02-03T17:11:23.971Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.511049809Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43302024-02-03T17:11:23.971Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.511149652Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43312024-02-03T17:11:23.971Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.511405767Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43322024-02-03T17:11:23.971Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.511496033Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43332024-02-03T17:11:23.971Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.511716111Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43342024-02-03T17:11:23.971Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.511842886Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43352024-02-03T17:11:23.971Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.512271645Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43362024-02-03T17:11:23.971Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.512503672Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43372024-02-03T17:11:23.971Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.513179073Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43382024-02-03T17:11:23.971Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.513276648Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43392024-02-03T17:11:23.972Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.513323844Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43402024-02-03T17:11:23.972Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.513381378Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43412024-02-03T17:11:23.972Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.51341177Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43422024-02-03T17:11:23.972Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.513480141Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43432024-02-03T17:11:23.972Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.513507962Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43442024-02-03T17:11:23.972Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.514045257Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43452024-02-03T17:11:23.972Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.514124836Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43462024-02-03T17:11:23.972Z{"msg":"[0]R flush_numbers[0..12]: [67, 63, 66, 66, 59, 64, 63, 56, 63, 58, 53, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.514187789Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43472024-02-03T17:11:23.972Z{"msg":"[0]R generation[0..12]: [39, 36, 38, 38, 34, 37, 36, 32, 36, 34, 29, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.514214227Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43482024-02-03T17:11:23.972Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.514238371Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43492024-02-03T17:11:23.972Z{"msg":"[1]R flush_numbers[0..12]: [67, 63, 66, 66, 59, 64, 63, 56, 63, 58, 53, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.514261294Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43502024-02-03T17:11:23.972Z{"msg":"[1]R generation[0..12]: [39, 36, 38, 38, 34, 37, 36, 32, 36, 34, 29, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.514283923Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43512024-02-03T17:11:23.972Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.514307369Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43522024-02-03T17:11:23.973Z{"msg":"[2]R flush_numbers[0..12]: [67, 63, 66, 66, 59, 64, 63, 56, 63, 58, 53, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.514331969Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43532024-02-03T17:11:23.973Z{"msg":"[2]R generation[0..12]: [39, 36, 38, 38, 34, 37, 36, 32, 36, 34, 29, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.51435478Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43542024-02-03T17:11:23.973Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.514378193Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43552024-02-03T17:11:23.973Z{"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
43562024-02-03T17:11:23.973Z2024-02-03T17:11:23.514425878Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43572024-02-03T17:11:23.973Z{"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.514453249Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43582024-02-03T17:11:23.973Z{"msg":"Next flush: 68","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.514474306Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43592024-02-03T17:11:23.973Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.51449607Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43602024-02-03T17:11:23.973Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.514547263Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43612024-02-03T17:11:23.973Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.514569085Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43622024-02-03T17:11:23.973Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.514590686Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43632024-02-03T17:11:23.977Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.51461766Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43642024-02-03T17:11:23.977Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.514688167Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43652024-02-03T17:11:23.977Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.514714234Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43662024-02-03T17:11:23.978Z{"msg":"1292f6bc-a7fb-49ac-be7a-6db67f8aa657 is now active with session: feb0855a-159d-470d-a95f-96cc7bde1466","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.514742176Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43672024-02-03T17:11:23.978Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.51480746Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289,"session_id":"feb0855a-159d-470d-a95f-96cc7bde1466"}
43682024-02-03T17:11:23.978Z{"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:23.51483356Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289}
43692024-02-03T17:11:23.978ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
43702024-02-03T17:11:23.978ZLoading write count information from file "/tmp/test_repair_verify.data"
43712024-02-03T17:11:23.978ZRun Repair workload
43722024-02-03T17:11:23.978Z01/30 Write block 195 len 36864 data: 5 3 3 2 2 2 2 3 3
43732024-02-03T17:11:23.978Z02/30 Flush
43742024-02-03T17:11:23.978Z03/30 Write block 22 len 28672 data: 5 5 5 5 5 5 5
43752024-02-03T17:11:23.982Z04/30 Flush
43762024-02-03T17:11:23.982Z05/30 Flush
43772024-02-03T17:11:23.982Z06/30 Flush
43782024-02-03T17:11:23.985Z07/30 Write block 486 len 4096 data: 3
43792024-02-03T17:11:23.985Z08/30 Read block 315 len 20480
43802024-02-03T17:11:23.988Z09/30 Read block 557 len 28672
43812024-02-03T17:11:23.988Z10/30 Read block 180 len 28672
43822024-02-03T17:11:23.991Z11/30 Write block 579 len 20480 data: 4 4 4 3 3
43832024-02-03T17:11:23.992Z12/30 Write block 89 len 24576 data: 4 4 3 3 5 5
43842024-02-03T17:11:23.992Z13/30 Flush
43852024-02-03T17:11:23.995Z14/30 Write block 361 len 4096 data: 6
43862024-02-03T17:11:23.998Z15/30 Write block 198 len 4096 data: 3
43872024-02-03T17:11:23.998Z16/30 Write block 390 len 36864 data: 5 5 6 6 5 5 3 2 3
43882024-02-03T17:11:23.998Z17/30 Flush
43892024-02-03T17:11:24.001Z18/30 Write block 344 len 24576 data: 3 3 2 3 6 6
43902024-02-03T17:11:24.005Z19/30 Read block 573 len 24576
43912024-02-03T17:11:24.005Z20/30 Flush
43922024-02-03T17:11:24.009Z21/30 Write block 176 len 8192 data: 5 6
43932024-02-03T17:11:24.009Z22/30 Write block 147 len 16384 data: 3 4 3 2
43942024-02-03T17:11:24.012Z23/30 Read block 194 len 12288
43952024-02-03T17:11:24.013Z24/30 Write block 533 len 32768 data: 3 2 2 2 2 2 2 2
43962024-02-03T17:11:24.013Z25/30 Read block 490 len 12288
43972024-02-03T17:11:24.016Z26/30 Flush
43982024-02-03T17:11:24.019Z27/30 Write block 216 len 4096 data: 4
43992024-02-03T17:11:24.019Z28/30 Write block 156 len 4096 data: 4
44002024-02-03T17:11:24.023Z29/30 Write block 469 len 28672 data: 2 3 2 2 2 4 4
44012024-02-03T17:11:24.023Z30/30 Write block 530 len 16384 data: 5 3 3 4
44022024-02-03T17:11:24.024Z----------------------------------------------------------------
44032024-02-03T17:11:24.024Z Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:28
44042024-02-03T17:11:24.024ZStates: Active Active Active
44052024-02-03T17:11:24.024ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
44062024-02-03T17:11:24.024Z 3 Acked 1002 Write 7 Done Sent Done false
44072024-02-03T17:11:24.024Z 4 Acked 1003 Flush 0 Done Sent Done false
44082024-02-03T17:11:24.024Z 5 Acked 1004 Flush 0 Done Sent Done false
44092024-02-03T17:11:24.024Z 6 Acked 1005 Flush 0 Done Sent Done false
44102024-02-03T17:11:24.024Z 7 Acked 1006 Write 1 Done Sent Done false
44112024-02-03T17:11:24.024Z 8 Acked 1007 Read 5 Done Sent Done false
44122024-02-03T17:11:24.024Z 9 Acked 1008 Read 7 Done Sent Done false
44132024-02-03T17:11:24.024Z 10 Acked 1009 Read 7 Done Sent Done false
44142024-02-03T17:11:24.024Z 11 Acked 1010 Write 5 Done Sent Done false
44152024-02-03T17:11:24.025Z 12 Acked 1011 Write 6 Done Sent Done false
44162024-02-03T17:11:24.025Z 13 Acked 1012 Flush 0 Done Sent Done false
44172024-02-03T17:11:24.025Z 14 Acked 1013 Write 1 Done Sent Done false
44182024-02-03T17:11:24.025Z 15 Acked 1014 Write 1 Done Sent Done false
44192024-02-03T17:11:24.025Z 16 Acked 1015 Write 9 Done Sent Done false
44202024-02-03T17:11:24.025Z 17 Acked 1016 Flush 0 Done Sent Done false
44212024-02-03T17:11:24.025Z 18 Acked 1017 Write 6 Done Sent Done false
44222024-02-03T17:11:24.025Z 19 Acked 1018 Read 6 Done Sent Done false
44232024-02-03T17:11:24.025Z 20 Acked 1019 Flush 0 Done Sent Done false
44242024-02-03T17:11:24.025Z 21 Acked 1020 Write 2 Done Sent Done false
44252024-02-03T17:11:24.025Z 22 Acked 1021 Write 4 Done Sent Done false
44262024-02-03T17:11:24.025Z 23 Acked 1022 Read 3 Done Sent Done false
44272024-02-03T17:11:24.026Z 24 Acked 1023 Write 8 Done Sent Done false
44282024-02-03T17:11:24.026Z 25 Acked 1024 Read 3 Done Sent Done false
44292024-02-03T17:11:24.026Z 26 Acked 1025 Flush 0 Done Sent Done false
44302024-02-03T17:11:24.026Z 27 Acked 1026 Write 1 Done Sent Done false
44312024-02-03T17:11:24.026Z 28 Acked 1027 Write 1 Done Sent Done false
44322024-02-03T17:11:24.026Z 29 Acked 1028 Write 7 Done Sent Done false
44332024-02-03T17:11:24.026Z 30 Acked 1029 Write 4 Sent Sent Sent false
44342024-02-03T17:11:24.026Z STATES DS:0 DS:1 DS:2 TOTAL
44352024-02-03T17:11:24.026Z New 0 0 0 0
44362024-02-03T17:11:24.026Z Sent 1 28 1 30
44372024-02-03T17:11:24.026Z Done 27 0 27 54
44382024-02-03T17:11:24.026Z Skipped 0 0 0 0
44392024-02-03T17:11:24.026Z Error 0 0 0 0
44402024-02-03T17:11:24.026ZLast Flush: 1025 1001 1025
44412024-02-03T17:11:24.027ZDownstairs last five completed: 1001 1000
44422024-02-03T17:11:24.027ZUpstairs last five completed: 30 29 28 27 26
44432024-02-03T17:11:24.027Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:11:23.568989129Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1289}
44442024-02-03T17:11:24.027ZWrote out file "/tmp/test_repair_verify.data"
44452024-02-03T17:11:24.039Z
44462024-02-03T17:11:24.043ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
44472024-02-03T17:11:24.060ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
44482024-02-03T17:11:24.060Z 0 000-019 39 39 39 67 67 67 F F F
44492024-02-03T17:11:24.060Z 1 020-039 40 36 40 69 63 69 F F F <---
44502024-02-03T17:11:24.060Z 2 040-059 38 38 38 66 66 66 F F F
44512024-02-03T17:11:24.060Z 3 060-079 38 38 38 66 66 66 F F F
44522024-02-03T17:11:24.063Z 4 080-099 40 34 40 72 59 72 F F F <---
44532024-02-03T17:11:24.064Z 5 100-119 37 37 37 64 64 64 F F F
44542024-02-03T17:11:24.064Z 6 120-139 36 36 36 63 63 63 F F F
44552024-02-03T17:11:24.064Z 7 140-159 40 32 40 75 56 75 T F T <---
44562024-02-03T17:11:24.064Z 8 160-179 40 36 40 75 63 75 F F F <---
44572024-02-03T17:11:24.064Z 9 180-199 40 40 40 73 68 73 F F F <---
44582024-02-03T17:11:24.064Z 10 200-219 40 40 40 68 68 68 T F T <---
44592024-02-03T17:11:24.064Z 11 220-239 36 36 36 63 63 63 F F F
44602024-02-03T17:11:24.064Z 12 240-259 38 38 38 66 66 66 F F F
44612024-02-03T17:11:24.064Z 13 260-279 38 38 38 66 66 66 F F F
44622024-02-03T17:11:24.064Z 14 280-299 36 36 36 63 63 63 F F F
44632024-02-03T17:11:24.064Z 15 300-319 38 38 38 66 66 66 F F F
44642024-02-03T17:11:24.064Z 16 320-339 38 38 38 65 65 65 F F F
44652024-02-03T17:11:24.065Z 17 340-359 40 34 40 74 58 74 F F F <---
44662024-02-03T17:11:24.065Z 18 360-379 40 34 40 73 58 73 F F F <---
44672024-02-03T17:11:24.065Z 19 380-399 40 38 40 73 65 73 F F F <---
44682024-02-03T17:11:24.065Z 20 400-419 38 38 38 66 66 66 F F F
44692024-02-03T17:11:24.065Z 21 420-439 38 38 38 66 66 66 F F F
44702024-02-03T17:11:24.065Z 22 440-459 39 39 39 67 67 67 F F F
44712024-02-03T17:11:24.065Z 23 460-479 34 34 34 59 59 59 T F T <---
44722024-02-03T17:11:24.065Z 24 480-499 40 34 40 72 59 72 F F F <---
44732024-02-03T17:11:24.065Z 25 500-519 36 36 36 63 63 63 F F F
44742024-02-03T17:11:24.065Z 26 520-539 40 38 40 75 66 75 T F T <---
44752024-02-03T17:11:24.065Z 27 540-559 40 38 40 75 65 75 F F F <---
44762024-02-03T17:11:24.065Z 28 560-579 40 37 40 72 64 72 F F F <---
44772024-02-03T17:11:24.065Z 29 580-599 40 39 40 72 67 72 F F F <---
44782024-02-03T17:11:24.065ZMax gen: 40, Max flush: 75
44792024-02-03T17:11:24.066ZError: Difference in extent metadata found!
44802024-02-03T17:11:24.069ZA Difference in extent metadata is expected here
44812024-02-03T17:11:24.069ZOn loop 20
44822024-02-03T17:11:24.069Z
44832024-02-03T17:11:24.069ZVerifying data now
44842024-02-03T17:11:29.344ZNo change after verify
44852024-02-03T17:11:29.344ZLoop: 20 Downstairs dump after verify (and repair):
44862024-02-03T17:11:29.366ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
44872024-02-03T17:11:29.367Z 0 000-019 39 39 39 67 67 67 F F F
44882024-02-03T17:11:29.370Z 1 020-039 40 40 40 69 69 69 F F F
44892024-02-03T17:11:29.370Z 2 040-059 38 38 38 66 66 66 F F F
44902024-02-03T17:11:29.370Z 3 060-079 38 38 38 66 66 66 F F F
44912024-02-03T17:11:29.370Z 4 080-099 40 40 40 72 72 72 F F F
44922024-02-03T17:11:29.370Z 5 100-119 37 37 37 64 64 64 F F F
44932024-02-03T17:11:29.370Z 6 120-139 36 36 36 63 63 63 F F F
44942024-02-03T17:11:29.370Z 7 140-159 41 41 41 76 76 76 F F F
44952024-02-03T17:11:29.370Z 8 160-179 40 40 40 75 75 75 F F F
44962024-02-03T17:11:29.370Z 9 180-199 40 40 40 73 73 73 F F F
44972024-02-03T17:11:29.370Z 10 200-219 41 41 41 76 76 76 F F F
44982024-02-03T17:11:29.370Z 11 220-239 36 36 36 63 63 63 F F F
44992024-02-03T17:11:29.370Z 12 240-259 38 38 38 66 66 66 F F F
45002024-02-03T17:11:29.370Z 13 260-279 38 38 38 66 66 66 F F F
45012024-02-03T17:11:29.370Z 14 280-299 36 36 36 63 63 63 F F F
45022024-02-03T17:11:29.370Z 15 300-319 38 38 38 66 66 66 F F F
45032024-02-03T17:11:29.370Z 16 320-339 38 38 38 65 65 65 F F F
45042024-02-03T17:11:29.370Z 17 340-359 40 40 40 74 74 74 F F F
45052024-02-03T17:11:29.370Z 18 360-379 40 40 40 73 73 73 F F F
45062024-02-03T17:11:29.370Z 19 380-399 40 40 40 73 73 73 F F F
45072024-02-03T17:11:29.371Z 20 400-419 38 38 38 66 66 66 F F F
45082024-02-03T17:11:29.371Z 21 420-439 38 38 38 66 66 66 F F F
45092024-02-03T17:11:29.371Z 22 440-459 39 39 39 67 67 67 F F F
45102024-02-03T17:11:29.371Z 23 460-479 41 41 41 76 76 76 F F F
45112024-02-03T17:11:29.371Z 24 480-499 40 40 40 72 72 72 F F F
45122024-02-03T17:11:29.371Z 25 500-519 36 36 36 63 63 63 F F F
45132024-02-03T17:11:29.371Z 26 520-539 41 41 41 76 76 76 F F F
45142024-02-03T17:11:29.371Z 27 540-559 40 40 40 75 75 75 F F F
45152024-02-03T17:11:29.371Z 28 560-579 40 40 40 72 72 72 F F F
45162024-02-03T17:11:29.371Z 29 580-599 40 40 40 72 72 72 F F F
45172024-02-03T17:11:29.371ZMax gen: 41, Max flush: 76
45182024-02-03T17:11:29.374Z
45192024-02-03T17:11:29.374ZBegin loop 21
45202024-02-03T17:11:29.374ZDownstairs to restart: 0
45212024-02-03T17:11:29.408Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.952205275Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297}
45222024-02-03T17:11:29.408Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.952678684Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297}
45232024-02-03T17:11:29.408Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.952709392Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297}
45242024-02-03T17:11:29.408Z{"msg":"Crucible a5f49c7e-f2c1-4a71-91cf-3cff4a4be385 has session id: cbeaebb0-8f3a-4424-b079-2567e2418b57","v":0,"name":"crucible","level":30,"timeCrucible runtime is spawned
45252024-02-03T17:11:29.413Z":"2024-02-03T17:11:28.952785175Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45262024-02-03T17:11:29.413Z{"msg":"Upstairs opts: Upstairs UUID: a5f49c7e-f2c1-4a71-91cf-3cff4a4be385, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.952955421Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45272024-02-03T17:11:29.413Z{"msg":"Crucible stats registered with UUID: a5f49c7e-f2c1-4a71-91cf-3cff4a4be385","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.953008567Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45282024-02-03T17:11:29.413Z{"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.953050073Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297}
45292024-02-03T17:11:29.413Z{"msg":"a5f49c7e-f2c1-4a71-91cf-3cff4a4be385 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.953172507Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45302024-02-03T17:11:29.413Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.953270988Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45312024-02-03T17:11:29.413Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.953319342Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45322024-02-03T17:11:29.413Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.953355426Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45332024-02-03T17:11:29.414Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.953393438Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45342024-02-03T17:11:29.414Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.953437342Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45352024-02-03T17:11:29.414Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.953689325Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45362024-02-03T17:11:29.414Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.953781645Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"":"io task","client":"2","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45372024-02-03T17:11:29.414Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.953828106Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"":"io task","client":"0","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45382024-02-03T17:11:29.414Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.953869315Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"":"io task","client":"2","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45392024-02-03T17:11:29.414Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.953947208Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"":"io task","client":"0","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45402024-02-03T17:11:29.414Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.954055554Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"":"io task","client":"1","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45412024-02-03T17:11:29.414Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.954098202Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"":"io task","client":"1","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45422024-02-03T17:11:29.414Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.954723339Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45432024-02-03T17:11:29.414Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.954850964Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45442024-02-03T17:11:29.414Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.954898238Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45452024-02-03T17:11:29.414Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.955116769Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45462024-02-03T17:11:29.414Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.955865337Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45472024-02-03T17:11:29.414Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.955934357Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45482024-02-03T17:11:29.414Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.955962151Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45492024-02-03T17:11:29.414Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.956124233Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45502024-02-03T17:11:29.414Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.956930381Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45512024-02-03T17:11:29.414Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.957060672Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45522024-02-03T17:11:29.414Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.95710794Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45532024-02-03T17:11:29.414Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.957223504Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45542024-02-03T17:11:29.414Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.957291291Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45552024-02-03T17:11:29.414Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.957335019Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45562024-02-03T17:11:29.414Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.957441237Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45572024-02-03T17:11:29.415Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.958099727Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45582024-02-03T17:11:29.415Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.958223159Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45592024-02-03T17:11:29.415Z{"msg":"[0]R flush_numbers[0..12]: [67, 69, 66, 66, 72, 64, 63, 76, 75, 73, 76, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.958268579Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45602024-02-03T17:11:29.415Z{"msg":"[0]R generation[0..12]: [39, 40, 38, 38, 40, 37, 36, 41, 40, 40, 41, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.958347688Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45612024-02-03T17:11:29.415Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.958396046Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45622024-02-03T17:11:29.415Z{"msg":"Wait for a query_work_queue command to finish before sending IO
45632024-02-03T17:11:29.415Z[1]R flush_numbers[0..12]: [67, 69, 66, 66, 72, 64, 63, 76, 75, 73, 76, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.958577908Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45642024-02-03T17:11:29.415Z{"msg":"[1]R generation[0..12]: [39, 40, 38, 38, 40, 37, 36, 41, 40, 40, 41, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.958662524Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45652024-02-03T17:11:29.415Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.958748866Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45662024-02-03T17:11:29.415Z{"msg":"[2]R flush_numbers[0..12]: [67, 69, 66, 66, 72, 64, 63, 76, 75, 73, 76, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.958831644Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45672024-02-03T17:11:29.415Z{"msg":"[2]R generation[0..12]: [39, 40, 38, 38, 40, 37, 36, 41, 40, 40, 41, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.9589128Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45682024-02-03T17:11:29.415Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.958962321Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45692024-02-03T17:11:29.415Z{"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.959046182Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45702024-02-03T17:11:29.419Z{"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.959080099Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45712024-02-03T17:11:29.419Z{"msg":"Next flush: 77","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.959144714Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45722024-02-03T17:11:29.419Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.959187609Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45732024-02-03T17:11:29.419Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.959269399Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45742024-02-03T17:11:29.419Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.959311449Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45752024-02-03T17:11:29.419Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.95938838Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45762024-02-03T17:11:29.419Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.95942055Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45772024-02-03T17:11:29.419Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.959443317Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45782024-02-03T17:11:29.419Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.959491593Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45792024-02-03T17:11:29.419Z{"msg":"a5f49c7e-f2c1-4a71-91cf-3cff4a4be385 is now active with session: cbeaebb0-8f3a-4424-b079-2567e2418b57","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.959515836Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45802024-02-03T17:11:29.419Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.959535704Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297,"session_id":"cbeaebb0-8f3a-4424-b079-2567e2418b57"}
45812024-02-03T17:11:29.419ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
45822024-02-03T17:11:29.419Z{"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:28.959561309Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297}
45832024-02-03T17:11:29.419ZLoading write count information from file "/tmp/test_repair_verify.data"
45842024-02-03T17:11:29.419ZRun Repair workload
45852024-02-03T17:11:29.419Z01/30 Write block 417 len 20480 data: 6 6 4 4 4
45862024-02-03T17:11:29.419Z02/30 Read block 393 len 16384
45872024-02-03T17:11:29.420Z03/30 Write block 393 len 8192 data: 7 6
45882024-02-03T17:11:29.420Z04/30 Read block 181 len 36864
45892024-02-03T17:11:29.422Z05/30 Flush
45902024-02-03T17:11:29.425Z06/30 Write block 550 len 32768 data: 5 4 4 6 5 4 3 3
45912024-02-03T17:11:29.428Z07/30 Read block 246 len 16384
45922024-02-03T17:11:29.428Z08/30 Write block 106 len 8192 data: 6 6
45932024-02-03T17:11:29.428Z09/30 Read block 102 len 8192
45942024-02-03T17:11:29.431Z10/30 Write block 129 len 24576 data: 3 4 3 3 3 3
45952024-02-03T17:11:29.431Z11/30 Read block 501 len 20480
45962024-02-03T17:11:29.434Z12/30 Flush
45972024-02-03T17:11:29.437Z13/30 Write block 36 len 16384 data: 6 6 6 6
45982024-02-03T17:11:29.437Z14/30 Read block 464 len 12288
45992024-02-03T17:11:29.440Z15/30 Write block 334 len 36864 data: 6 4 4 4 5 4 5 4 4
46002024-02-03T17:11:29.443Z16/30 Write block 160 len 32768 data: 4 4 5 5 5 5 5 5
46012024-02-03T17:11:29.443Z17/30 Write block 412 len 4096 data: 3
46022024-02-03T17:11:29.443Z18/30 Write block 541 len 28672 data: 3 3 3 3 4 5 7
46032024-02-03T17:11:29.446Z19/30 Read block 190 len 36864
46042024-02-03T17:11:29.446Z20/30 Read block 64 len 8192
46052024-02-03T17:11:29.449Z21/30 Read block 592 len 12288
46062024-02-03T17:11:29.449Z22/30 Read block 563 len 4096
46072024-02-03T17:11:29.449Z23/30 Read block 285 len 36864
46082024-02-03T17:11:29.452Z24/30 Write block 432 len 16384 data: 4 4 4 5
46092024-02-03T17:11:29.452Z25/30 Read block 180 len 32768
46102024-02-03T17:11:29.455Z26/30 Write block 463 len 28672 data: 3 3 4 4 3 2 3
46112024-02-03T17:11:29.457Z27/30 Write block 90 len 32768 data: 5 4 4 6 6 4 4 5
46122024-02-03T17:11:29.457Z28/30 Write block 21 len 28672 data: 4 6 6 6 6 6 6
46132024-02-03T17:11:29.460Z29/30 Read block 436 len 32768
46142024-02-03T17:11:29.460Z30/30 Read block 111 len 16384
46152024-02-03T17:11:29.466Z----------------------------------------------------------------
46162024-02-03T17:11:29.466Z Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30
46172024-02-03T17:11:29.466ZStates: Active Active Active
46182024-02-03T17:11:29.466ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
46192024-02-03T17:11:29.466Z 1 Acked 1000 Write 5 Done Done Done false
46202024-02-03T17:11:29.467Z 2 Acked 1001 Read 4 Done Done Done false
46212024-02-03T17:11:29.467Z 3 Acked 1002 Write 2 Done Done Done false
46222024-02-03T17:11:29.467Z 4 Acked 1003 Read 9 Sent Done Done false
46232024-02-03T17:11:29.467Z 5 Acked 1004 Flush 0 Sent Done Done false
46242024-02-03T17:11:29.467Z 6 Acked 1005 Write 8 Sent Done Done false
46252024-02-03T17:11:29.467Z 7 Acked 1006 Read 4 Sent Done Done false
46262024-02-03T17:11:29.467Z 8 Acked 1007 Write 2 Sent Done Done false
46272024-02-03T17:11:29.467Z 9 Acked 1008 Read 2 Sent Done Done false
46282024-02-03T17:11:29.467Z 10 Acked 1009 Write 6 Sent Done Done false
46292024-02-03T17:11:29.467Z 11 Acked 1010 Read 5 Sent Done Done false
46302024-02-03T17:11:29.467Z 12 Acked 1011 Flush 0 Sent Done Done false
46312024-02-03T17:11:29.467Z 13 Acked 1012 Write 4 Sent Done Done false
46322024-02-03T17:11:29.467Z 14 Acked 1013 Read 3 Sent Done Done false
46332024-02-03T17:11:29.467Z 15 Acked 1014 Write 9 Sent Done Done false
46342024-02-03T17:11:29.467Z 16 Acked 1015 Write 8 Sent Done Done false
46352024-02-03T17:11:29.467Z 17 Acked 1016 Write 1 Sent Done Done false
46362024-02-03T17:11:29.467Z 18 Acked 1017 Write 7 Sent Done Done false
46372024-02-03T17:11:29.467Z 19 Acked 1018 Read 9 Sent Done Done false
46382024-02-03T17:11:29.467Z 20 Acked 1019 Read 2 Sent Done Done false
46392024-02-03T17:11:29.467Z 21 Acked 1020 Read 3 Sent Done Done false
46402024-02-03T17:11:29.467Z 22 Acked 1021 Read 1 Sent Done Done false
46412024-02-03T17:11:29.467Z 23 Acked 1022 Read 9 Sent Done Done false
46422024-02-03T17:11:29.467Z 24 Acked 1023 Write 4 Sent Done Done false
46432024-02-03T17:11:29.468Z 25 Acked 1024 Read 8 Sent Done Done false
46442024-02-03T17:11:29.468Z 26 Acked 1025 Write 7 Sent Done Done false
46452024-02-03T17:11:29.468Z 27 Acked 1026 Write 8 Sent Done Done false
46462024-02-03T17:11:29.468Z 28 Acked 1027 Write 7 Sent Done Done false
46472024-02-03T17:11:29.468Z 29 Acked 1028 Read 8 Sent Done Done false
46482024-02-03T17:11:29.468Z 30 Acked 1029 Read 4 Sent Sent Done false
46492024-02-03T17:11:29.468Z STATES DS:0 DS:1 DS:2 TOTAL
46502024-02-03T17:11:29.468Z New 0 0 0 0
46512024-02-03T17:11:29.468Z Sent 27 1 0 28
46522024-02-03T17:11:29.468Z Done 3 29 30 62
46532024-02-03T17:11:29.468Z Skipped 0 0 0 0
46542024-02-03T17:11:29.468Z Error 0 0 0 0
46552024-02-03T17:11:29.468ZLast Flush: 0 1011 1011
46562024-02-03T17:11:29.468ZDownstairs last five completed:
46572024-02-03T17:11:29.468ZUpstairs last five completed: 30 29 28 27 26
46582024-02-03T17:11:29.468Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:11:29.009063569Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1297}
46592024-02-03T17:11:29.468ZWrote out file "/tmp/test_repair_verify.data"
46602024-02-03T17:11:29.472Z
46612024-02-03T17:11:29.475ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
46622024-02-03T17:11:29.504ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
46632024-02-03T17:11:29.504Z 0 000-019 39 39 39 67 67 67 F F F
46642024-02-03T17:11:29.504Z 1 020-039 40 40 40 69 69 69 F T T <---
46652024-02-03T17:11:29.504Z 2 040-059 38 38 38 66 66 66 F F F
46662024-02-03T17:11:29.504Z 3 060-079 38 38 38 66 66 66 F F F
46672024-02-03T17:11:29.504Z 4 080-099 40 40 40 72 72 72 F T T <---
46682024-02-03T17:11:29.508Z 5 100-119 37 42 42 64 78 78 F F F <---
46692024-02-03T17:11:29.508Z 6 120-139 36 42 42 63 78 78 F F F <---
46702024-02-03T17:11:29.508Z 7 140-159 41 41 41 76 76 76 F F F
46712024-02-03T17:11:29.508Z 8 160-179 40 40 40 75 75 75 F T T <---
46722024-02-03T17:11:29.508Z 9 180-199 40 40 40 73 73 73 F F F
46732024-02-03T17:11:29.508Z 10 200-219 41 41 41 76 76 76 F F F
46742024-02-03T17:11:29.508Z 11 220-239 36 36 36 63 63 63 F F F
46752024-02-03T17:11:29.508Z 12 240-259 38 38 38 66 66 66 F F F
46762024-02-03T17:11:29.508Z 13 260-279 38 38 38 66 66 66 F F F
46772024-02-03T17:11:29.508Z 14 280-299 36 36 36 63 63 63 F F F
46782024-02-03T17:11:29.508Z 15 300-319 38 38 38 66 66 66 F F F
46792024-02-03T17:11:29.508Z 16 320-339 38 38 38 65 65 65 F T T <---
46802024-02-03T17:11:29.508Z 17 340-359 40 40 40 74 74 74 F T T <---
46812024-02-03T17:11:29.508Z 18 360-379 40 40 40 73 73 73 F F F
46822024-02-03T17:11:29.508Z 19 380-399 40 42 42 73 77 77 T F F <---
46832024-02-03T17:11:29.508Z 20 400-419 38 42 42 66 77 77 T T T <---
46842024-02-03T17:11:29.508Z 21 420-439 38 42 42 66 77 77 T T T <---
46852024-02-03T17:11:29.508Z 22 440-459 39 39 39 67 67 67 F F F
46862024-02-03T17:11:29.508Z 23 460-479 41 41 41 76 76 76 F T T <---
46872024-02-03T17:11:29.508Z 24 480-499 40 40 40 72 72 72 F F F
46882024-02-03T17:11:29.508Z 25 500-519 36 36 36 63 63 63 F F F
46892024-02-03T17:11:29.508Z 26 520-539 41 41 41 76 76 76 F F F
46902024-02-03T17:11:29.508Z 27 540-559 40 42 42 75 78 78 F T T <---
46912024-02-03T17:11:29.509Z 28 560-579 40 40 40 72 72 72 F F F
46922024-02-03T17:11:29.509Z 29 580-599 40 40 40 72 72 72 F F F
46932024-02-03T17:11:29.509ZMax gen: 42, Max flush: 78
46942024-02-03T17:11:29.509ZError: Difference in extent metadata found!
46952024-02-03T17:11:29.512ZA Difference in extent metadata is expected here
46962024-02-03T17:11:29.512ZOn loop 21
46972024-02-03T17:11:29.512Z
46982024-02-03T17:11:29.514ZVerifying data now
46992024-02-03T17:11:35.601ZNo change after verify
47002024-02-03T17:11:35.601ZLoop: 21 Downstairs dump after verify (and repair):
47012024-02-03T17:11:35.625ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
47022024-02-03T17:11:35.625Z 0 000-019 39 39 39 67 67 67 F F F
47032024-02-03T17:11:35.629Z 1 020-039 43 43 43 79 79 79 F F F
47042024-02-03T17:11:35.629Z 2 040-059 38 38 38 66 66 66 F F F
47052024-02-03T17:11:35.629Z 3 060-079 38 38 38 66 66 66 F F F
47062024-02-03T17:11:35.629Z 4 080-099 43 43 43 79 79 79 F F F
47072024-02-03T17:11:35.629Z 5 100-119 42 42 42 78 78 78 F F F
47082024-02-03T17:11:35.629Z 6 120-139 42 42 42 78 78 78 F F F
47092024-02-03T17:11:35.629Z 7 140-159 41 41 41 76 76 76 F F F
47102024-02-03T17:11:35.629Z 8 160-179 43 43 43 79 79 79 F F F
47112024-02-03T17:11:35.629Z 9 180-199 40 40 40 73 73 73 F F F
47122024-02-03T17:11:35.629Z 10 200-219 41 41 41 76 76 76 F F F
47132024-02-03T17:11:35.629Z 11 220-239 36 36 36 63 63 63 F F F
47142024-02-03T17:11:35.629Z 12 240-259 38 38 38 66 66 66 F F F
47152024-02-03T17:11:35.629Z 13 260-279 38 38 38 66 66 66 F F F
47162024-02-03T17:11:35.629Z 14 280-299 36 36 36 63 63 63 F F F
47172024-02-03T17:11:35.629Z 15 300-319 38 38 38 66 66 66 F F F
47182024-02-03T17:11:35.629Z 16 320-339 43 43 43 79 79 79 F F F
47192024-02-03T17:11:35.629Z 17 340-359 43 43 43 79 79 79 F F F
47202024-02-03T17:11:35.629Z 18 360-379 40 40 40 73 73 73 F F F
47212024-02-03T17:11:35.629Z 19 380-399 42 42 42 77 77 77 F F F
47222024-02-03T17:11:35.629Z 20 400-419 43 43 43 79 79 79 F F F
47232024-02-03T17:11:35.629Z 21 420-439 43 43 43 79 79 79 F F F
47242024-02-03T17:11:35.630Z 22 440-459 39 39 39 67 67 67 F F F
47252024-02-03T17:11:35.630Z 23 460-479 43 43 43 79 79 79 F F F
47262024-02-03T17:11:35.630Z 24 480-499 40 40 40 72 72 72 F F F
47272024-02-03T17:11:35.630Z 25 500-519 36 36 36 63 63 63 F F F
47282024-02-03T17:11:35.630Z 26 520-539 41 41 41 76 76 76 F F F
47292024-02-03T17:11:35.630Z 27 540-559 43 43 43 79 79 79 F F F
47302024-02-03T17:11:35.630Z 28 560-579 40 40 40 72 72 72 F F F
47312024-02-03T17:11:35.630Z 29 580-599 40 40 40 72 72 72 F F F
47322024-02-03T17:11:35.630ZMax gen: 43, Max flush: 79
47332024-02-03T17:11:35.633Z
47342024-02-03T17:11:35.633ZBegin loop 22
47352024-02-03T17:11:35.633ZDownstairs to restart: 0
47362024-02-03T17:11:35.661Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.208098716Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305}
47372024-02-03T17:11:35.661Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.208598161Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305}
47382024-02-03T17:11:35.661Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.208626982Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305}
47392024-02-03T17:11:35.665Z{"msg":"Crucible 25179267-5b45-4743-b50b-7095bcce9c0e has session id: 5de70e90-7124-4b56-9b27-a7b9dc1fca1c","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.208701196Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47402024-02-03T17:11:35.665Z{"msg":"Upstairs opts: Upstairs UUID: 25179267-5b45-4743-b50b-7095bcce9c0e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
47412024-02-03T17:11:35.665Z,"time":"2024-02-03T17:11:35.208792929Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47422024-02-03T17:11:35.665Z{"msg":"Crucible stats registered with UUID: 25179267-5b45-4743-b50b-7095bcce9c0e","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.208850647Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47432024-02-03T17:11:35.665Z{"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.208956781Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305}
47442024-02-03T17:11:35.665Z{"msg":"25179267-5b45-4743-b50b-7095bcce9c0e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.209327278Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47452024-02-03T17:11:35.665Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.209415198Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47462024-02-03T17:11:35.665Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.209503321Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47472024-02-03T17:11:35.665Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.209549249Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47482024-02-03T17:11:35.665Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.209634636Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47492024-02-03T17:11:35.665Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.209679332Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47502024-02-03T17:11:35.665Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.209783701Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47512024-02-03T17:11:35.665Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.209830272Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"":"io task","client":"2","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47522024-02-03T17:11:35.665Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.209874038Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"":"io task","client":"0","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47532024-02-03T17:11:35.665Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.209919572Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"":"io task","client":"1","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47542024-02-03T17:11:35.665Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.209959372Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"":"io task","client":"2","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47552024-02-03T17:11:35.665Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.210046314Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"":"io task","client":"0","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47562024-02-03T17:11:35.665Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.210178135Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"":"io task","client":"1","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47572024-02-03T17:11:35.666Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.211197239Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47582024-02-03T17:11:35.666Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.211326995Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47592024-02-03T17:11:35.666Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.211559054Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47602024-02-03T17:11:35.666Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.211656697Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47612024-02-03T17:11:35.670Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.212371995Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47622024-02-03T17:11:35.670Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.212596189Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47632024-02-03T17:11:35.670Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.212717758Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47642024-02-03T17:11:35.670Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.212812924Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47652024-02-03T17:11:35.670Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.213569471Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47662024-02-03T17:11:35.670Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.213644696Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47672024-02-03T17:11:35.670Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.213672359Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47682024-02-03T17:11:35.670Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.213857126Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47692024-02-03T17:11:35.670Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.213918144Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47702024-02-03T17:11:35.670Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.213943856Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47712024-02-03T17:11:35.670Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.214307789Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47722024-02-03T17:11:35.670Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.215100232Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47732024-02-03T17:11:35.670Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.215169198Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47742024-02-03T17:11:35.670Z{"msg":"[0]R flush_numbers[0..12]: [67, 79, 66, 66, 79, 78, 78, 76, 79, 73, 76, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.215196441Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47752024-02-03T17:11:35.670Z{"msg":"[0]R generation[0..12]: [39, 43, 38, 38, 43, 42, 42, 41, 43, 40, 41, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.215221379Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47762024-02-03T17:11:35.671Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.21524629Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47772024-02-03T17:11:35.671Z{"msg":"[1]R flush_numbers[0..12]: [67, 79, 66, 66, 79, 78, 78, 76, 79, 73, 76, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.215269891Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47782024-02-03T17:11:35.671Z{"msg":"[1]R generation[0..12]: [39, 43, 38, 38, 43, 42, 42, 41, 43, 40, 41, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.215321125Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47792024-02-03T17:11:35.671Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,Wait for a query_work_queue command to finish before sending IO
47802024-02-03T17:11:35.671Z"time":"2024-02-03T17:11:35.215363767Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47812024-02-03T17:11:35.671Z{"msg":"[2]R flush_numbers[0..12]: [67, 79, 66, 66, 79, 78, 78, 76, 79, 73, 76, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.215525983Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47822024-02-03T17:11:35.671Z{"msg":"[2]R generation[0..12]: [39, 43, 38, 38, 43, 42, 42, 41, 43, 40, 41, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.215589576Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47832024-02-03T17:11:35.671Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.215616464Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47842024-02-03T17:11:35.671Z{"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.215638228Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47852024-02-03T17:11:35.671Z{"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.215706781Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47862024-02-03T17:11:35.671Z{"msg":"Next flush: 80","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.215776591Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47872024-02-03T17:11:35.671Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.215805132Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47882024-02-03T17:11:35.671Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.215884244Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47892024-02-03T17:11:35.671Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.215954684Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47902024-02-03T17:11:35.671Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.215981556Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47912024-02-03T17:11:35.671Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.216032453Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47922024-02-03T17:11:35.671Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.216074023Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47932024-02-03T17:11:35.671Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.216143936Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47942024-02-03T17:11:35.671Z{"msg":"25179267-5b45-4743-b50b-7095bcce9c0e is now active with session: 5de70e90-7124-4b56-9b27-a7b9dc1fca1c","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.216170686Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47952024-02-03T17:11:35.671Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.216235972Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305,"session_id":"5de70e90-7124-4b56-9b27-a7b9dc1fca1c"}
47962024-02-03T17:11:35.671Z{"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:35.216267126Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305}
47972024-02-03T17:11:35.671ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
47982024-02-03T17:11:35.678ZLoading write count information from file "/tmp/test_repair_verify.data"
47992024-02-03T17:11:35.678ZRun Repair workload
48002024-02-03T17:11:35.678Z01/30 Write block 185 len 20480 data: 7 5 6 8 8
48012024-02-03T17:11:35.678Z02/30 Read block 40 len 12288
48022024-02-03T17:11:35.678Z03/30 Write block 160 len 32768 data: 5 5 6 6 6 6 6 6
48032024-02-03T17:11:35.681Z04/30 Write block 28 len 28672 data: 6 4 5 5 5 4 5
48042024-02-03T17:11:35.681Z05/30 Write block 339 len 8192 data: 5 6
48052024-02-03T17:11:35.681Z06/30 Write block 186 len 24576 data: 6 7 9 9 10 9
48062024-02-03T17:11:35.681Z07/30 Read block 140 len 32768
48072024-02-03T17:11:35.684Z08/30 Read block 440 len 16384
48082024-02-03T17:11:35.684Z09/30 Write block 166 len 8192 data: 7 7
48092024-02-03T17:11:35.684Z10/30 Read block 214 len 12288
48102024-02-03T17:11:35.687Z11/30 Read block 403 len 4096
48112024-02-03T17:11:35.687Z12/30 Flush
48122024-02-03T17:11:35.690Z13/30 Write block 310 len 8192 data: 6 5
48132024-02-03T17:11:35.693Z14/30 Write block 282 len 16384 data: 6 8 6 6
48142024-02-03T17:11:35.693Z15/30 Write block 592 len 24576 data: 4 4 4 4 5 3
48152024-02-03T17:11:35.693Z16/30 Read block 116 len 36864
48162024-02-03T17:11:35.697Z17/30 Write block 407 len 12288 data: 3 3 3
48172024-02-03T17:11:35.697Z18/30 Read block 211 len 20480
48182024-02-03T17:11:35.699Z19/30 Read block 472 len 20480
48192024-02-03T17:11:35.699Z20/30 Read block 146 len 36864
48202024-02-03T17:11:35.703Z21/30 Write block 509 len 12288 data: 6 6 7
48212024-02-03T17:11:35.703Z22/30 Write block 447 len 32768 data: 4 4 3 3 3 3 3 3
48222024-02-03T17:11:35.706Z23/30 Write block 228 len 4096 data: 3
48232024-02-03T17:11:35.706Z24/30 Read block 292 len 4096
48242024-02-03T17:11:35.706Z25/30 Read block 182 len 12288
48252024-02-03T17:11:35.709Z26/30 Read block 293 len 4096
48262024-02-03T17:11:35.709Z27/30 Read block 168 len 16384
48272024-02-03T17:11:35.712Z28/30 Read block 3 len 40960
48282024-02-03T17:11:35.712Z29/30 Read block 157 len 12288
48292024-02-03T17:11:35.715Z30/30 Read block 502 len 8192
48302024-02-03T17:11:35.715Z----------------------------------------------------------------
48312024-02-03T17:11:35.715Z Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:30
48322024-02-03T17:11:35.715ZStates: Active Active Active
48332024-02-03T17:11:35.715ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
48342024-02-03T17:11:35.716Z 1 Acked 1000 Write 5 Done Done Done false
48352024-02-03T17:11:35.716Z 2 Acked 1001 Read 3 Done Done Done false
48362024-02-03T17:11:35.716Z 3 Acked 1002 Write 8 Done Done Done false
48372024-02-03T17:11:35.716Z 4 Acked 1003 Write 7 Done Done Done false
48382024-02-03T17:11:35.716Z 5 Acked 1004 Write 2 Done Done Done false
48392024-02-03T17:11:35.716Z 6 Acked 1005 Write 6 Done Done Done false
48402024-02-03T17:11:35.716Z 7 Acked 1006 Read 8 Done Done Done false
48412024-02-03T17:11:35.716Z 8 Acked 1007 Read 4 Done Done Done false
48422024-02-03T17:11:35.716Z 9 Acked 1008 Write 2 Sent Done Done false
48432024-02-03T17:11:35.716Z 10 Acked 1009 Read 3 Sent Done Done false
48442024-02-03T17:11:35.716Z 11 Acked 1010 Read 1 Sent Done Done false
48452024-02-03T17:11:35.716Z 12 Acked 1011 Flush 0 Sent Done Done false
48462024-02-03T17:11:35.716Z 13 Acked 1012 Write 2 Sent Done Done false
48472024-02-03T17:11:35.716Z 14 Acked 1013 Write 4 Sent Done Done false
48482024-02-03T17:11:35.716Z 15 Acked 1014 Write 6 Sent Done Done false
48492024-02-03T17:11:35.716Z 16 Acked 1015 Read 9 Sent Done Done false
48502024-02-03T17:11:35.716Z 17 Acked 1016 Write 3 Sent Done Done false
48512024-02-03T17:11:35.716Z 18 Acked 1017 Read 5 Sent Done Done false
48522024-02-03T17:11:35.716Z 19 Acked 1018 Read 5 Sent Done Done false
48532024-02-03T17:11:35.716Z 20 Acked 1019 Read 9 Sent Done Done false
48542024-02-03T17:11:35.716Z 21 Acked 1020 Write 3 Sent Done Done false
48552024-02-03T17:11:35.716Z 22 Acked 1021 Write 8 Sent Done Done false
48562024-02-03T17:11:35.716Z 23 Acked 1022 Write 1 Sent Done Done false
48572024-02-03T17:11:35.716Z 24 Acked 1023 Read 1 Sent Done Done false
48582024-02-03T17:11:35.717Z 25 Acked 1024 Read 3 Sent Done Done false
48592024-02-03T17:11:35.717Z 26 Acked 1025 Read 1 Sent Done Done false
48602024-02-03T17:11:35.717Z 27 Acked 1026 Read 4 Sent Done Done false
48612024-02-03T17:11:35.717Z 28 Acked 1027 Read 10 Sent Done Done false
48622024-02-03T17:11:35.717Z 29 Acked 1028 Read 3 Sent Done Done false
48632024-02-03T17:11:35.717Z 30 Acked 1029 Read 2 Sent Sent Done false
48642024-02-03T17:11:35.717Z STATES DS:0 DS:1 DS:2 TOTAL
48652024-02-03T17:11:35.717Z New 0 0 0 0
48662024-02-03T17:11:35.717Z Sent 22 1 0 23
48672024-02-03T17:11:35.717Z Done 8 29 30 67
48682024-02-03T17:11:35.717Z Skipped 0 0 0 0
48692024-02-03T17:11:35.717Z Error 0 0 0 0
48702024-02-03T17:11:35.717ZLast Flush: 0 1011 1011
48712024-02-03T17:11:35.717ZDownstairs last five completed:
48722024-02-03T17:11:35.717ZUpstairs last five completed: 30 29 28 27 26
48732024-02-03T17:11:35.717Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:11:35.261642131Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1305}
48742024-02-03T17:11:35.717ZWrote out file "/tmp/test_repair_verify.data"
48752024-02-03T17:11:35.721Z
48762024-02-03T17:11:35.728ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
48772024-02-03T17:11:35.756ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
48782024-02-03T17:11:35.756Z 0 000-019 39 39 39 67 67 67 F F F
48792024-02-03T17:11:35.763Z 1 020-039 43 44 44 79 80 80 T F F <---
48802024-02-03T17:11:35.763Z 2 040-059 38 38 38 66 66 66 F F F
48812024-02-03T17:11:35.763Z 3 060-079 38 38 38 66 66 66 F F F
48822024-02-03T17:11:35.763Z 4 080-099 43 43 43 79 79 79 F F F
48832024-02-03T17:11:35.763Z 5 100-119 42 42 42 78 78 78 F F F
48842024-02-03T17:11:35.763Z 6 120-139 42 42 42 78 78 78 F F F
48852024-02-03T17:11:35.763Z 7 140-159 41 41 41 76 76 76 F F F
48862024-02-03T17:11:35.764Z 8 160-179 43 44 44 79 80 80 T F F <---
48872024-02-03T17:11:35.764Z 9 180-199 40 44 44 73 80 80 T F F <---
48882024-02-03T17:11:35.764Z 10 200-219 41 41 41 76 76 76 F F F
48892024-02-03T17:11:35.764Z 11 220-239 36 36 36 63 63 63 F T T <---
48902024-02-03T17:11:35.764Z 12 240-259 38 38 38 66 66 66 F F F
48912024-02-03T17:11:35.764Z 13 260-279 38 38 38 66 66 66 F F F
48922024-02-03T17:11:35.764Z 14 280-299 36 36 36 63 63 63 F T T <---
48932024-02-03T17:11:35.764Z 15 300-319 38 38 38 66 66 66 F T T <---
48942024-02-03T17:11:35.764Z 16 320-339 43 44 44 79 80 80 T F F <---
48952024-02-03T17:11:35.764Z 17 340-359 43 44 44 79 80 80 T F F <---
48962024-02-03T17:11:35.764Z 18 360-379 40 40 40 73 73 73 F F F
48972024-02-03T17:11:35.764Z 19 380-399 42 42 42 77 77 77 F F F
48982024-02-03T17:11:35.764Z 20 400-419 43 43 43 79 79 79 F T T <---
48992024-02-03T17:11:35.764Z 21 420-439 43 43 43 79 79 79 F F F
49002024-02-03T17:11:35.764Z 22 440-459 39 39 39 67 67 67 F T T <---
49012024-02-03T17:11:35.764Z 23 460-479 43 43 43 79 79 79 F F F
49022024-02-03T17:11:35.764Z 24 480-499 40 40 40 72 72 72 F F F
49032024-02-03T17:11:35.764Z 25 500-519 36 36 36 63 63 63 F T T <---
49042024-02-03T17:11:35.764Z 26 520-539 41 41 41 76 76 76 F F F
49052024-02-03T17:11:35.764Z 27 540-559 43 43 43 79 79 79 F F F
49062024-02-03T17:11:35.764Z 28 560-579 40 40 40 72 72 72 F F F
49072024-02-03T17:11:35.764Z 29 580-599 40 40 40 72 72 72 F T T <---
49082024-02-03T17:11:35.764ZMax gen: 44, Max flush: 80
49092024-02-03T17:11:35.764ZError: Difference in extent metadata found!
49102024-02-03T17:11:35.768ZA Difference in extent metadata is expected here
49112024-02-03T17:11:35.768ZOn loop 22
49122024-02-03T17:11:35.768Z
49132024-02-03T17:11:35.768ZVerifying data now
49142024-02-03T17:11:41.832ZNo change after verify
49152024-02-03T17:11:41.832ZLoop: 22 Downstairs dump after verify (and repair):
49162024-02-03T17:11:41.847ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
49172024-02-03T17:11:41.847Z 0 000-019 39 39 39 67 67 67 F F F
49182024-02-03T17:11:41.847Z 1 020-039 44 44 44 80 80 80 F F F
49192024-02-03T17:11:41.847Z 2 040-059 38 38 38 66 66 66 F F F
49202024-02-03T17:11:41.847Z 3 060-079 38 38 38 66 66 66 F F F
49212024-02-03T17:11:41.851Z 4 080-099 43 43 43 79 79 79 F F F
49222024-02-03T17:11:41.851Z 5 100-119 42 42 42 78 78 78 F F F
49232024-02-03T17:11:41.851Z 6 120-139 42 42 42 78 78 78 F F F
49242024-02-03T17:11:41.851Z 7 140-159 41 41 41 76 76 76 F F F
49252024-02-03T17:11:41.851Z 8 160-179 44 44 44 80 80 80 F F F
49262024-02-03T17:11:41.851Z 9 180-199 44 44 44 80 80 80 F F F
49272024-02-03T17:11:41.851Z 10 200-219 41 41 41 76 76 76 F F F
49282024-02-03T17:11:41.851Z 11 220-239 45 45 45 81 81 81 F F F
49292024-02-03T17:11:41.851Z 12 240-259 38 38 38 66 66 66 F F F
49302024-02-03T17:11:41.851Z 13 260-279 38 38 38 66 66 66 F F F
49312024-02-03T17:11:41.851Z 14 280-299 45 45 45 81 81 81 F F F
49322024-02-03T17:11:41.851Z 15 300-319 45 45 45 81 81 81 F F F
49332024-02-03T17:11:41.852Z 16 320-339 44 44 44 80 80 80 F F F
49342024-02-03T17:11:41.852Z 17 340-359 44 44 44 80 80 80 F F F
49352024-02-03T17:11:41.852Z 18 360-379 40 40 40 73 73 73 F F F
49362024-02-03T17:11:41.852Z 19 380-399 42 42 42 77 77 77 F F F
49372024-02-03T17:11:41.852Z 20 400-419 45 45 45 81 81 81 F F F
49382024-02-03T17:11:41.852Z 21 420-439 43 43 43 79 79 79 F F F
49392024-02-03T17:11:41.852Z 22 440-459 45 45 45 81 81 81 F F F
49402024-02-03T17:11:41.852Z 23 460-479 43 43 43 79 79 79 F F F
49412024-02-03T17:11:41.852Z 24 480-499 40 40 40 72 72 72 F F F
49422024-02-03T17:11:41.852Z 25 500-519 45 45 45 81 81 81 F F F
49432024-02-03T17:11:41.852Z 26 520-539 41 41 41 76 76 76 F F F
49442024-02-03T17:11:41.852Z 27 540-559 43 43 43 79 79 79 F F F
49452024-02-03T17:11:41.852Z 28 560-579 40 40 40 72 72 72 F F F
49462024-02-03T17:11:41.853Z 29 580-599 45 45 45 81 81 81 F F F
49472024-02-03T17:11:41.853ZMax gen: 45, Max flush: 81
49482024-02-03T17:11:41.853Z
49492024-02-03T17:11:41.853ZBegin loop 23
49502024-02-03T17:11:41.853ZDownstairs to restart: 2
49512024-02-03T17:11:41.882Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.430163932Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313}
49522024-02-03T17:11:41.883Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.430639634Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313}
49532024-02-03T17:11:41.883Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.430669229Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313}
49542024-02-03T17:11:41.887Z{"msg":"Crucible ee2fc0c6-3204-43f0-ba3c-a695644fd6b4 has session id: fe754cf5-3a61-45fc-84aa-073b648a3574","v":0,"Crucible runtime is spawned
49552024-02-03T17:11:41.887Zname":"crucible","level":30,"time":"2024-02-03T17:11:41.430962543Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49562024-02-03T17:11:41.887Z{"msg":"Upstairs opts: Upstairs UUID: ee2fc0c6-3204-43f0-ba3c-a695644fd6b4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.431080229Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49572024-02-03T17:11:41.887Z{"msg":"Crucible stats registered with UUID: ee2fc0c6-3204-43f0-ba3c-a695644fd6b4","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.431158084Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49582024-02-03T17:11:41.887Z{"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.431185567Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313}
49592024-02-03T17:11:41.887Z{"msg":"ee2fc0c6-3204-43f0-ba3c-a695644fd6b4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.431258865Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49602024-02-03T17:11:41.887Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.431296199Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49612024-02-03T17:11:41.887Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.431339505Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49622024-02-03T17:11:41.887Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.43140383Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49632024-02-03T17:11:41.887Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.431447171Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49642024-02-03T17:11:41.888Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.431484165Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49652024-02-03T17:11:41.888Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.431524164Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49662024-02-03T17:11:41.888Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.431680002Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"":"io task","client":"2","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49672024-02-03T17:11:41.888Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.431724098Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"":"io task","client":"0","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49682024-02-03T17:11:41.888Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.431806325Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"":"io task","client":"1","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49692024-02-03T17:11:41.888Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.431841002Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"":"io task","client":"1","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49702024-02-03T17:11:41.888Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.431877069Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"":"io task","client":"2","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49712024-02-03T17:11:41.888Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.431922041Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"":"io task","client":"0","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49722024-02-03T17:11:41.888Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.432746739Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49732024-02-03T17:11:41.888Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.432821594Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49742024-02-03T17:11:41.888Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.433008519Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49752024-02-03T17:11:41.888Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.433070605Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49762024-02-03T17:11:41.888Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.433643742Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49772024-02-03T17:11:41.889Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.43388388Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49782024-02-03T17:11:41.889Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.43403361Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49792024-02-03T17:11:41.889Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.43408289Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49802024-02-03T17:11:42.006Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.434564707Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49812024-02-03T17:11:42.006Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.434625475Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49822024-02-03T17:11:42.006Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.434669507Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49832024-02-03T17:11:42.007Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.434953572Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49842024-02-03T17:11:42.007Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.43502907Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49852024-02-03T17:11:42.007Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.435135541Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49862024-02-03T17:11:42.007Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.435214737Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49872024-02-03T17:11:42.007Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.435813146Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49882024-02-03T17:11:42.007Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.435898656Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49892024-02-03T17:11:42.007Z{"msg":"[0]R flush_numbers[0..12]: [67, 80, 66, 66, 79, 78, 78, 76, 80, 80, 76, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.435926967Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49902024-02-03T17:11:42.007Z{"msg":"[0]R generation[0..12]: [39, 44, 38, 38, 43, 42, 42, 41, 44, 44, 41, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.435984017Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49912024-02-03T17:11:42.007Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.436025671Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49922024-02-03T17:11:42.007Z{"msg":"[1]R flush_numbers[0..12]: [67, 80, 66, 66, 79, 78, 78, 76, 80, 80, 76, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.436082635Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49932024-02-03T17:11:42.007Z{"msg":"[1]R generation[0..12]: [39, 44, 38, 38, 43, 42, 42, 41, 44, 44, 41, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.436110284Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49942024-02-03T17:11:42.007Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
49952024-02-03T17:11:42.007Z,"time":"2024-02-03T17:11:41.436166626Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49962024-02-03T17:11:42.007Z{"msg":"[2]R flush_numbers[0..12]: [67, 80, 66, 66, 79, 78, 78, 76, 80, 80, 76, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.436197752Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49972024-02-03T17:11:42.007Z{"msg":"[2]R generation[0..12]: [39, 44, 38, 38, 43, 42, 42, 41, 44, 44, 41, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.436252122Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49982024-02-03T17:11:42.007Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.436278792Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
49992024-02-03T17:11:42.007Z{"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.436326114Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
50002024-02-03T17:11:42.007Z{"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.436355842Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
50012024-02-03T17:11:42.007Z{"msg":"Next flush: 82","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.436431435Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
50022024-02-03T17:11:42.007Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.4364695Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
50032024-02-03T17:11:42.007Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.436493308Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
50042024-02-03T17:11:42.007Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.436513441Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
50052024-02-03T17:11:42.007Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.436534808Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
50062024-02-03T17:11:42.008Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.436556232Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
50072024-02-03T17:11:42.008Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.436578199Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
50082024-02-03T17:11:42.008Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.436625726Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
50092024-02-03T17:11:42.008Z{"msg":"ee2fc0c6-3204-43f0-ba3c-a695644fd6b4 is now active with session: fe754cf5-3a61-45fc-84aa-073b648a3574","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.436649856Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
50102024-02-03T17:11:42.008Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.436669838Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313,"session_id":"fe754cf5-3a61-45fc-84aa-073b648a3574"}
50112024-02-03T17:11:42.008Z{"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:41.436715778Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313}
50122024-02-03T17:11:42.008ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
50132024-02-03T17:11:42.008ZLoading write count information from file "/tmp/test_repair_verify.data"
50142024-02-03T17:11:42.008ZRun Repair workload
50152024-02-03T17:11:42.008Z01/30 Write block 561 len 32768 data: 5 5 5 6 5 5 4 7
50162024-02-03T17:11:42.008Z02/30 Read block 568 len 4096
50172024-02-03T17:11:42.014Z03/30 Write block 84 len 24576 data: 5 5 5 4 4 5
50182024-02-03T17:11:42.014Z04/30 Read block 517 len 16384
50192024-02-03T17:11:42.014Z05/30 Read block 114 len 8192
50202024-02-03T17:11:42.014Z06/30 Read block 202 len 36864
50212024-02-03T17:11:42.014Z07/30 Write block 442 len 32768 data: 6 5 4 4 5 5 5 4
50222024-02-03T17:11:42.014Z08/30 Read block 371 len 8192
50232024-02-03T17:11:42.014Z09/30 Read block 202 len 32768
50242024-02-03T17:11:42.014Z10/30 Write block 324 len 8192 data: 5 4
50252024-02-03T17:11:42.015Z11/30 Read block 353 len 12288
50262024-02-03T17:11:42.015Z12/30 Flush
50272024-02-03T17:11:42.015Z13/30 Write block 185 len 4096 data: 8
50282024-02-03T17:11:42.015Z14/30 Read block 335 len 8192
50292024-02-03T17:11:42.015Z15/30 Write block 441 len 8192 data: 6 7
50302024-02-03T17:11:42.015Z16/30 Write block 281 len 8192 data: 6 7
50312024-02-03T17:11:42.015Z17/30 Read block 591 len 8192
50322024-02-03T17:11:42.015Z18/30 Write block 162 len 40960 data: 7 7 7 7 8 8 4 4 4 3
50332024-02-03T17:11:42.015Z19/30 Read block 361 len 20480
50342024-02-03T17:11:42.015Z20/30 Write block 390 len 32768 data: 6 6 7 8 7 6 4 3
50352024-02-03T17:11:42.015Z21/30 Read block 267 len 40960
50362024-02-03T17:11:42.015Z22/30 Write block 402 len 16384 data: 3 3 3 3
50372024-02-03T17:11:42.015Z23/30 Read block 534 len 16384
50382024-02-03T17:11:42.015Z24/30 Read block 480 len 24576
50392024-02-03T17:11:42.015Z25/30 Flush
50402024-02-03T17:11:42.015Z26/30 Write block 165 len 28672 data: 8 9 9 5 5 5 4
50412024-02-03T17:11:42.015Z27/30 Read block 194 len 12288
50422024-02-03T17:11:42.016Z28/30 Write block 209 len 20480 data: 4 5 4 4 5
50432024-02-03T17:11:42.016Z29/30 Write block 412 len 40960 data: 4 4 5 5 5 7 7 5 5 5
50442024-02-03T17:11:42.016Z30/30 Write block 316 len 40960 data: 5 6 4 7 7 7 7 8 6 5
50452024-02-03T17:11:42.016Z----------------------------------------------------------------
50462024-02-03T17:11:42.016Z Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:30
50472024-02-03T17:11:42.016ZStates: Active Active Active
50482024-02-03T17:11:42.016ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
50492024-02-03T17:11:42.016Z 1 Acked 1000 Write 8 Done Done Done false
50502024-02-03T17:11:42.016Z 2 Acked 1001 Read 1 Done Done Done false
50512024-02-03T17:11:42.016Z 3 Acked 1002 Write 6 Done Done Sent false
50522024-02-03T17:11:42.016Z 4 Acked 1003 Read 4 Done Done Sent false
50532024-02-03T17:11:42.016Z 5 Acked 1004 Read 2 Done Done Sent false
50542024-02-03T17:11:42.016Z 6 Acked 1005 Read 9 Done Done Sent false
50552024-02-03T17:11:42.016Z 7 Acked 1006 Write 8 Done Done Sent false
50562024-02-03T17:11:42.016Z 8 Acked 1007 Read 2 Done Done Sent false
50572024-02-03T17:11:42.016Z 9 Acked 1008 Read 8 Done Done Sent false
50582024-02-03T17:11:42.016Z 10 Acked 1009 Write 2 Done Done Sent false
50592024-02-03T17:11:42.016Z 11 Acked 1010 Read 3 Done Done Sent false
50602024-02-03T17:11:42.016Z 12 Acked 1011 Flush 0 Done Done Sent false
50612024-02-03T17:11:42.016Z 13 Acked 1012 Write 1 Done Done Sent false
50622024-02-03T17:11:42.016Z 14 Acked 1013 Read 2 Done Done Sent false
50632024-02-03T17:11:42.016Z 15 Acked 1014 Write 2 Done Done Sent false
50642024-02-03T17:11:42.016Z 16 Acked 1015 Write 2 Done Done Sent false
50652024-02-03T17:11:42.016Z 17 Acked 1016 Read 2 Done Done Sent false
50662024-02-03T17:11:42.017Z 18 Acked 1017 Write 10 Done Done Sent false
50672024-02-03T17:11:42.017Z 19 Acked 1018 Read 5 Done Done Sent false
50682024-02-03T17:11:42.017Z 20 Acked 1019 Write 8 Done Done Sent false
50692024-02-03T17:11:42.017Z 21 Acked 1020 Read 10 Done Done Sent false
50702024-02-03T17:11:42.017Z 22 Acked 1021 Write 4 Done Done Sent false
50712024-02-03T17:11:42.017Z 23 Acked 1022 Read 4 Done Done Sent false
50722024-02-03T17:11:42.017Z 24 Acked 1023 Read 6 Done Done Sent false
50732024-02-03T17:11:42.017Z 25 Acked 1024 Flush 0 Done Done Sent false
50742024-02-03T17:11:42.017Z 26 Acked 1025 Write 7 Done Done Sent false
50752024-02-03T17:11:42.017Z 27 Acked 1026 Read 3 Done Done Sent false
50762024-02-03T17:11:42.017Z 28 Acked 1027 Write 5 Done Done Sent false
50772024-02-03T17:11:42.017Z 29 Acked 1028 Write 10 Done Done Sent false
50782024-02-03T17:11:42.017Z 30 Acked 1029 Write 10 Sent Sent Sent false
50792024-02-03T17:11:42.017Z STATES DS:0 DS:1 DS:2 TOTAL
50802024-02-03T17:11:42.017Z New 0 0 0 0
50812024-02-03T17:11:42.017Z Sent 1 1 28 30
50822024-02-03T17:11:42.017Z Done 29 29 2 60
50832024-02-03T17:11:42.017Z Skipped 0 0 0 0
50842024-02-03T17:11:42.017Z Error 0 0 0 0
50852024-02-03T17:11:42.017ZLast Flush: 1024 1024 0
50862024-02-03T17:11:42.017ZDownstairs last five completed:
50872024-02-03T17:11:42.017ZUpstairs last five completed: 30 29 28 27 26
50882024-02-03T17:11:42.017Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:11:41.486790947Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1313}
50892024-02-03T17:11:42.017ZWrote out file "/tmp/test_repair_verify.data"
50902024-02-03T17:11:42.017Z
50912024-02-03T17:11:42.018ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
50922024-02-03T17:11:42.018ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
50932024-02-03T17:11:42.018Z 0 000-019 39 39 39 67 67 67 F F F
50942024-02-03T17:11:42.018Z 1 020-039 44 44 44 80 80 80 F F F
50952024-02-03T17:11:42.018Z 2 040-059 38 38 38 66 66 66 F F F
50962024-02-03T17:11:42.018Z 3 060-079 38 38 38 66 66 66 F F F
50972024-02-03T17:11:42.018Z 4 080-099 46 46 43 82 82 79 F F F <---
50982024-02-03T17:11:42.018Z 5 100-119 42 42 42 78 78 78 F F F
50992024-02-03T17:11:42.018Z 6 120-139 42 42 42 78 78 78 F F F
51002024-02-03T17:11:42.018Z 7 140-159 41 41 41 76 76 76 F F F
51012024-02-03T17:11:42.018Z 8 160-179 46 46 44 83 83 80 T T F <---
51022024-02-03T17:11:42.018Z 9 180-199 46 46 44 83 83 80 F F F <---
51032024-02-03T17:11:42.018Z 10 200-219 41 41 41 76 76 76 T T F <---
51042024-02-03T17:11:42.018Z 11 220-239 45 45 45 81 81 81 F F F
51052024-02-03T17:11:42.018Z 12 240-259 38 38 38 66 66 66 F F F
51062024-02-03T17:11:42.018Z 13 260-279 38 38 38 66 66 66 F F F
51072024-02-03T17:11:42.018Z 14 280-299 46 46 45 83 83 81 F F F <---
51082024-02-03T17:11:42.018Z 15 300-319 45 45 45 81 81 81 T T F <---
51092024-02-03T17:11:42.018Z 16 320-339 46 46 44 82 82 80 T T F <---
51102024-02-03T17:11:42.018Z 17 340-359 44 44 44 80 80 80 F F F
51112024-02-03T17:11:42.018Z 18 360-379 40 40 40 73 73 73 F F F
51122024-02-03T17:11:42.018Z 19 380-399 46 46 42 83 83 77 F F F <---
51132024-02-03T17:11:42.018Z 20 400-419 46 46 45 83 83 81 T T F <---
51142024-02-03T17:11:42.018Z 21 420-439 43 43 43 79 79 79 T T F <---
51152024-02-03T17:11:42.018Z 22 440-459 46 46 45 83 83 81 F F F <---
51162024-02-03T17:11:42.019Z 23 460-479 43 43 43 79 79 79 F F F
51172024-02-03T17:11:42.022Z 24 480-499 40 40 40 72 72 72 F F F
51182024-02-03T17:11:42.022Z 25 500-519 45 45 45 81 81 81 F F F
51192024-02-03T17:11:42.022Z 26 520-539 41 41 41 76 76 76 F F F
51202024-02-03T17:11:42.022Z 27 540-559 43 43 43 79 79 79 F F F
51212024-02-03T17:11:42.022Z 28 560-579 46 46 40 82 82 72 F F T <---
51222024-02-03T17:11:42.023Z 29 580-599 45 45 45 81 81 81 F F F
51232024-02-03T17:11:42.023ZMax gen: 46, Max flush: 83
51242024-02-03T17:11:42.023ZError: Difference in extent metadata found!
51252024-02-03T17:11:42.023ZA Difference in extent metadata is expected here
51262024-02-03T17:11:42.023ZOn loop 23
51272024-02-03T17:11:42.023Z
51282024-02-03T17:11:42.023ZVerifying data now
51292024-02-03T17:11:47.056ZNo change after verify
51302024-02-03T17:11:47.056ZLoop: 23 Downstairs dump after verify (and repair):
51312024-02-03T17:11:47.079ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51322024-02-03T17:11:47.079Z 0 000-019 39 39 39 67 67 67 F F F
51332024-02-03T17:11:47.079Z 1 020-039 44 44 44 80 80 80 F F F
51342024-02-03T17:11:47.079Z 2 040-059 38 38 38 66 66 66 F F F
51352024-02-03T17:11:47.079Z 3 060-079 38 38 38 66 66 66 F F F
51362024-02-03T17:11:47.079Z 4 080-099 46 46 46 82 82 82 F F F
51372024-02-03T17:11:47.080Z 5 100-119 42 42 42 78 78 78 F F F
51382024-02-03T17:11:47.080Z 6 120-139 42 42 42 78 78 78 F F F
51392024-02-03T17:11:47.080Z 7 140-159 41 41 41 76 76 76 F F F
51402024-02-03T17:11:47.080Z 8 160-179 47 47 47 84 84 84 F F F
51412024-02-03T17:11:47.080Z 9 180-199 46 46 46 83 83 83 F F F
51422024-02-03T17:11:47.080Z 10 200-219 47 47 47 84 84 84 F F F
51432024-02-03T17:11:47.083Z 11 220-239 45 45 45 81 81 81 F F F
51442024-02-03T17:11:47.083Z 12 240-259 38 38 38 66 66 66 F F F
51452024-02-03T17:11:47.083Z 13 260-279 38 38 38 66 66 66 F F F
51462024-02-03T17:11:47.083Z 14 280-299 46 46 46 83 83 83 F F F
51472024-02-03T17:11:47.083Z 15 300-319 47 47 47 84 84 84 F F F
51482024-02-03T17:11:47.083Z 16 320-339 47 47 47 84 84 84 F F F
51492024-02-03T17:11:47.083Z 17 340-359 44 44 44 80 80 80 F F F
51502024-02-03T17:11:47.084Z 18 360-379 40 40 40 73 73 73 F F F
51512024-02-03T17:11:47.084Z 19 380-399 46 46 46 83 83 83 F F F
51522024-02-03T17:11:47.084Z 20 400-419 47 47 47 84 84 84 F F F
51532024-02-03T17:11:47.084Z 21 420-439 47 47 47 84 84 84 F F F
51542024-02-03T17:11:47.084Z 22 440-459 46 46 46 83 83 83 F F F
51552024-02-03T17:11:47.084Z 23 460-479 43 43 43 79 79 79 F F F
51562024-02-03T17:11:47.084Z 24 480-499 40 40 40 72 72 72 F F F
51572024-02-03T17:11:47.084Z 25 500-519 45 45 45 81 81 81 F F F
51582024-02-03T17:11:47.084Z 26 520-539 41 41 41 76 76 76 F F F
51592024-02-03T17:11:47.084Z 27 540-559 43 43 43 79 79 79 F F F
51602024-02-03T17:11:47.084Z 28 560-579 46 46 46 82 82 82 F F F
51612024-02-03T17:11:47.084Z 29 580-599 45 45 45 81 81 81 F F F
51622024-02-03T17:11:47.084ZMax gen: 47, Max flush: 84
51632024-02-03T17:11:47.087Z
51642024-02-03T17:11:47.088ZBegin loop 24
51652024-02-03T17:11:47.088ZDownstairs to restart: 1
51662024-02-03T17:11:47.118Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.666298643Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321}
51672024-02-03T17:11:47.122Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
51682024-02-03T17:11:47.123Z,"time":"2024-02-03T17:11:46.666856922Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321}
51692024-02-03T17:11:47.123Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.666979459Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321}
51702024-02-03T17:11:47.123Z{"msg":"Crucible e5c30261-eb3d-47c6-b4b5-8ad57d63c87d has session id: 48cd408f-5c5e-49db-96f2-1828366e8e3b","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.66705655Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
51712024-02-03T17:11:47.123Z{"msg":"Upstairs opts: Upstairs UUID: e5c30261-eb3d-47c6-b4b5-8ad57d63c87d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.667102071Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
51722024-02-03T17:11:47.123Z{"msg":"Crucible stats registered with UUID: e5c30261-eb3d-47c6-b4b5-8ad57d63c87d","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.667126212Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
51732024-02-03T17:11:47.123Z{"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.667174956Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321}
51742024-02-03T17:11:47.123Z{"msg":"e5c30261-eb3d-47c6-b4b5-8ad57d63c87d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.667196707Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
51752024-02-03T17:11:47.123Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.667217041Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
51762024-02-03T17:11:47.123Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.667271205Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
51772024-02-03T17:11:47.123Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.667315888Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
51782024-02-03T17:11:47.123Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.667446215Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
51792024-02-03T17:11:47.123Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.667494242Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
51802024-02-03T17:11:47.123Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.667579319Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
51812024-02-03T17:11:47.123Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.667809017Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"":"io task","client":"0","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
51822024-02-03T17:11:47.123Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.667919514Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"":"io task","client":"2","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
51832024-02-03T17:11:47.123Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.667951922Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"":"io task","client":"1","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
51842024-02-03T17:11:47.123Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.668021099Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"":"io task","client":"1","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
51852024-02-03T17:11:47.123Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.66805737Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"":"io task","client":"2","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
51862024-02-03T17:11:47.124Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.668087276Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"":"io task","client":"0","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
51872024-02-03T17:11:47.124Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.669323688Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
51882024-02-03T17:11:47.124Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.669403524Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
51892024-02-03T17:11:47.124Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.669520248Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
51902024-02-03T17:11:47.124Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.669679456Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
51912024-02-03T17:11:47.124Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.669727384Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
51922024-02-03T17:11:47.124Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.669772243Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
51932024-02-03T17:11:47.124Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.670354028Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
51942024-02-03T17:11:47.128Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.670642247Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
51952024-02-03T17:11:47.128Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.671269881Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
51962024-02-03T17:11:47.128Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.671575194Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
51972024-02-03T17:11:47.128Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.671642096Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
51982024-02-03T17:11:47.128Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.671678409Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
51992024-02-03T17:11:47.128Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.671755275Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
52002024-02-03T17:11:47.129Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.67181739Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
52012024-02-03T17:11:47.129Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.671954419Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
52022024-02-03T17:11:47.129Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.672136626Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
52032024-02-03T17:11:47.129Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.672216808Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
52042024-02-03T17:11:47.129Z{"msg":"[0]R flush_numbers[0..12]: [67, 80, 66, 66, 82, 78, 78, 76, 84, 83, 84, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.672245901Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
52052024-02-03T17:11:47.129Z{"msg":"[0]R generation[0..12]: [39, 44, 38, 38, 46, 42, 42, 41, 47, 46, 47, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.672269373Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
52062024-02-03T17:11:47.129Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.672293372Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
52072024-02-03T17:11:47.129Z{"msg":"[1]R flush_numbers[0..12]: [67, 80, 66, 66, 82, 78, 78, 76, 84, 83, 84, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.67231837Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
52082024-02-03T17:11:47.129Z{"msg":"[1]R generation[0..12]: [39, 44, 38, 38, 46, 42, 42, 41, 47, 46, 47, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.67234163Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
52092024-02-03T17:11:47.129Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.672392011Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
52102024-02-03T17:11:47.129Z{"msg":"[2]R flush_numbers[0..12]: [67, 80, 66, 66, 82, 78, 78, 76, 84, 83, 84, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.672418147Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
52112024-02-03T17:11:47.129ZWait for a query_work_queue command to finish before sending IO
52122024-02-03T17:11:47.129Z{"msg":"[2]R generation[0..12]: [39, 44, 38, 38, 46, 42, 42, 41, 47, 46, 47, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.672471498Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
52132024-02-03T17:11:47.129Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.672497503Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
52142024-02-03T17:11:47.129Z{"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.672519165Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
52152024-02-03T17:11:47.129Z{"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.672541428Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
52162024-02-03T17:11:47.129Z{"msg":"Next flush: 85","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.67256258Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
52172024-02-03T17:11:47.129Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.672594012Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
52182024-02-03T17:11:47.129Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.672665929Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
52192024-02-03T17:11:47.129Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.672736793Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
52202024-02-03T17:11:47.130Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.672763142Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
52212024-02-03T17:11:47.130Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.672784954Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
52222024-02-03T17:11:47.130Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.672846858Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
52232024-02-03T17:11:47.130Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.67288446Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
52242024-02-03T17:11:47.130Z{"msg":"e5c30261-eb3d-47c6-b4b5-8ad57d63c87d is now active with session: 48cd408f-5c5e-49db-96f2-1828366e8e3b","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.672925178Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
52252024-02-03T17:11:47.130Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.672988611Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321,"session_id":"48cd408f-5c5e-49db-96f2-1828366e8e3b"}
52262024-02-03T17:11:47.130Z{"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:46.673030004Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321}
52272024-02-03T17:11:47.130ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
52282024-02-03T17:11:47.130ZLoading write count information from file "/tmp/test_repair_verify.data"
52292024-02-03T17:11:47.130ZRun Repair workload
52302024-02-03T17:11:47.130Z01/30 Write block 260 len 24576 data: 4 4 3 5 5 6
52312024-02-03T17:11:47.134Z02/30 Read block 480 len 4096
52322024-02-03T17:11:47.134Z03/30 Read block 106 len 4096
52332024-02-03T17:11:47.134Z04/30 Read block 180 len 20480
52342024-02-03T17:11:47.134Z05/30 Read block 376 len 32768
52352024-02-03T17:11:47.137Z06/30 Write block 361 len 28672 data: 7 4 3 5 5 5 6
52362024-02-03T17:11:47.137Z07/30 Read block 508 len 12288
52372024-02-03T17:11:47.137Z08/30 Flush
52382024-02-03T17:11:47.141Z09/30 Write block 235 len 4096 data: 3
52392024-02-03T17:11:47.141Z10/30 Read block 558 len 32768
52402024-02-03T17:11:47.144Z11/30 Read block 56 len 28672
52412024-02-03T17:11:47.147Z12/30 Write block 256 len 28672 data: 3 3 5 4 5 5 4
52422024-02-03T17:11:47.147Z13/30 Write block 586 len 12288 data: 5 4 4
52432024-02-03T17:11:47.147Z14/30 Write block 246 len 20480 data: 6 4 7 7 6
52442024-02-03T17:11:47.147Z15/30 Read block 409 len 16384
52452024-02-03T17:11:47.150Z16/30 Read block 393 len 28672
52462024-02-03T17:11:47.157Z17/30 Write block 98 len 28672 data: 4 6 6 6 3 3 4
52472024-02-03T17:11:47.157Z18/30 Read block 10 len 28672
52482024-02-03T17:11:47.160Z19/30 Read block 529 len 32768
52492024-02-03T17:11:47.160Z20/30 Read block 351 len 24576
52502024-02-03T17:11:47.160Z21/30 Write block 83 len 4096 data: 5
52512024-02-03T17:11:47.160Z22/30 Write block 308 len 4096 data: 5
52522024-02-03T17:11:47.160Z23/30 Write block 295 len 20480 data: 6 6 5 4 5
52532024-02-03T17:11:47.160Z24/30 Write block 70 len 4096 data: 4
52542024-02-03T17:11:47.163Z25/30 Read block 44 len 28672
52552024-02-03T17:11:47.163Z26/30 Write block 114 len 4096 data: 5
52562024-02-03T17:11:47.163Z27/30 Read block 133 len 24576
52572024-02-03T17:11:47.178Z28/30 Read block 338 len 12288
52582024-02-03T17:11:47.178Z29/30 Write block 525 len 32768 data: 5 5 5 5 5 6 4 4
52592024-02-03T17:11:47.184Z30/30 Read block 49 len 4096
52602024-02-03T17:11:47.184Z----------------------------------------------------------------
52612024-02-03T17:11:47.184Z Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30
52622024-02-03T17:11:47.184ZStates: Active Active Active
52632024-02-03T17:11:47.184ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
52642024-02-03T17:11:47.184Z 1 Acked 1000 Write 6 Done Sent Done false
52652024-02-03T17:11:47.184Z 2 Acked 1001 Read 1 Done Sent Done false
52662024-02-03T17:11:47.184Z 3 Acked 1002 Read 1 Done Sent Done false
52672024-02-03T17:11:47.184Z 4 Acked 1003 Read 5 Done Sent Done false
52682024-02-03T17:11:47.184Z 5 Acked 1004 Read 8 Done Sent Done false
52692024-02-03T17:11:47.184Z 6 Acked 1005 Write 7 Done Sent Done false
52702024-02-03T17:11:47.184Z 7 Acked 1006 Read 3 Done Sent Done false
52712024-02-03T17:11:47.185Z 8 Acked 1007 Flush 0 Done Sent Done false
52722024-02-03T17:11:47.185Z 9 Acked 1008 Write 1 Done Sent Done false
52732024-02-03T17:11:47.185Z 10 Acked 1009 Read 8 Done Sent Done false
52742024-02-03T17:11:47.185Z 11 Acked 1010 Read 7 Done Sent Done false
52752024-02-03T17:11:47.185Z 12 Acked 1011 Write 7 Done Sent Done false
52762024-02-03T17:11:47.185Z 13 Acked 1012 Write 3 Done Sent Done false
52772024-02-03T17:11:47.185Z 14 Acked 1013 Write 5 Done Sent Done false
52782024-02-03T17:11:47.185Z 15 Acked 1014 Read 4 Done Sent Done false
52792024-02-03T17:11:47.185Z 16 Acked 1015 Read 7 Done Sent Done false
52802024-02-03T17:11:47.185Z 17 Acked 1016 Write 7 Done Sent Done false
52812024-02-03T17:11:47.185Z 18 Acked 1017 Read 7 Done Sent Done false
52822024-02-03T17:11:47.185Z 19 Acked 1018 Read 8 Done Sent Done false
52832024-02-03T17:11:47.185Z 20 Acked 1019 Read 6 Done Sent Done false
52842024-02-03T17:11:47.185Z 21 Acked 1020 Write 1 Done Sent Done false
52852024-02-03T17:11:47.185Z 22 Acked 1021 Write 1 Done Sent Done false
52862024-02-03T17:11:47.185Z 23 Acked 1022 Write 5 Done Sent Done false
52872024-02-03T17:11:47.185Z 24 Acked 1023 Write 1 Done Sent Done false
52882024-02-03T17:11:47.185Z 25 Acked 1024 Read 7 Done Sent Done false
52892024-02-03T17:11:47.185Z 26 Acked 1025 Write 1 Done Sent Done false
52902024-02-03T17:11:47.185Z 27 Acked 1026 Read 6 Done Sent Done false
52912024-02-03T17:11:47.185Z 28 Acked 1027 Read 3 Done Sent Done false
52922024-02-03T17:11:47.185Z 29 Acked 1028 Write 8 Done Sent Done false
52932024-02-03T17:11:47.186Z 30 Acked 1029 Read 1 Sent Sent Done false
52942024-02-03T17:11:47.186Z STATES DS:0 DS:1 DS:2 TOTAL
52952024-02-03T17:11:47.186Z New 0 0 0 0
52962024-02-03T17:11:47.186Z Sent 1 30 0 31
52972024-02-03T17:11:47.186Z Done 29 0 30 59
52982024-02-03T17:11:47.186Z Skipped 0 0 0 0
52992024-02-03T17:11:47.186Z Error 0 0 0 0
53002024-02-03T17:11:47.186ZLast Flush: 1007 0 1007
53012024-02-03T17:11:47.186ZDownstairs last five completed:
53022024-02-03T17:11:47.186ZUpstairs last five completed: 30 29 28 27 26
53032024-02-03T17:11:47.186Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:11:46.718281901Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1321}
53042024-02-03T17:11:47.186ZWrote out file "/tmp/test_repair_verify.data"
53052024-02-03T17:11:47.186Z
53062024-02-03T17:11:47.190ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
53072024-02-03T17:11:47.212ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53082024-02-03T17:11:47.212Z 0 000-019 39 39 39 67 67 67 F F F
53092024-02-03T17:11:47.212Z 1 020-039 44 44 44 80 80 80 F F F
53102024-02-03T17:11:47.212Z 2 040-059 38 38 38 66 66 66 F F F
53112024-02-03T17:11:47.212Z 3 060-079 38 38 38 66 66 66 T F T <---
53122024-02-03T17:11:47.216Z 4 080-099 46 46 46 82 82 82 T F T <---
53132024-02-03T17:11:47.216Z 5 100-119 42 42 42 78 78 78 T F T <---
53142024-02-03T17:11:47.216Z 6 120-139 42 42 42 78 78 78 F F F
53152024-02-03T17:11:47.216Z 7 140-159 41 41 41 76 76 76 F F F
53162024-02-03T17:11:47.216Z 8 160-179 47 47 47 84 84 84 F F F
53172024-02-03T17:11:47.216Z 9 180-199 46 46 46 83 83 83 F F F
53182024-02-03T17:11:47.217Z 10 200-219 47 47 47 84 84 84 F F F
53192024-02-03T17:11:47.217Z 11 220-239 45 45 45 81 81 81 T F T <---
53202024-02-03T17:11:47.217Z 12 240-259 38 38 38 66 66 66 T F T <---
53212024-02-03T17:11:47.217Z 13 260-279 48 38 48 85 66 85 T F T <---
53222024-02-03T17:11:47.217Z 14 280-299 46 46 46 83 83 83 T F T <---
53232024-02-03T17:11:47.217Z 15 300-319 47 47 47 84 84 84 T F T <---
53242024-02-03T17:11:47.217Z 16 320-339 47 47 47 84 84 84 F F F
53252024-02-03T17:11:47.217Z 17 340-359 44 44 44 80 80 80 F F F
53262024-02-03T17:11:47.217Z 18 360-379 48 40 48 85 73 85 F F F <---
53272024-02-03T17:11:47.218Z 19 380-399 46 46 46 83 83 83 F F F
53282024-02-03T17:11:47.218Z 20 400-419 47 47 47 84 84 84 F F F
53292024-02-03T17:11:47.218Z 21 420-439 47 47 47 84 84 84 F F F
53302024-02-03T17:11:47.218Z 22 440-459 46 46 46 83 83 83 F F F
53312024-02-03T17:11:47.218Z 23 460-479 43 43 43 79 79 79 F F F
53322024-02-03T17:11:47.218Z 24 480-499 40 40 40 72 72 72 F F F
53332024-02-03T17:11:47.218Z 25 500-519 45 45 45 81 81 81 F F F
53342024-02-03T17:11:47.218Z 26 520-539 41 41 41 76 76 76 T F T <---
53352024-02-03T17:11:47.218Z 27 540-559 43 43 43 79 79 79 F F F
53362024-02-03T17:11:47.218Z 28 560-579 46 46 46 82 82 82 F F F
53372024-02-03T17:11:47.219Z 29 580-599 45 45 45 81 81 81 T F T <---
53382024-02-03T17:11:47.219ZMax gen: 48, Max flush: 85
53392024-02-03T17:11:47.219ZError: Difference in extent metadata found!
53402024-02-03T17:11:47.233ZA Difference in extent metadata is expected here
53412024-02-03T17:11:47.233ZOn loop 24
53422024-02-03T17:11:47.233Z
53432024-02-03T17:11:47.234ZVerifying data now
53442024-02-03T17:11:52.234ZNo change after verify
53452024-02-03T17:11:52.234ZLoop: 24 Downstairs dump after verify (and repair):
53462024-02-03T17:11:52.276ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53472024-02-03T17:11:52.277Z 0 000-019 39 39 39 67 67 67 F F F
53482024-02-03T17:11:52.292Z 1 020-039 44 44 44 80 80 80 F F F
53492024-02-03T17:11:52.292Z 2 040-059 38 38 38 66 66 66 F F F
53502024-02-03T17:11:52.292Z 3 060-079 49 49 49 86 86 86 F F F
53512024-02-03T17:11:52.292Z 4 080-099 49 49 49 86 86 86 F F F
53522024-02-03T17:11:52.292Z 5 100-119 49 49 49 86 86 86 F F F
53532024-02-03T17:11:52.293Z 6 120-139 42 42 42 78 78 78 F F F
53542024-02-03T17:11:52.293Z 7 140-159 41 41 41 76 76 76 F F F
53552024-02-03T17:11:52.293Z 8 160-179 47 47 47 84 84 84 F F F
53562024-02-03T17:11:52.293Z 9 180-199 46 46 46 83 83 83 F F F
53572024-02-03T17:11:52.293Z 10 200-219 47 47 47 84 84 84 F F F
53582024-02-03T17:11:52.293Z 11 220-239 49 49 49 86 86 86 F F F
53592024-02-03T17:11:52.293Z 12 240-259 49 49 49 86 86 86 F F F
53602024-02-03T17:11:52.293Z 13 260-279 49 49 49 86 86 86 F F F
53612024-02-03T17:11:52.293Z 14 280-299 49 49 49 86 86 86 F F F
53622024-02-03T17:11:52.294Z 15 300-319 49 49 49 86 86 86 F F F
53632024-02-03T17:11:52.294Z 16 320-339 47 47 47 84 84 84 F F F
53642024-02-03T17:11:52.294Z 17 340-359 44 44 44 80 80 80 F F F
53652024-02-03T17:11:52.294Z 18 360-379 48 48 48 85 85 85 F F F
53662024-02-03T17:11:52.294Z 19 380-399 46 46 46 83 83 83 F F F
53672024-02-03T17:11:52.294Z 20 400-419 47 47 47 84 84 84 F F F
53682024-02-03T17:11:52.294Z 21 420-439 47 47 47 84 84 84 F F F
53692024-02-03T17:11:52.294Z 22 440-459 46 46 46 83 83 83 F F F
53702024-02-03T17:11:52.294Z 23 460-479 43 43 43 79 79 79 F F F
53712024-02-03T17:11:52.294Z 24 480-499 40 40 40 72 72 72 F F F
53722024-02-03T17:11:52.294Z 25 500-519 45 45 45 81 81 81 F F F
53732024-02-03T17:11:52.295Z 26 520-539 49 49 49 86 86 86 F F F
53742024-02-03T17:11:52.295Z 27 540-559 43 43 43 79 79 79 F F F
53752024-02-03T17:11:52.295Z 28 560-579 46 46 46 82 82 82 F F F
53762024-02-03T17:11:52.295Z 29 580-599 49 49 49 86 86 86 F F F
53772024-02-03T17:11:52.295ZMax gen: 49, Max flush: 86
53782024-02-03T17:11:52.295Z
53792024-02-03T17:11:52.295ZBegin loop 25
53802024-02-03T17:11:52.295ZDownstairs to restart: 2
53812024-02-03T17:11:52.725Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.836323848Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329}
53822024-02-03T17:11:52.726Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.836792918Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329}
53832024-02-03T17:11:52.726Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.83687797Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329}
53842024-02-03T17:11:52.726Z{"msg":"Crucible d20456a8-9515-4a1d-93c7-9a9041e58510 has session id: d973df6a-00f2-481e-979c-ad76840af2ae","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.836904605Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
53852024-02-03T17:11:52.726Z{"msg":Crucible runtime is spawned
53862024-02-03T17:11:52.727Z"Upstairs opts: Upstairs UUID: d20456a8-9515-4a1d-93c7-9a9041e58510, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.837092324Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
53872024-02-03T17:11:52.727Z{"msg":"Crucible stats registered with UUID: d20456a8-9515-4a1d-93c7-9a9041e58510","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.837171047Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
53882024-02-03T17:11:52.727Z{"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.837210323Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329}
53892024-02-03T17:11:52.727Z{"msg":"d20456a8-9515-4a1d-93c7-9a9041e58510 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.837587632Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
53902024-02-03T17:11:52.727Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.837647988Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
53912024-02-03T17:11:52.728Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.837674389Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
53922024-02-03T17:11:52.728Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.837696605Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
53932024-02-03T17:11:52.728Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.837718962Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
53942024-02-03T17:11:52.728Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.837812287Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
53952024-02-03T17:11:52.728Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.83803323Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
53962024-02-03T17:11:52.729Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.838120325Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"":"io task","client":"2","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
53972024-02-03T17:11:52.729Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.838169405Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"":"io task","client":"0","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
53982024-02-03T17:11:52.729Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.838278975Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"":"io task","client":"1","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
53992024-02-03T17:11:52.729Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.838410823Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"":"io task","client":"1","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54002024-02-03T17:11:52.730Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.838504749Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"":"io task","client":"0","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54012024-02-03T17:11:52.730Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.838590504Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"":"io task","client":"2","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54022024-02-03T17:11:52.730Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.839682036Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54032024-02-03T17:11:52.730Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.83975208Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54042024-02-03T17:11:52.730Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.840086627Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54052024-02-03T17:11:52.731Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.840209922Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54062024-02-03T17:11:52.731Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.840307058Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54072024-02-03T17:11:52.731Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.840353028Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54082024-02-03T17:11:52.731Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.840763835Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54092024-02-03T17:11:52.731Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.84085867Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54102024-02-03T17:11:52.732Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.841346101Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54112024-02-03T17:11:52.732Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.841648006Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54122024-02-03T17:11:52.732Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.841738591Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54132024-02-03T17:11:52.732Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.841770226Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54142024-02-03T17:11:52.732Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.841791921Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54152024-02-03T17:11:52.733Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.841841628Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54162024-02-03T17:11:52.733Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.841867271Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54172024-02-03T17:11:52.733Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.842145297Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54182024-02-03T17:11:52.733Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.842237087Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54192024-02-03T17:11:52.733Z{"msg":"[0]R flush_numbers[0..12]: [67, 80, 66, 86, 86, 86, 78, 76, 84, 83, 84, 86]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.842269299Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54202024-02-03T17:11:52.734Z{"msg":"[0]R generation[0..12]: [39, 44, 38, 49, 49, 49, 42, 41, 47, 46, 47, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.842398534Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54212024-02-03T17:11:52.734Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.842425778Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54222024-02-03T17:11:52.734Z{"msg":"[1]R flush_numbers[0..12]: [67, 80, 66, 86, 86, 86, 78, 76, 84, 83, 84, 86]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.842449502Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54232024-02-03T17:11:52.734Z{"msg":"[1]R generation[0..12]: [39, 44, 38, 49, 49, 49, 42, 41, 47, 46, 47, 49]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
54242024-02-03T17:11:52.734Z,"time":"2024-02-03T17:11:51.842497252Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54252024-02-03T17:11:52.735Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.842526792Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54262024-02-03T17:11:52.735Z{"msg":"[2]R flush_numbers[0..12]: [67, 80, 66, 86, 86, 86, 78, 76, 84, 83, 84, 86]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.842550088Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54272024-02-03T17:11:52.735Z{"msg":"[2]R generation[0..12]: [39, 44, 38, 49, 49, 49, 42, 41, 47, 46, 47, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.842599504Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54282024-02-03T17:11:52.735Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.842625693Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54292024-02-03T17:11:52.735Z{"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.842647521Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54302024-02-03T17:11:52.736Z{"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.842669571Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54312024-02-03T17:11:52.736Z{"msg":"Next flush: 87","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.842690729Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54322024-02-03T17:11:52.736Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.842712202Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54332024-02-03T17:11:52.736Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.842761017Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54342024-02-03T17:11:52.736Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.842783403Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54352024-02-03T17:11:52.737Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.842805434Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54362024-02-03T17:11:52.737Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.842829062Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54372024-02-03T17:11:52.737Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.842851236Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54382024-02-03T17:11:52.737Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.84287204Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54392024-02-03T17:11:52.737Z{"msg":"d20456a8-9515-4a1d-93c7-9a9041e58510 is now active with session: d973df6a-00f2-481e-979c-ad76840af2ae","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.842920233Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54402024-02-03T17:11:52.737Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.842942687Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329,"session_id":"d973df6a-00f2-481e-979c-ad76840af2ae"}
54412024-02-03T17:11:52.738Z{"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:51.842962994Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329}
54422024-02-03T17:11:52.738ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
54432024-02-03T17:11:52.738ZLoading write count information from file "/tmp/test_repair_verify.data"
54442024-02-03T17:11:52.738ZRun Repair workload
54452024-02-03T17:11:52.738Z01/30 Write block 427 len 24576 data: 4 4 6 6 5 5
54462024-02-03T17:11:52.739Z02/30 Read block 326 len 32768
54472024-02-03T17:11:52.739Z03/30 Write block 527 len 24576 data: 6 6 6 7 5 5
54482024-02-03T17:11:52.739Z04/30 Read block 185 len 12288
54492024-02-03T17:11:52.750Z05/30 Write block 391 len 36864 data: 7 8 9 8 7 5 4 4 4
54502024-02-03T17:11:52.750Z06/30 Flush
54512024-02-03T17:11:52.750Z07/30 Write block 533 len 12288 data: 5 3 3
54522024-02-03T17:11:52.750Z08/30 Write block 322 len 32768 data: 8 9 7 6 4 4 4 5
54532024-02-03T17:11:52.751Z09/30 Flush
54542024-02-03T17:11:52.751Z10/30 Write block 374 len 28672 data: 5 4 4 5 4 6 5
54552024-02-03T17:11:52.751Z11/30 Read block 9 len 4096
54562024-02-03T17:11:52.751Z12/30 Read block 525 len 24576
54572024-02-03T17:11:52.751Z13/30 Write block 472 len 20480 data: 3 3 5 5 4
54582024-02-03T17:11:52.752Z14/30 Flush
54592024-02-03T17:11:52.752Z15/30 Write block 65 len 24576 data: 2 2 3 3 3 5
54602024-02-03T17:11:52.752Z16/30 Write block 205 len 4096 data: 4
54612024-02-03T17:11:52.752Z17/30 Read block 312 len 32768
54622024-02-03T17:11:52.753Z18/30 Read block 114 len 20480
54632024-02-03T17:11:52.753Z19/30 Read block 350 len 12288
54642024-02-03T17:11:52.753Z20/30 Read block 192 len 28672
54652024-02-03T17:11:52.753Z21/30 Write block 396 len 24576 data: 6 5 5 5 4 4
54662024-02-03T17:11:52.753Z22/30 Read block 542 len 24576
54672024-02-03T17:11:52.753Z23/30 Flush
54682024-02-03T17:11:52.754Z24/30 Write block 79 len 36864 data: 6 5 7 6 6 6 6 6 5
54692024-02-03T17:11:52.754Z25/30 Write block 432 len 24576 data: 6 5 5 6 6 5
54702024-02-03T17:11:52.754Z26/30 Write block 488 len 8192 data: 4 4
54712024-02-03T17:11:52.754Z27/30 Write block 479 len 32768 data: 6 6 5 4 5 4 3 4
54722024-02-03T17:11:52.755Z28/30 Read block 423 len 16384
54732024-02-03T17:11:52.755Z29/30 Write block 149 len 40960 data: 4 3 3 3 3 3 4 5 3 4
54742024-02-03T17:11:52.755Z30/30 Write block 535 len 8192 data: 4 3
54752024-02-03T17:11:52.755Z----------------------------------------------------------------
54762024-02-03T17:11:52.756Z Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:30
54772024-02-03T17:11:52.756ZStates: Active Active Active
54782024-02-03T17:11:52.756ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
54792024-02-03T17:11:52.756Z 1 Acked 1000 Write 6 Done Done Done false
54802024-02-03T17:11:52.756Z 2 Acked 1001 Read 8 Done Done Done false
54812024-02-03T17:11:52.757Z 3 Acked 1002 Write 6 Done Done Sent false
54822024-02-03T17:11:52.757Z 4 Acked 1003 Read 3 Done Done Sent false
54832024-02-03T17:11:52.757Z 5 Acked 1004 Write 9 Done Done Sent false
54842024-02-03T17:11:52.757Z 6 Acked 1005 Flush 0 Done Done Sent false
54852024-02-03T17:11:52.757Z 7 Acked 1006 Write 3 Done Done Sent false
54862024-02-03T17:11:52.758Z 8 Acked 1007 Write 8 Done Done Sent false
54872024-02-03T17:11:52.758Z 9 Acked 1008 Flush 0 Done Done Sent false
54882024-02-03T17:11:52.758Z 10 Acked 1009 Write 7 Done Done Sent false
54892024-02-03T17:11:52.758Z 11 Acked 1010 Read 1 Done Done Sent false
54902024-02-03T17:11:52.758Z 12 Acked 1011 Read 6 Done Done Sent false
54912024-02-03T17:11:52.759Z 13 Acked 1012 Write 5 Done Done Sent false
54922024-02-03T17:11:52.759Z 14 Acked 1013 Flush 0 Done Done Sent false
54932024-02-03T17:11:52.759Z 15 Acked 1014 Write 6 Done Done Sent false
54942024-02-03T17:11:52.759Z 16 Acked 1015 Write 1 Done Done Sent false
54952024-02-03T17:11:52.759Z 17 Acked 1016 Read 8 Done Done Sent false
54962024-02-03T17:11:52.759Z 18 Acked 1017 Read 5 Done Done Sent false
54972024-02-03T17:11:52.760Z 19 Acked 1018 Read 3 Done Done Sent false
54982024-02-03T17:11:52.760Z 20 Acked 1019 Read 7 Done Done Sent false
54992024-02-03T17:11:52.760Z 21 Acked 1020 Write 6 Done Done Sent false
55002024-02-03T17:11:52.760Z 22 Acked 1021 Read 6 Done Done Sent false
55012024-02-03T17:11:52.760Z 23 Acked 1022 Flush 0 Done Done Sent false
55022024-02-03T17:11:52.761Z 24 Acked 1023 Write 9 Done Done Sent false
55032024-02-03T17:11:52.761Z 25 Acked 1024 Write 6 Done Done Sent false
55042024-02-03T17:11:52.761Z 26 Acked 1025 Write 2 Done Done Sent false
55052024-02-03T17:11:52.761Z 27 Acked 1026 Write 8 Done Done Sent false
55062024-02-03T17:11:52.761Z 28 Acked 1027 Read 4 Done Done Sent false
55072024-02-03T17:11:52.762Z 29 Acked 1028 Write 10 Sent Sent Sent false
55082024-02-03T17:11:52.762Z 30 Acked 1029 Write 2 Sent Sent Sent false
55092024-02-03T17:11:52.762Z STATES DS:0 DS:1 DS:2 TOTAL
55102024-02-03T17:11:52.762Z New 0 0 0 0
55112024-02-03T17:11:52.762Z Sent 2 2 28 32
55122024-02-03T17:11:52.763Z Done 28 28 2 58
55132024-02-03T17:11:52.763Z Skipped 0 0 0 0
55142024-02-03T17:11:52.763Z Error 0 0 0 0
55152024-02-03T17:11:52.763ZLast Flush: 1022 1022 0
55162024-02-03T17:11:52.763ZDownstairs last five completed:
55172024-02-03T17:11:52.764ZUpstairs last five completed: 30 29 28 27 26
55182024-02-03T17:11:52.764Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:11:51.898777952Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1329}
55192024-02-03T17:11:52.764ZWrote out file "/tmp/test_repair_verify.data"
55202024-02-03T17:11:52.764Z
55212024-02-03T17:11:52.764ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
55222024-02-03T17:11:52.765ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
55232024-02-03T17:11:52.765Z 0 000-019 39 39 39 67 67 67 F F F
55242024-02-03T17:11:52.765Z 1 020-039 44 44 44 80 80 80 F F F
55252024-02-03T17:11:52.765Z 2 040-059 38 38 38 66 66 66 F F F
55262024-02-03T17:11:52.765Z 3 060-079 50 50 49 90 90 86 T T F <---
55272024-02-03T17:11:52.765Z 4 080-099 49 49 49 86 86 86 T T F <---
55282024-02-03T17:11:52.766Z 5 100-119 49 49 49 86 86 86 F F F
55292024-02-03T17:11:52.766Z 6 120-139 42 42 42 78 78 78 F F F
55302024-02-03T17:11:52.766Z 7 140-159 41 41 41 76 76 76 T T F <---
55312024-02-03T17:11:52.766Z 8 160-179 47 47 47 84 84 84 F F F
55322024-02-03T17:11:52.767Z 9 180-199 46 46 46 83 83 83 F F F
55332024-02-03T17:11:52.767Z 10 200-219 50 50 47 90 90 84 F F F <---
55342024-02-03T17:11:52.767Z 11 220-239 49 49 49 86 86 86 F F F
55352024-02-03T17:11:52.767Z 12 240-259 49 49 49 86 86 86 F F F
55362024-02-03T17:11:52.767Z 13 260-279 49 49 49 86 86 86 F F F
55372024-02-03T17:11:52.767Z 14 280-299 49 49 49 86 86 86 F F F
55382024-02-03T17:11:52.768Z 15 300-319 49 49 49 86 86 86 F F F
55392024-02-03T17:11:52.768Z 16 320-339 50 50 47 88 88 84 F F F <---
55402024-02-03T17:11:52.768Z 17 340-359 44 44 44 80 80 80 F F F
55412024-02-03T17:11:52.768Z 18 360-379 50 50 48 89 89 85 F F F <---
55422024-02-03T17:11:52.768Z 19 380-399 50 50 46 90 90 83 F F F <---
55432024-02-03T17:11:52.768Z 20 400-419 50 50 47 90 90 84 F F F <---
55442024-02-03T17:11:52.769Z 21 420-439 50 50 47 87 87 84 T T T <---
55452024-02-03T17:11:52.769Z 22 440-459 46 46 46 83 83 83 F F F
55462024-02-03T17:11:52.769Z 23 460-479 50 50 43 89 89 79 T T F <---
55472024-02-03T17:11:52.769Z 24 480-499 40 40 40 72 72 72 T T F <---
55482024-02-03T17:11:52.769Z 25 500-519 45 45 45 81 81 81 F F F
55492024-02-03T17:11:52.778Z 26 520-539 50 50 49 88 88 86 T T F <---
55502024-02-03T17:11:52.778Z 27 540-559 43 43 43 79 79 79 F F F
55512024-02-03T17:11:52.779Z 28 560-579 46 46 46 82 82 82 F F F
55522024-02-03T17:11:52.779Z 29 580-599 49 49 49 86 86 86 F F F
55532024-02-03T17:11:52.779ZMax gen: 50, Max flush: 90
55542024-02-03T17:11:52.779ZError: Difference in extent metadata found!
55552024-02-03T17:11:52.780ZA Difference in extent metadata is expected here
55562024-02-03T17:11:52.780ZOn loop 25
55572024-02-03T17:11:52.780Z
55582024-02-03T17:11:52.780ZVerifying data now
55592024-02-03T17:11:57.477ZNo change after verify
55602024-02-03T17:11:57.477ZLoop: 25 Downstairs dump after verify (and repair):
55612024-02-03T17:11:57.500ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
55622024-02-03T17:11:57.501Z 0 000-019 39 39 39 67 67 67 F F F
55632024-02-03T17:11:57.501Z 1 020-039 44 44 44 80 80 80 F F F
55642024-02-03T17:11:57.501Z 2 040-059 38 38 38 66 66 66 F F F
55652024-02-03T17:11:57.501Z 3 060-079 51 51 51 91 91 91 F F F
55662024-02-03T17:11:57.505Z 4 080-099 51 51 51 91 91 91 F F F
55672024-02-03T17:11:57.505Z 5 100-119 49 49 49 86 86 86 F F F
55682024-02-03T17:11:57.505Z 6 120-139 42 42 42 78 78 78 F F F
55692024-02-03T17:11:57.505Z 7 140-159 51 51 51 91 91 91 F F F
55702024-02-03T17:11:57.505Z 8 160-179 47 47 47 84 84 84 F F F
55712024-02-03T17:11:57.505Z 9 180-199 46 46 46 83 83 83 F F F
55722024-02-03T17:11:57.505Z 10 200-219 50 50 50 90 90 90 F F F
55732024-02-03T17:11:57.505Z 11 220-239 49 49 49 86 86 86 F F F
55742024-02-03T17:11:57.505Z 12 240-259 49 49 49 86 86 86 F F F
55752024-02-03T17:11:57.506Z 13 260-279 49 49 49 86 86 86 F F F
55762024-02-03T17:11:57.506Z 14 280-299 49 49 49 86 86 86 F F F
55772024-02-03T17:11:57.506Z 15 300-319 49 49 49 86 86 86 F F F
55782024-02-03T17:11:57.506Z 16 320-339 50 50 50 88 88 88 F F F
55792024-02-03T17:11:57.506Z 17 340-359 44 44 44 80 80 80 F F F
55802024-02-03T17:11:57.506Z 18 360-379 50 50 50 89 89 89 F F F
55812024-02-03T17:11:57.506Z 19 380-399 50 50 50 90 90 90 F F F
55822024-02-03T17:11:57.506Z 20 400-419 50 50 50 90 90 90 F F F
55832024-02-03T17:11:57.506Z 21 420-439 51 51 51 91 91 91 F F F
55842024-02-03T17:11:57.506Z 22 440-459 46 46 46 83 83 83 F F F
55852024-02-03T17:11:57.507Z 23 460-479 51 51 51 91 91 91 F F F
55862024-02-03T17:11:57.507Z 24 480-499 51 51 51 91 91 91 F F F
55872024-02-03T17:11:57.507Z 25 500-519 45 45 45 81 81 81 F F F
55882024-02-03T17:11:57.507Z 26 520-539 51 51 51 91 91 91 F F F
55892024-02-03T17:11:57.507Z 27 540-559 43 43 43 79 79 79 F F F
55902024-02-03T17:11:57.507Z 28 560-579 46 46 46 82 82 82 F F F
55912024-02-03T17:11:57.507Z 29 580-599 49 49 49 86 86 86 F F F
55922024-02-03T17:11:57.508ZMax gen: 51, Max flush: 91
55932024-02-03T17:11:57.515Z
55942024-02-03T17:11:57.516ZBegin loop 26
55952024-02-03T17:11:57.516ZDownstairs to restart: 2
55962024-02-03T17:11:57.539Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.084663277Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337}
55972024-02-03T17:11:57.539Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.085113853Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337}
55982024-02-03T17:11:57.539Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.085148799Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337}
55992024-02-03T17:11:57.548Z{"msg":"Crucible a2341d81-bec7-427f-aa9c-d691032d7915 has session id: e232dd9b-e348-425b-98e4-2b1f6cece893","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.085224973Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56002024-02-03T17:11:57.548Z{"msg":"Upstairs opts: Upstairs UUID: a2341d81-bec7-427f-aa9c-d691032d7915, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.08535849Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56012024-02-03T17:11:57.549Z{"msg":"Crucible stats registered with UUID: a2341d81-bec7-427f-aa9c-d691032d7915","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.085404445Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56022024-02-03T17:11:57.549ZCrucible runtime is spawned
56032024-02-03T17:11:57.549Z{"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.085708435Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337}
56042024-02-03T17:11:57.549Z{"msg":"a2341d81-bec7-427f-aa9c-d691032d7915 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.085967393Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56052024-02-03T17:11:57.549Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.086054959Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56062024-02-03T17:11:57.550Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.086167455Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56072024-02-03T17:11:57.550Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.086279802Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56082024-02-03T17:11:57.550Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.08632687Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56092024-02-03T17:11:57.550Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.086370264Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56102024-02-03T17:11:57.550Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.086459783Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56112024-02-03T17:11:57.550Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.086506379Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"":"io task","client":"0","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56122024-02-03T17:11:57.551Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.086554946Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"":"io task","client":"2","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56132024-02-03T17:11:57.551Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.086653148Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"":"io task","client":"1","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56142024-02-03T17:11:57.551Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.086700211Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"":"io task","client":"2","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56152024-02-03T17:11:57.551Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.086782098Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"":"io task","client":"0","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56162024-02-03T17:11:57.551Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.086898049Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"":"io task","client":"1","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56172024-02-03T17:11:57.551Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.088016234Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56182024-02-03T17:11:57.552Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.088106396Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56192024-02-03T17:11:57.552Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.088217644Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56202024-02-03T17:11:57.552Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.08837067Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56212024-02-03T17:11:57.552Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.088936249Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56222024-02-03T17:11:57.552Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.089252586Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56232024-02-03T17:11:57.552Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.089318939Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56242024-02-03T17:11:57.553Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.089346186Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56252024-02-03T17:11:57.553Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.089906912Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56262024-02-03T17:11:57.553Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.08999691Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56272024-02-03T17:11:57.553Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.090038653Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56282024-02-03T17:11:57.553Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.090334666Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56292024-02-03T17:11:57.553Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.090406681Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56302024-02-03T17:11:57.554Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.090433481Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56312024-02-03T17:11:57.554Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.090491303Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56322024-02-03T17:11:57.554Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.091249799Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56332024-02-03T17:11:57.554Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.091372591Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56342024-02-03T17:11:57.554Z{"msg":"[0]R flush_numbers[0..12]: [67, 80, 66, 91, 91, 86, 78, 91, 84, 83, 90, 86]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.091454797Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56352024-02-03T17:11:57.554Z{"msg":"[0]R generation[0..12]: [39, 44, 38, 51, 51, 49, 42, 51, 47, 46, 50, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.091488681Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56362024-02-03T17:11:57.554Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
56372024-02-03T17:11:57.555Z,"time":"2024-02-03T17:11:57.091555456Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56382024-02-03T17:11:57.555Z{"msg":"[1]R flush_numbers[0..12]: [67, 80, 66, 91, 91, 86, 78, 91, 84, 83, 90, 86]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.091583959Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56392024-02-03T17:11:57.555Z{"msg":"[1]R generation[0..12]: [39, 44, 38, 51, 51, 49, 42, 51, 47, 46, 50, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.091614013Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56402024-02-03T17:11:57.555Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.09168077Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56412024-02-03T17:11:57.555Z{"msg":"[2]R flush_numbers[0..12]: [67, 80, 66, 91, 91, 86, 78, 91, 84, 83, 90, 86]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.091708566Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56422024-02-03T17:11:57.555Z{"msg":"[2]R generation[0..12]: [39, 44, 38, 51, 51, 49, 42, 51, 47, 46, 50, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.09177687Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56432024-02-03T17:11:57.556Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.091805917Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56442024-02-03T17:11:57.556Z{"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.091827777Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56452024-02-03T17:11:57.556Z{"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.091894077Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56462024-02-03T17:11:57.556Z{"msg":"Next flush: 92","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.091920306Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56472024-02-03T17:11:57.556Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.091946302Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56482024-02-03T17:11:57.556Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.092009217Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56492024-02-03T17:11:57.557Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.092033497Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56502024-02-03T17:11:57.557Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.092097342Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56512024-02-03T17:11:57.557Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.092124187Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56522024-02-03T17:11:57.557Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.092190354Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56532024-02-03T17:11:57.557Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.092215575Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56542024-02-03T17:11:57.557Z{"msg":"a2341d81-bec7-427f-aa9c-d691032d7915 is now active with session: e232dd9b-e348-425b-98e4-2b1f6cece893","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.092237148Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56552024-02-03T17:11:57.558Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.092256642Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337,"session_id":"e232dd9b-e348-425b-98e4-2b1f6cece893"}
56562024-02-03T17:11:57.558Z{"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:11:57.092284558Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337}
56572024-02-03T17:11:57.558ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
56582024-02-03T17:11:57.567ZLoading write count information from file "/tmp/test_repair_verify.data"
56592024-02-03T17:11:57.568ZRun Repair workload
56602024-02-03T17:11:57.568Z01/30 Write block 51 len 8192 data: 3 3
56612024-02-03T17:11:57.568Z02/30 Read block 583 len 12288
56622024-02-03T17:11:57.568Z03/30 Write block 414 len 36864 data: 6 6 6 8 8 6 6 6 4
56632024-02-03T17:11:57.568Z04/30 Write block 178 len 8192 data: 4 3
56642024-02-03T17:11:57.568Z05/30 Read block 154 len 8192
56652024-02-03T17:11:57.569Z06/30 Read block 328 len 20480
56662024-02-03T17:11:57.569Z07/30 Write block 122 len 32768 data: 4 3 2 2 2 3 3 4
56672024-02-03T17:11:57.569Z08/30 Read block 501 len 40960
56682024-02-03T17:11:57.569Z09/30 Write block 81 len 36864 data: 8 7 7 7 7 7 6 5 6
56692024-02-03T17:11:57.570Z10/30 Read block 217 len 8192
56702024-02-03T17:11:57.570Z11/30 Write block 304 len 16384 data: 6 5 7 5
56712024-02-03T17:11:57.570Z12/30 Write block 264 len 40960 data: 6 7 7 8 8 6 6 5 5 5
56722024-02-03T17:11:57.570Z13/30 Read block 74 len 12288
56732024-02-03T17:11:57.570Z14/30 Write block 62 len 36864 data: 2 2 2 3 3 4 4 4 6
56742024-02-03T17:11:57.570Z15/30 Flush
56752024-02-03T17:11:57.576Z16/30 Write block 406 len 12288 data: 3 4 4
56762024-02-03T17:11:57.576Z17/30 Write block 327 len 20480 data: 5 5 6 4 5
56772024-02-03T17:11:57.576Z18/30 Write block 392 len 36864 data: 9 10 9 8 7 6 6 6 5
56782024-02-03T17:11:57.577Z19/30 Read block 259 len 12288
56792024-02-03T17:11:57.592Z20/30 Read block 363 len 32768
56802024-02-03T17:11:57.592Z21/30 Write block 28 len 8192 data: 7 5
56812024-02-03T17:11:57.592Z22/30 Read block 473 len 24576
56822024-02-03T17:11:57.596Z23/30 Flush
56832024-02-03T17:11:57.596Z24/30 Write block 586 len 8192 data: 6 5
56842024-02-03T17:11:57.596Z25/30 Write block 9 len 24576 data: 6 6 5 4 3 4
56852024-02-03T17:11:57.596Z26/30 Write block 151 len 4096 data: 4
56862024-02-03T17:11:57.596Z27/30 Write block 459 len 40960 data: 2 2 2 2 4 4 5 5 4 3
56872024-02-03T17:11:57.596Z28/30 Write block 259 len 40960 data: 5 6 6 5 6 7 8 8 9 9
56882024-02-03T17:11:57.596Z29/30 Read block 393 len 20480
56892024-02-03T17:11:57.596Z30/30 Write block 225 len 32768 data: 2 2 2 4 3 2 3 4
56902024-02-03T17:11:57.597Z----------------------------------------------------------------
56912024-02-03T17:11:57.597Z Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30
56922024-02-03T17:11:57.597ZStates: Active Active Active
56932024-02-03T17:11:57.597ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
56942024-02-03T17:11:57.597Z 1 Acked 1000 Write 2 Done Done Done false
56952024-02-03T17:11:57.597Z 2 Acked 1001 Read 3 Done Done Done false
56962024-02-03T17:11:57.597Z 3 Acked 1002 Write 9 Done Done Done false
56972024-02-03T17:11:57.597Z 4 Acked 1003 Write 2 Done Done Sent false
56982024-02-03T17:11:57.597Z 5 Acked 1004 Read 2 Done Done Sent false
56992024-02-03T17:11:57.597Z 6 Acked 1005 Read 5 Done Done Sent false
57002024-02-03T17:11:57.597Z 7 Acked 1006 Write 8 Done Done Sent false
57012024-02-03T17:11:57.597Z 8 Acked 1007 Read 10 Done Done Sent false
57022024-02-03T17:11:57.597Z 9 Acked 1008 Write 9 Done Done Sent false
57032024-02-03T17:11:57.598Z 10 Acked 1009 Read 2 Done Done Sent false
57042024-02-03T17:11:57.598Z 11 Acked 1010 Write 4 Done Done Sent false
57052024-02-03T17:11:57.598Z 12 Acked 1011 Write 10 Done Done Sent false
57062024-02-03T17:11:57.598Z 13 Acked 1012 Read 3 Done Done Sent false
57072024-02-03T17:11:57.598Z 14 Acked 1013 Write 9 Done Done Sent false
57082024-02-03T17:11:57.598Z 15 Acked 1014 Flush 0 Done Done Sent false
57092024-02-03T17:11:57.598Z 16 Acked 1015 Write 3 Done Done Sent false
57102024-02-03T17:11:57.598Z 17 Acked 1016 Write 5 Done Done Sent false
57112024-02-03T17:11:57.598Z 18 Acked 1017 Write 9 Done Done Sent false
57122024-02-03T17:11:57.598Z 19 Acked 1018 Read 3 Done Done Sent false
57132024-02-03T17:11:57.598Z 20 Acked 1019 Read 8 Done Done Sent false
57142024-02-03T17:11:57.598Z 21 Acked 1020 Write 2 Done Done Sent false
57152024-02-03T17:11:57.598Z 22 Acked 1021 Read 6 Done Done Sent false
57162024-02-03T17:11:57.598Z 23 Acked 1022 Flush 0 Done Done Sent false
57172024-02-03T17:11:57.598Z 24 Acked 1023 Write 2 Done Done Sent false
57182024-02-03T17:11:57.599Z 25 Acked 1024 Write 6 Done Done Sent false
57192024-02-03T17:11:57.599Z 26 Acked 1025 Write 1 Done Done Sent false
57202024-02-03T17:11:57.599Z 27 Acked 1026 Write 10 Done Done Sent false
57212024-02-03T17:11:57.599Z 28 Acked 1027 Write 10 Done Done Sent false
57222024-02-03T17:11:57.599Z 29 Acked 1028 Read 5 Done Done Sent false
57232024-02-03T17:11:57.599Z 30 Acked 1029 Write 8 Sent Sent Sent false
57242024-02-03T17:11:57.599Z STATES DS:0 DS:1 DS:2 TOTAL
57252024-02-03T17:11:57.599Z New 0 0 0 0
57262024-02-03T17:11:57.599Z Sent 1 1 27 29
57272024-02-03T17:11:57.599Z Done 29 29 3 61
57282024-02-03T17:11:57.599Z Skipped 0 0 0 0
57292024-02-03T17:11:57.599Z Error 0 0 0 0
57302024-02-03T17:11:57.599ZLast Flush: 1022 1022 0
57312024-02-03T17:11:57.599ZDownstairs last five completed:
57322024-02-03T17:11:57.599ZUpstairs last five completed: 30 29 28 27 26
57332024-02-03T17:11:57.599Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:11:57.143493189Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1337}
57342024-02-03T17:11:57.600ZWrote out file "/tmp/test_repair_verify.data"
57352024-02-03T17:11:57.604Z
57362024-02-03T17:11:57.611ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
57372024-02-03T17:11:57.640ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
57382024-02-03T17:11:57.640Z 0 000-019 39 39 39 67 67 67 T T F <---
57392024-02-03T17:11:57.641Z 1 020-039 52 52 44 93 93 80 F F F <---
57402024-02-03T17:11:57.641Z 2 040-059 52 52 38 92 92 66 F F T <---
57412024-02-03T17:11:57.641Z 3 060-079 52 52 51 92 92 91 F F F <---
57422024-02-03T17:11:57.644Z 4 080-099 52 52 51 92 92 91 F F F <---
57432024-02-03T17:11:57.644Z 5 100-119 49 49 49 86 86 86 F F F
57442024-02-03T17:11:57.644Z 6 120-139 52 52 42 92 92 78 F F F <---
57452024-02-03T17:11:57.644Z 7 140-159 51 51 51 91 91 91 T T F <---
57462024-02-03T17:11:57.644Z 8 160-179 52 52 47 92 92 84 F F F <---
57472024-02-03T17:11:57.644Z 9 180-199 46 46 46 83 83 83 F F F
57482024-02-03T17:11:57.644Z 10 200-219 50 50 50 90 90 90 F F F
57492024-02-03T17:11:57.644Z 11 220-239 49 49 49 86 86 86 T T F <---
57502024-02-03T17:11:57.644Z 12 240-259 49 49 49 86 86 86 T T F <---
57512024-02-03T17:11:57.645Z 13 260-279 52 52 49 92 92 86 T T F <---
57522024-02-03T17:11:57.645Z 14 280-299 49 49 49 86 86 86 F F F
57532024-02-03T17:11:57.645Z 15 300-319 52 52 49 92 92 86 F F F <---
57542024-02-03T17:11:57.645Z 16 320-339 52 52 50 93 93 88 F F F <---
57552024-02-03T17:11:57.645Z 17 340-359 44 44 44 80 80 80 F F F
57562024-02-03T17:11:57.645Z 18 360-379 50 50 50 89 89 89 F F F
57572024-02-03T17:11:57.645Z 19 380-399 52 52 50 93 93 90 F F F <---
57582024-02-03T17:11:57.645Z 20 400-419 52 52 50 93 93 90 F F T <---
57592024-02-03T17:11:57.645Z 21 420-439 52 52 51 92 92 91 F F T <---
57602024-02-03T17:11:57.645Z 22 440-459 46 46 46 83 83 83 T T F <---
57612024-02-03T17:11:57.645Z 23 460-479 51 51 51 91 91 91 T T F <---
57622024-02-03T17:11:57.645Z 24 480-499 51 51 51 91 91 91 F F F
57632024-02-03T17:11:57.645Z 25 500-519 45 45 45 81 81 81 F F F
57642024-02-03T17:11:57.645Z 26 520-539 51 51 51 91 91 91 F F F
57652024-02-03T17:11:57.645Z 27 540-559 43 43 43 79 79 79 F F F
57662024-02-03T17:11:57.645Z 28 560-579 46 46 46 82 82 82 F F F
57672024-02-03T17:11:57.645Z 29 580-599 49 49 49 86 86 86 T T F <---
57682024-02-03T17:11:57.645ZMax gen: 52, Max flush: 93
57692024-02-03T17:11:57.645ZError: Difference in extent metadata found!
57702024-02-03T17:11:57.649ZA Difference in extent metadata is expected here
57712024-02-03T17:11:57.649ZOn loop 26
57722024-02-03T17:11:57.649Z
57732024-02-03T17:11:57.653ZVerifying data now
57742024-02-03T17:12:04.251ZNo change after verify
57752024-02-03T17:12:04.251ZLoop: 26 Downstairs dump after verify (and repair):
57762024-02-03T17:12:04.285ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
57772024-02-03T17:12:04.285Z 0 000-019 53 53 53 94 94 94 F F F
57782024-02-03T17:12:04.285Z 1 020-039 52 52 52 93 93 93 F F F
57792024-02-03T17:12:04.285Z 2 040-059 52 52 52 92 92 92 F F F
57802024-02-03T17:12:04.285Z 3 060-079 52 52 52 92 92 92 F F F
57812024-02-03T17:12:04.290Z 4 080-099 52 52 52 92 92 92 F F F
57822024-02-03T17:12:04.291Z 5 100-119 49 49 49 86 86 86 F F F
57832024-02-03T17:12:04.291Z 6 120-139 52 52 52 92 92 92 F F F
57842024-02-03T17:12:04.291Z 7 140-159 53 53 53 94 94 94 F F F
57852024-02-03T17:12:04.291Z 8 160-179 52 52 52 92 92 92 F F F
57862024-02-03T17:12:04.291Z 9 180-199 46 46 46 83 83 83 F F F
57872024-02-03T17:12:04.291Z 10 200-219 50 50 50 90 90 90 F F F
57882024-02-03T17:12:04.291Z 11 220-239 53 53 53 94 94 94 F F F
57892024-02-03T17:12:04.291Z 12 240-259 53 53 53 94 94 94 F F F
57902024-02-03T17:12:04.291Z 13 260-279 53 53 53 94 94 94 F F F
57912024-02-03T17:12:04.291Z 14 280-299 49 49 49 86 86 86 F F F
57922024-02-03T17:12:04.291Z 15 300-319 52 52 52 92 92 92 F F F
57932024-02-03T17:12:04.291Z 16 320-339 52 52 52 93 93 93 F F F
57942024-02-03T17:12:04.291Z 17 340-359 44 44 44 80 80 80 F F F
57952024-02-03T17:12:04.291Z 18 360-379 50 50 50 89 89 89 F F F
57962024-02-03T17:12:04.291Z 19 380-399 52 52 52 93 93 93 F F F
57972024-02-03T17:12:04.291Z 20 400-419 52 52 52 93 93 93 F F F
57982024-02-03T17:12:04.291Z 21 420-439 52 52 52 92 92 92 F F F
57992024-02-03T17:12:04.291Z 22 440-459 53 53 53 94 94 94 F F F
58002024-02-03T17:12:04.291Z 23 460-479 53 53 53 94 94 94 F F F
58012024-02-03T17:12:04.291Z 24 480-499 51 51 51 91 91 91 F F F
58022024-02-03T17:12:04.291Z 25 500-519 45 45 45 81 81 81 F F F
58032024-02-03T17:12:04.291Z 26 520-539 51 51 51 91 91 91 F F F
58042024-02-03T17:12:04.291Z 27 540-559 43 43 43 79 79 79 F F F
58052024-02-03T17:12:04.292Z 28 560-579 46 46 46 82 82 82 F F F
58062024-02-03T17:12:04.292Z 29 580-599 53 53 53 94 94 94 F F F
58072024-02-03T17:12:04.292ZMax gen: 53, Max flush: 94
58082024-02-03T17:12:04.292Z
58092024-02-03T17:12:04.292ZBegin loop 27
58102024-02-03T17:12:04.292ZDownstairs to restart: 0
58112024-02-03T17:12:04.324Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.85892127Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345}
58122024-02-03T17:12:04.325Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.859455083Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345}
58132024-02-03T17:12:04.325Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.859544729Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345}
58142024-02-03T17:12:04.325Z{"msg":"Crucible 7cb0049d-0b18-42c5-ad70-2428e9aeefec has session id: 5889718a-b0b9-42e2-bdab-f24eb2785e71","v":0,"name":"crucible"Crucible runtime is spawned
58152024-02-03T17:12:04.325Z,"level":30,"time":"2024-02-03T17:12:03.859717676Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58162024-02-03T17:12:04.325Z{"msg":"Upstairs opts: Upstairs UUID: 7cb0049d-0b18-42c5-ad70-2428e9aeefec, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.859795026Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58172024-02-03T17:12:04.325Z{"msg":"Crucible stats registered with UUID: 7cb0049d-0b18-42c5-ad70-2428e9aeefec","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.859822586Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58182024-02-03T17:12:04.325Z{"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.859869716Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345}
58192024-02-03T17:12:04.325Z{"msg":"7cb0049d-0b18-42c5-ad70-2428e9aeefec active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.859890896Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58202024-02-03T17:12:04.325Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.85991167Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58212024-02-03T17:12:04.325Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.859966281Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58222024-02-03T17:12:04.325Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.860011506Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58232024-02-03T17:12:04.325Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.860092123Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58242024-02-03T17:12:04.325Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.860140557Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58252024-02-03T17:12:04.325Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.860223175Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58262024-02-03T17:12:04.325Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.860273205Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"":"io task","client":"2","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58272024-02-03T17:12:04.325Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.860349884Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"":"io task","client":"0","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58282024-02-03T17:12:04.325Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.860395042Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"":"io task","client":"1","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58292024-02-03T17:12:04.325Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.860442052Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"":"io task","client":"2","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58302024-02-03T17:12:04.325Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.860539555Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"":"io task","client":"1","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58312024-02-03T17:12:04.325Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.8608142Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"":"io task","client":"0","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58322024-02-03T17:12:04.326Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.862134661Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58332024-02-03T17:12:04.326Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.862207666Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58342024-02-03T17:12:04.326Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.86223456Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58352024-02-03T17:12:04.326Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.86225753Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58362024-02-03T17:12:04.326Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.86251625Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58372024-02-03T17:12:04.326Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.862621946Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58382024-02-03T17:12:04.326Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.863026847Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58392024-02-03T17:12:04.326Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.863254453Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58402024-02-03T17:12:04.326Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.86387151Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58412024-02-03T17:12:04.326Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.863970352Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58422024-02-03T17:12:04.326Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.864021312Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58432024-02-03T17:12:04.326Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.864128367Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58442024-02-03T17:12:04.326Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.86421905Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58452024-02-03T17:12:04.326Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.864270828Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58462024-02-03T17:12:04.326Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.864311139Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58472024-02-03T17:12:04.326Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.865098182Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58482024-02-03T17:12:04.326Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.865170373Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58492024-02-03T17:12:04.326Z{"msg":"[0]R flush_numbers[0..12]: [94, 93, 92, 92, 92, 86, 92, 94, 92, 83, 90, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.86520006Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58502024-02-03T17:12:04.326Z{"msg":"[0]R generation[0..12]: [53, 52, 52, 52, 52, 49, 52, 53, 52, 46, 50, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.865250353Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58512024-02-03T17:12:04.326Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.865277338Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58522024-02-03T17:12:04.326Z{"msg":"[1]R flush_numbers[0..12]: [94, 93, 92, 92, 92, 86, 92, 94, 92, 83, 90, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.865301406Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58532024-02-03T17:12:04.326Z{"msg":"[1]R generation[0..12]: [53, 52, 52, 52, 52, 49, 52, 53, 52, 46, 50, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.865349547Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58542024-02-03T17:12:04.326Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.86537572Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58552024-02-03T17:12:04.326Z{"msg":"[2]R flush_numbers[0..12]: [94, 93, 92, 92, 92, 86, 92, 94, 92, 83, 90, 94]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
58562024-02-03T17:12:04.327Z2024-02-03T17:12:03.865399236Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58572024-02-03T17:12:04.327Z{"msg":"[2]R generation[0..12]: [53, 52, 52, 52, 52, 49, 52, 53, 52, 46, 50, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.865496587Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58582024-02-03T17:12:04.327Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.865552067Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58592024-02-03T17:12:04.327Z{"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.865577187Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58602024-02-03T17:12:04.327Z{"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.865599569Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58612024-02-03T17:12:04.327Z{"msg":"Next flush: 95","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.865645283Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58622024-02-03T17:12:04.327Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.865669316Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58632024-02-03T17:12:04.327Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.865691624Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58642024-02-03T17:12:04.327Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.865736522Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58652024-02-03T17:12:04.327Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.865760551Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58662024-02-03T17:12:04.327Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.865873332Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58672024-02-03T17:12:04.334Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.86592183Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58682024-02-03T17:12:04.334Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.866060824Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
58692024-02-03T17:12:04.334Zip-10-150-1-9.us-west-2.compute.internal","pid":1345,"session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58702024-02-03T17:12:04.334Z{"msg":"7cb0049d-0b18-42c5-ad70-2428e9aeefec is now active with session: 5889718a-b0b9-42e2-bdab-f24eb2785e71","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.866225266Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58712024-02-03T17:12:04.334Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.866288559Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345,"session_id":"5889718a-b0b9-42e2-bdab-f24eb2785e71"}
58722024-02-03T17:12:04.334Z{"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:03.866312696Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345}
58732024-02-03T17:12:04.334ZLoading write count information from file "/tmp/test_repair_verify.data"
58742024-02-03T17:12:04.335ZRun Repair workload
58752024-02-03T17:12:04.335Z01/30 Write block 196 len 16384 data: 4 4 4 3
58762024-02-03T17:12:04.335Z02/30 Write block 202 len 20480 data: 4 4 3 5 5
58772024-02-03T17:12:04.335Z03/30 Read block 18 len 28672
58782024-02-03T17:12:04.335Z04/30 Read block 347 len 40960
58792024-02-03T17:12:04.335Z05/30 Write block 357 len 28672 data: 6 6 7 7 8 5 4
58802024-02-03T17:12:04.335Z06/30 Write block 588 len 28672 data: 5 4 4 3 5 5 5
58812024-02-03T17:12:04.335Z07/30 Read block 517 len 8192
58822024-02-03T17:12:04.335Z08/30 Write block 66 len 12288 data: 4 5 5
58832024-02-03T17:12:04.335Z09/30 Write block 268 len 36864 data: 10 7 7 6 6 6 5 7 6
58842024-02-03T17:12:04.335Z10/30 Write block 355 len 4096 data: 7
58852024-02-03T17:12:04.346Z11/30 Write block 216 len 40960 data: 5 4 3 3 2 3 3 3 2 3
58862024-02-03T17:12:04.346Z12/30 Write block 501 len 16384 data: 5 5 5 6
58872024-02-03T17:12:04.347Z13/30 Read block 423 len 36864
58882024-02-03T17:12:04.347Z14/30 Read block 158 len 36864
58892024-02-03T17:12:04.351Z15/30 Write block 230 len 24576 data: 3 4 5 3 4 4
58902024-02-03T17:12:04.351Z16/30 Read block 169 len 24576
58912024-02-03T17:12:04.351Z17/30 Write block 585 len 28672 data: 3 7 6 6 5 5 4
58922024-02-03T17:12:04.351Z18/30 Write block 198 len 32768 data: 5 4 3 3 5 5 4 6
58932024-02-03T17:12:04.352Z19/30 Write block 512 len 24576 data: 6 6 5 5 5 5
58942024-02-03T17:12:04.352Z20/30 Read block 115 len 20480
58952024-02-03T17:12:04.352Z21/30 Read block 45 len 36864
58962024-02-03T17:12:04.359Z22/30 Read block 207 len 36864
58972024-02-03T17:12:04.359Z23/30 Read block 25 len 20480
58982024-02-03T17:12:04.359Z24/30 Read block 361 len 32768
58992024-02-03T17:12:04.366Z25/30 Read block 126 len 8192
59002024-02-03T17:12:04.366Z26/30 Read block 371 len 16384
59012024-02-03T17:12:04.366Z27/30 Write block 520 len 32768 data: 4 3 3 4 4 6 6 7
59022024-02-03T17:12:04.366Z28/30 Read block 8 len 4096
59032024-02-03T17:12:04.366Z29/30 Read block 245 len 36864
59042024-02-03T17:12:04.366Z30/30 Read block 508 len 32768
59052024-02-03T17:12:04.375Z----------------------------------------------------------------
59062024-02-03T17:12:04.375Z Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:30
59072024-02-03T17:12:04.375ZStates: Active Active Active
59082024-02-03T17:12:04.375ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
59092024-02-03T17:12:04.375Z 1 Acked 1000 Write 4 Done Done Done false
59102024-02-03T17:12:04.375Z 2 Acked 1001 Write 5 Done Done Done false
59112024-02-03T17:12:04.375Z 3 Acked 1002 Read 7 Done Done Done false
59122024-02-03T17:12:04.375Z 4 Acked 1003 Read 10 Sent Done Done false
59132024-02-03T17:12:04.375Z 5 Acked 1004 Write 7 Sent Done Done false
59142024-02-03T17:12:04.375Z 6 Acked 1005 Write 7 Sent Done Done false
59152024-02-03T17:12:04.375Z 7 Acked 1006 Read 2 Sent Done Done false
59162024-02-03T17:12:04.375Z 8 Acked 1007 Write 3 Sent Done Done false
59172024-02-03T17:12:04.375Z 9 Acked 1008 Write 9 Sent Done Done false
59182024-02-03T17:12:04.375Z 10 Acked 1009 Write 1 Sent Done Done false
59192024-02-03T17:12:04.375Z 11 Acked 1010 Write 10 Sent Done Done false
59202024-02-03T17:12:04.375Z 12 Acked 1011 Write 4 Sent Done Done false
59212024-02-03T17:12:04.376Z 13 Acked 1012 Read 9 Sent Done Done false
59222024-02-03T17:12:04.376Z 14 Acked 1013 Read 9 Sent Done Done false
59232024-02-03T17:12:04.376Z 15 Acked 1014 Write 6 Sent Done Done false
59242024-02-03T17:12:04.376Z 16 Acked 1015 Read 6 Sent Done Done false
59252024-02-03T17:12:04.376Z 17 Acked 1016 Write 7 Sent Done Done false
59262024-02-03T17:12:04.376Z 18 Acked 1017 Write 8 Sent Done Done false
59272024-02-03T17:12:04.376Z 19 Acked 1018 Write 6 Sent Done Done false
59282024-02-03T17:12:04.376Z 20 Acked 1019 Read 5 Sent Done Done false
59292024-02-03T17:12:04.376Z 21 Acked 1020 Read 9 Sent Done Done false
59302024-02-03T17:12:04.376Z 22 Acked 1021 Read 9 Sent Done Done false
59312024-02-03T17:12:04.376Z 23 Acked 1022 Read 5 Sent Done Done false
59322024-02-03T17:12:04.376Z 24 Acked 1023 Read 8 Sent Done Done false
59332024-02-03T17:12:04.376Z 25 Acked 1024 Read 2 Sent Done Done false
59342024-02-03T17:12:04.376Z 26 Acked 1025 Read 4 Sent Done Done false
59352024-02-03T17:12:04.376Z 27 Acked 1026 Write 8 Sent Done Done false
59362024-02-03T17:12:04.376Z 28 Acked 1027 Read 1 Sent Done Done false
59372024-02-03T17:12:04.376Z 29 Acked 1028 Read 9 Sent Done Done false
59382024-02-03T17:12:04.376Z 30 Acked 1029 Read 8 Sent Sent Done false
59392024-02-03T17:12:04.376Z STATES DS:0 DS:1 DS:2 TOTAL
59402024-02-03T17:12:04.376Z New 0 0 0 0
59412024-02-03T17:12:04.376Z Sent 27 1 0 28
59422024-02-03T17:12:04.376Z Done 3 29 30 62
59432024-02-03T17:12:04.376Z Skipped 0 0 0 0
59442024-02-03T17:12:04.376Z Error 0 0 0 0
59452024-02-03T17:12:04.376ZLast Flush: 0 0 0
59462024-02-03T17:12:04.377ZDownstairs last five completed:
59472024-02-03T17:12:04.377ZUpstairs last five completed: 30 29 28 27 26
59482024-02-03T17:12:04.377Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:12:03.914963441Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1345}
59492024-02-03T17:12:04.377ZWrote out file "/tmp/test_repair_verify.data"
59502024-02-03T17:12:04.389Z
59512024-02-03T17:12:04.390ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
59522024-02-03T17:12:04.409ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
59532024-02-03T17:12:04.409Z 0 000-019 53 53 53 94 94 94 F F F
59542024-02-03T17:12:04.409Z 1 020-039 52 52 52 93 93 93 F F F
59552024-02-03T17:12:04.409Z 2 040-059 52 52 52 92 92 92 F F F
59562024-02-03T17:12:04.409Z 3 060-079 52 52 52 92 92 92 F T T <---
59572024-02-03T17:12:04.409Z 4 080-099 52 52 52 92 92 92 F F F
59582024-02-03T17:12:04.409Z 5 100-119 49 49 49 86 86 86 F F F
59592024-02-03T17:12:04.409Z 6 120-139 52 52 52 92 92 92 F F F
59602024-02-03T17:12:04.409Z 7 140-159 53 53 53 94 94 94 F F F
59612024-02-03T17:12:04.419Z 8 160-179 52 52 52 92 92 92 F F F
59622024-02-03T17:12:04.419Z 9 180-199 46 46 46 83 83 83 T T T
59632024-02-03T17:12:04.419Z 10 200-219 50 50 50 90 90 90 T T T
59642024-02-03T17:12:04.420Z 11 220-239 53 53 53 94 94 94 F T T <---
59652024-02-03T17:12:04.420Z 12 240-259 53 53 53 94 94 94 F F F
59662024-02-03T17:12:04.420Z 13 260-279 53 53 53 94 94 94 F T T <---
59672024-02-03T17:12:04.420Z 14 280-299 49 49 49 86 86 86 F F F
59682024-02-03T17:12:04.420Z 15 300-319 52 52 52 92 92 92 F F F
59692024-02-03T17:12:04.420Z 16 320-339 52 52 52 93 93 93 F F F
59702024-02-03T17:12:04.421Z 17 340-359 44 44 44 80 80 80 F T T <---
59712024-02-03T17:12:04.421Z 18 360-379 50 50 50 89 89 89 F T T <---
59722024-02-03T17:12:04.421Z 19 380-399 52 52 52 93 93 93 F F F
59732024-02-03T17:12:04.421Z 20 400-419 52 52 52 93 93 93 F F F
59742024-02-03T17:12:04.421Z 21 420-439 52 52 52 92 92 92 F F F
59752024-02-03T17:12:04.421Z 22 440-459 53 53 53 94 94 94 F F F
59762024-02-03T17:12:04.421Z 23 460-479 53 53 53 94 94 94 F F F
59772024-02-03T17:12:04.421Z 24 480-499 51 51 51 91 91 91 F F F
59782024-02-03T17:12:04.421Z 25 500-519 45 45 45 81 81 81 F T T <---
59792024-02-03T17:12:04.421Z 26 520-539 51 51 51 91 91 91 F T T <---
59802024-02-03T17:12:04.421Z 27 540-559 43 43 43 79 79 79 F F F
59812024-02-03T17:12:04.421Z 28 560-579 46 46 46 82 82 82 F F F
59822024-02-03T17:12:04.421Z 29 580-599 53 53 53 94 94 94 F T T <---
59832024-02-03T17:12:04.421ZMax gen: 53, Max flush: 94
59842024-02-03T17:12:04.422ZError: Difference in extent metadata found!
59852024-02-03T17:12:04.422ZA Difference in extent metadata is expected here
59862024-02-03T17:12:04.422ZOn loop 27
59872024-02-03T17:12:04.422Z
59882024-02-03T17:12:04.422ZVerifying data now
59892024-02-03T17:12:10.500ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
59902024-02-03T17:12:10.507Z200a201
59912024-02-03T17:12:10.507Z> 5,
59922024-02-03T17:12:10.507Z203,206d203
59932024-02-03T17:12:10.507Z< 2,
59942024-02-03T17:12:10.507Z< 2,
59952024-02-03T17:12:10.508Z< 4,
59962024-02-03T17:12:10.508Z< 4,
59972024-02-03T17:12:10.508Z210a208,209
59982024-02-03T17:12:10.508Z> 6,
59992024-02-03T17:12:10.508Z> 5,
60002024-02-03T17:12:10.508Z212a212
60012024-02-03T17:12:10.508Z> 4,
60022024-02-03T17:12:10.508Z218a219
60032024-02-03T17:12:10.508Z> 5,
60042024-02-03T17:12:10.508Z220a222
60052024-02-03T17:12:10.508Z> 3,
60062024-02-03T17:12:10.508Z222,223d223
60072024-02-03T17:12:10.508Z< 2,
60082024-02-03T17:12:10.509Z< 2,
60092024-02-03T17:12:10.509Zdiff found after verify
60102024-02-03T17:12:10.513ZLoop: 27 Downstairs dump after verify (and repair):
60112024-02-03T17:12:10.561ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
60122024-02-03T17:12:10.561Z 0 000-019 53 53 53 94 94 94 F F F
60132024-02-03T17:12:10.561Z 1 020-039 52 52 52 93 93 93 F F F
60142024-02-03T17:12:10.561Z 2 040-059 52 52 52 92 92 92 F F F
60152024-02-03T17:12:10.561Z 3 060-079 54 54 54 95 95 95 F F F
60162024-02-03T17:12:10.561Z 4 080-099 52 52 52 92 92 92 F F F
60172024-02-03T17:12:10.561Z 5 100-119 49 49 49 86 86 86 F F F
60182024-02-03T17:12:10.561Z 6 120-139 52 52 52 92 92 92 F F F
60192024-02-03T17:12:10.561Z 7 140-159 53 53 53 94 94 94 F F F
60202024-02-03T17:12:10.569Z 8 160-179 52 52 52 92 92 92 F F F
60212024-02-03T17:12:10.569Z 9 180-199 54 54 54 95 95 95 F F F
60222024-02-03T17:12:10.569Z 10 200-219 54 54 54 95 95 95 F F F
60232024-02-03T17:12:10.569Z 11 220-239 54 54 54 95 95 95 F F F
60242024-02-03T17:12:10.569Z 12 240-259 53 53 53 94 94 94 F F F
60252024-02-03T17:12:10.569Z 13 260-279 54 54 54 95 95 95 F F F
60262024-02-03T17:12:10.570Z 14 280-299 49 49 49 86 86 86 F F F
60272024-02-03T17:12:10.570Z 15 300-319 52 52 52 92 92 92 F F F
60282024-02-03T17:12:10.570Z 16 320-339 52 52 52 93 93 93 F F F
60292024-02-03T17:12:10.570Z 17 340-359 54 54 54 95 95 95 F F F
60302024-02-03T17:12:10.570Z 18 360-379 54 54 54 95 95 95 F F F
60312024-02-03T17:12:10.570Z 19 380-399 52 52 52 93 93 93 F F F
60322024-02-03T17:12:10.570Z 20 400-419 52 52 52 93 93 93 F F F
60332024-02-03T17:12:10.570Z 21 420-439 52 52 52 92 92 92 F F F
60342024-02-03T17:12:10.570Z 22 440-459 53 53 53 94 94 94 F F F
60352024-02-03T17:12:10.570Z 23 460-479 53 53 53 94 94 94 F F F
60362024-02-03T17:12:10.570Z 24 480-499 51 51 51 91 91 91 F F F
60372024-02-03T17:12:10.570Z 25 500-519 54 54 54 95 95 95 F F F
60382024-02-03T17:12:10.571Z 26 520-539 54 54 54 95 95 95 F F F
60392024-02-03T17:12:10.571Z 27 540-559 43 43 43 79 79 79 F F F
60402024-02-03T17:12:10.571Z 28 560-579 46 46 46 82 82 82 F F F
60412024-02-03T17:12:10.571Z 29 580-599 54 54 54 95 95 95 F F F
60422024-02-03T17:12:10.571ZMax gen: 54, Max flush: 95
60432024-02-03T17:12:10.571Z
60442024-02-03T17:12:10.571ZBegin loop 28
60452024-02-03T17:12:10.571ZDownstairs to restart: 0
60462024-02-03T17:12:10.584Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.117661954Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356}
60472024-02-03T17:12:10.584Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.118136465Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356}
60482024-02-03T17:12:10.585Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
60492024-02-03T17:12:10.585Z,"time":"2024-02-03T17:12:10.118204363Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356}
60502024-02-03T17:12:10.585Z{"msg":"Crucible 03517de7-9e5e-4014-b16d-b0a9d37c9166 has session id: 7027bbe9-edbc-431f-8d9d-eedd04808ceb","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.118249328Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60512024-02-03T17:12:10.585Z{"msg":"Upstairs opts: Upstairs UUID: 03517de7-9e5e-4014-b16d-b0a9d37c9166, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.118334538Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60522024-02-03T17:12:10.585Z{"msg":"Crucible stats registered with UUID: 03517de7-9e5e-4014-b16d-b0a9d37c9166","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.118380778Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60532024-02-03T17:12:10.585Z{"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.118441515Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356}
60542024-02-03T17:12:10.585Z{"msg":"03517de7-9e5e-4014-b16d-b0a9d37c9166 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.11847972Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60552024-02-03T17:12:10.585Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.118545098Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"client":"0","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60562024-02-03T17:12:10.585Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.118588906Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"client":"0","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60572024-02-03T17:12:10.585Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.118626115Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"client":"1","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60582024-02-03T17:12:10.585Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.118665004Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"client":"1","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60592024-02-03T17:12:10.585Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.118851777Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"client":"2","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60602024-02-03T17:12:10.585Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.118955491Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"client":"2","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60612024-02-03T17:12:10.585Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.119005084Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"":"io task","client":"0","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60622024-02-03T17:12:10.585Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.119073929Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"":"io task","client":"1","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60632024-02-03T17:12:10.585Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.119191971Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"":"io task","client":"2","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60642024-02-03T17:12:10.585Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.119239765Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"":"io task","client":"2","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60652024-02-03T17:12:10.585Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.119284265Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"":"io task","client":"0","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60662024-02-03T17:12:10.585Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.119371597Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"":"io task","client":"1","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60672024-02-03T17:12:10.585Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.120419747Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"client":"1","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60682024-02-03T17:12:10.585Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.12051516Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"client":"1","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60692024-02-03T17:12:10.586Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.120543628Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"client":"2","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60702024-02-03T17:12:10.586Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.120566456Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"client":"2","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60712024-02-03T17:12:10.586Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.121380727Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"client":"0","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60722024-02-03T17:12:10.586Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.121422087Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"client":"0","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60732024-02-03T17:12:10.586Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.12169833Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"client":"1","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60742024-02-03T17:12:10.586Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.121763952Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"client":"2","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60752024-02-03T17:12:10.586Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.122413801Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"client":"0","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60762024-02-03T17:12:10.586Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.122645897Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"client":"1","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60772024-02-03T17:12:10.586Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.122723858Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"client":"1","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60782024-02-03T17:12:10.586Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.122836448Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60792024-02-03T17:12:10.586Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.122895624Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"client":"2","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60802024-02-03T17:12:10.586Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.122939862Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"client":"2","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60812024-02-03T17:12:10.586Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.123005239Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60822024-02-03T17:12:10.586Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.123282873Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"client":"0","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60832024-02-03T17:12:10.586Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.123408008Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"client":"0","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60842024-02-03T17:12:10.586Z{"msg":"[0]R flush_numbers[0..12]: [94, 93, 92, 95, 92, 86, 92, 94, 92, 95, 95, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.123436235Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"":"downstairs","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60852024-02-03T17:12:10.586Z{"msg":"[0]R generation[0..12]: [53, 52, 52, 54, 52, 49, 52, 53, 52, 54, 54, 54]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.123459621Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"":"downstairs","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60862024-02-03T17:12:10.586Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.123485101Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"":"downstairs","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60872024-02-03T17:12:10.586Z{"msg":"[1]R flush_numbers[0..12]: [94, 93, 92, 95, 92, 86, 92, 94, 92, 95, 95, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.123534257Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"":"downstairs","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60882024-02-03T17:12:10.586Z{"msg":"[1]R generation[0..12]: [53, 52, 52, 54, 52, 49, 52, 53, 52, 54, 54, 54]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.123559941Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"":"downstairs","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60892024-02-03T17:12:10.586Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.123583715Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"":"downstairs","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60902024-02-03T17:12:10.586Z{"msg":"[2]R flush_numbers[0..12]: [94, 93, 92, 95, 92, 86, 92, 94, 92, 95, 95, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.123613308Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"":"downstairs","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60912024-02-03T17:12:10.586Z{"msg":"[2]R generation[0..12]: [53, 52, 52, 54, 52, 49, 52, 53, 52, 54, 54, 54]","v":0Wait for a query_work_queue command to finish before sending IO
60922024-02-03T17:12:10.586Z,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.123657027Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"":"downstairs","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60932024-02-03T17:12:10.587Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.123715785Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"":"downstairs","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60942024-02-03T17:12:10.587Z{"msg":"Max found gen is 55","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.12374093Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"":"downstairs","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60952024-02-03T17:12:10.587Z{"msg":"Generation requested: 56 >= found:55","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.123788592Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"":"downstairs","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60962024-02-03T17:12:10.587Z{"msg":"Next flush: 96","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.123812248Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"":"downstairs","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60972024-02-03T17:12:10.587Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.12383547Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"":"downstairs","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60982024-02-03T17:12:10.587Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.12385815Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
60992024-02-03T17:12:10.587Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.123878311Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"client":"0","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
61002024-02-03T17:12:10.587Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.123924816Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"client":"1","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
61012024-02-03T17:12:10.587Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.123949031Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"client":"2","":"downstairs","session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
61022024-02-03T17:12:10.587Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.123996977Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
61032024-02-03T17:12:10.587Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.124019597Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
61042024-02-03T17:12:10.587Z{"msg":"03517de7-9e5e-4014-b16d-b0a9d37c9166 is now active with session: 7027bbe9-edbc-431f-8d9d-eedd04808ceb","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.124043087Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
61052024-02-03T17:12:10.587Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.124062871Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356,"session_id":"7027bbe9-edbc-431f-8d9d-eedd04808ceb"}
61062024-02-03T17:12:10.587Z{"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:10.124108927Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356}
61072024-02-03T17:12:10.587ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
61082024-02-03T17:12:10.587ZLoading write count information from file "/tmp/test_repair_verify.data"
61092024-02-03T17:12:10.587ZRun Repair workload
61102024-02-03T17:12:10.591Z01/30 Write block 301 len 40960 data: 5 5 5 7 6 8 6 6 6 7
61112024-02-03T17:12:10.591Z02/30 Read block 422 len 16384
61122024-02-03T17:12:10.591Z03/30 Write block 440 len 36864 data: 5 7 8 6 5 5 6 6 6
61132024-02-03T17:12:10.591Z04/30 Read block 44 len 28672
61142024-02-03T17:12:10.591Z05/30 Read block 96 len 8192
61152024-02-03T17:12:10.592Z06/30 Flush
61162024-02-03T17:12:10.592Z07/30 Write block 136 len 36864 data: 2 2 2 2 2 4 4 4 4
61172024-02-03T17:12:10.592Z08/30 Read block 314 len 36864
61182024-02-03T17:12:10.595Z09/30 Write block 442 len 32768 data: 9 7 6 6 7 7 7 5
61192024-02-03T17:12:10.595Z10/30 Read block 23 len 12288
61202024-02-03T17:12:10.598Z11/30 Read block 33 len 8192
61212024-02-03T17:12:10.598Z12/30 Read block 385 len 4096
61222024-02-03T17:12:10.601Z13/30 Write block 166 len 12288 data: 10 10 6
61232024-02-03T17:12:10.601Z14/30 Flush
61242024-02-03T17:12:10.605Z15/30 Write block 22 len 32768 data: 7 7 7 7 7 7 8 6
61252024-02-03T17:12:10.608Z16/30 Write block 338 len 16384 data: 6 6 7 5
61262024-02-03T17:12:10.608Z17/30 Write block 118 len 40960 data: 7 6 5 4 5 4 3 3 3 4
61272024-02-03T17:12:10.608Z18/30 Write block 327 len 16384 data: 6 6 7 5
61282024-02-03T17:12:10.611Z19/30 Read block 455 len 40960
61292024-02-03T17:12:10.611Z20/30 Read block 285 len 36864
61302024-02-03T17:12:10.614Z21/30 Write block 549 len 24576 data: 6 6 5 5 7 6
61312024-02-03T17:12:10.614Z22/30 Write block 412 len 8192 data: 5 5
61322024-02-03T17:12:10.617Z23/30 Read block 94 len 36864
61332024-02-03T17:12:10.617Z24/30 Read block 74 len 20480
61342024-02-03T17:12:10.620Z25/30 Read block 40 len 40960
61352024-02-03T17:12:10.620Z26/30 Read block 70 len 32768
61362024-02-03T17:12:10.624Z27/30 Read block 304 len 32768
61372024-02-03T17:12:10.624Z28/30 Write block 429 len 28672 data: 7 7 6 7 6 6 7
61382024-02-03T17:12:10.628Z29/30 Write block 17 len 32768 data: 4 5 6 4 5 8 8 8
61392024-02-03T17:12:10.628Z30/30 Write block 547 len 32768 data: 8 6 7 7 6 6 8 7
61402024-02-03T17:12:10.628Z----------------------------------------------------------------
61412024-02-03T17:12:10.628Z Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30
61422024-02-03T17:12:10.628ZStates: Active Active Active
61432024-02-03T17:12:10.628ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
61442024-02-03T17:12:10.628Z 1 Acked 1000 Write 10 Done Done Done false
61452024-02-03T17:12:10.628Z 2 Acked 1001 Read 4 Done Done Done false
61462024-02-03T17:12:10.628Z 3 Acked 1002 Write 9 Done Done Done false
61472024-02-03T17:12:10.628Z 4 Acked 1003 Read 7 Sent Done Done false
61482024-02-03T17:12:10.628Z 5 Acked 1004 Read 2 Sent Done Done false
61492024-02-03T17:12:10.628Z 6 Acked 1005 Flush 0 Sent Done Done false
61502024-02-03T17:12:10.628Z 7 Acked 1006 Write 9 Sent Done Done false
61512024-02-03T17:12:10.628Z 8 Acked 1007 Read 9 Sent Done Done false
61522024-02-03T17:12:10.628Z 9 Acked 1008 Write 8 Sent Done Done false
61532024-02-03T17:12:10.628Z 10 Acked 1009 Read 3 Sent Done Done false
61542024-02-03T17:12:10.628Z 11 Acked 1010 Read 2 Sent Done Done false
61552024-02-03T17:12:10.629Z 12 Acked 1011 Read 1 Sent Done Done false
61562024-02-03T17:12:10.629Z 13 Acked 1012 Write 3 Sent Done Done false
61572024-02-03T17:12:10.629Z 14 Acked 1013 Flush 0 Sent Done Done false
61582024-02-03T17:12:10.629Z 15 Acked 1014 Write 8 Sent Done Done false
61592024-02-03T17:12:10.629Z 16 Acked 1015 Write 4 Sent Done Done false
61602024-02-03T17:12:10.629Z 17 Acked 1016 Write 10 Sent Done Done false
61612024-02-03T17:12:10.629Z 18 Acked 1017 Write 4 Sent Done Done false
61622024-02-03T17:12:10.629Z 19 Acked 1018 Read 10 Sent Done Done false
61632024-02-03T17:12:10.629Z 20 Acked 1019 Read 9 Sent Done Done false
61642024-02-03T17:12:10.629Z 21 Acked 1020 Write 6 Sent Done Done false
61652024-02-03T17:12:10.629Z 22 Acked 1021 Write 2 Sent Done Done false
61662024-02-03T17:12:10.629Z 23 Acked 1022 Read 9 Sent Done Done false
61672024-02-03T17:12:10.629Z 24 Acked 1023 Read 5 Sent Done Done false
61682024-02-03T17:12:10.629Z 25 Acked 1024 Read 10 Sent Done Done false
61692024-02-03T17:12:10.629Z 26 Acked 1025 Read 8 Sent Done Done false
61702024-02-03T17:12:10.629Z 27 Acked 1026 Read 8 Sent Done Done false
61712024-02-03T17:12:10.629Z 28 Acked 1027 Write 7 Sent Done Done false
61722024-02-03T17:12:10.629Z 29 Acked 1028 Write 8 Sent Done Done false
61732024-02-03T17:12:10.629Z 30 Acked 1029 Write 8 Sent Sent Sent false
61742024-02-03T17:12:10.629Z STATES DS:0 DS:1 DS:2 TOTAL
61752024-02-03T17:12:10.629Z New 0 0 0 0
61762024-02-03T17:12:10.629Z Sent 27 1 1 29
61772024-02-03T17:12:10.629Z Done 3 29 29 61
61782024-02-03T17:12:10.629Z Skipped 0 0 0 0
61792024-02-03T17:12:10.629Z Error 0 0 0 0
61802024-02-03T17:12:10.630ZLast Flush: 0 1013 1013
61812024-02-03T17:12:10.630ZDownstairs last five completed:
61822024-02-03T17:12:10.630ZUpstairs last five completed: 30 29 28 27 26
61832024-02-03T17:12:10.633Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:12:10.178030769Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1356}
61842024-02-03T17:12:10.633ZWrote out file "/tmp/test_repair_verify.data"
61852024-02-03T17:12:10.636Z
61862024-02-03T17:12:10.642ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
61872024-02-03T17:12:10.671ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
61882024-02-03T17:12:10.671Z 0 000-019 53 53 53 94 94 94 F T T <---
61892024-02-03T17:12:10.675Z 1 020-039 52 52 52 93 93 93 F T T <---
61902024-02-03T17:12:10.675Z 2 040-059 52 52 52 92 92 92 F F F
61912024-02-03T17:12:10.675Z 3 060-079 54 54 54 95 95 95 F F F
61922024-02-03T17:12:10.675Z 4 080-099 52 52 52 92 92 92 F F F
61932024-02-03T17:12:10.675Z 5 100-119 49 49 49 86 86 86 F T T <---
61942024-02-03T17:12:10.675Z 6 120-139 52 56 56 92 97 97 F T T <---
61952024-02-03T17:12:10.675Z 7 140-159 53 56 56 94 97 97 F F F <---
61962024-02-03T17:12:10.675Z 8 160-179 52 56 56 92 97 97 F F F <---
61972024-02-03T17:12:10.675Z 9 180-199 54 54 54 95 95 95 F F F
61982024-02-03T17:12:10.675Z 10 200-219 54 54 54 95 95 95 F F F
61992024-02-03T17:12:10.675Z 11 220-239 54 54 54 95 95 95 F F F
62002024-02-03T17:12:10.675Z 12 240-259 53 53 53 94 94 94 F F F
62012024-02-03T17:12:10.675Z 13 260-279 54 54 54 95 95 95 F F F
62022024-02-03T17:12:10.675Z 14 280-299 49 49 49 86 86 86 F F F
62032024-02-03T17:12:10.675Z 15 300-319 52 56 56 92 96 96 T F F <---
62042024-02-03T17:12:10.675Z 16 320-339 52 52 52 93 93 93 F T T <---
62052024-02-03T17:12:10.675Z 17 340-359 54 54 54 95 95 95 F T T <---
62062024-02-03T17:12:10.675Z 18 360-379 54 54 54 95 95 95 F F F
62072024-02-03T17:12:10.675Z 19 380-399 52 52 52 93 93 93 F F F
62082024-02-03T17:12:10.675Z 20 400-419 52 52 52 93 93 93 F T T <---
62092024-02-03T17:12:10.676Z 21 420-439 52 52 52 92 92 92 F T T <---
62102024-02-03T17:12:10.676Z 22 440-459 53 56 56 94 97 97 T F F <---
62112024-02-03T17:12:10.676Z 23 460-479 53 53 53 94 94 94 F F F
62122024-02-03T17:12:10.676Z 24 480-499 51 51 51 91 91 91 F F F
62132024-02-03T17:12:10.676Z 25 500-519 54 54 54 95 95 95 F F F
62142024-02-03T17:12:10.676Z 26 520-539 54 54 54 95 95 95 F F F
62152024-02-03T17:12:10.676Z 27 540-559 43 43 43 79 79 79 F T T <---
62162024-02-03T17:12:10.676Z 28 560-579 46 46 46 82 82 82 F F F
62172024-02-03T17:12:10.676Z 29 580-599 54 54 54 95 95 95 F F F
62182024-02-03T17:12:10.676ZMax gen: 56, Max flush: 97
62192024-02-03T17:12:10.676ZError: Difference in extent metadata found!
62202024-02-03T17:12:10.680ZA Difference in extent metadata is expected here
62212024-02-03T17:12:10.680ZOn loop 28
62222024-02-03T17:12:10.680Z
62232024-02-03T17:12:10.680ZVerifying data now
62242024-02-03T17:12:16.832ZNo change after verify
62252024-02-03T17:12:16.832ZLoop: 28 Downstairs dump after verify (and repair):
62262024-02-03T17:12:16.856ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
62272024-02-03T17:12:16.856Z 0 000-019 57 57 57 98 98 98 F F F
62282024-02-03T17:12:16.856Z 1 020-039 57 57 57 98 98 98 F F F
62292024-02-03T17:12:16.856Z 2 040-059 52 52 52 92 92 92 F F F
62302024-02-03T17:12:16.856Z 3 060-079 54 54 54 95 95 95 F F F
62312024-02-03T17:12:16.856Z 4 080-099 52 52 52 92 92 92 F F F
62322024-02-03T17:12:16.856Z 5 100-119 57 57 57 98 98 98 F F F
62332024-02-03T17:12:16.856Z 6 120-139 57 57 57 98 98 98 F F F
62342024-02-03T17:12:16.862Z 7 140-159 56 56 56 97 97 97 F F F
62352024-02-03T17:12:16.862Z 8 160-179 56 56 56 97 97 97 F F F
62362024-02-03T17:12:16.862Z 9 180-199 54 54 54 95 95 95 F F F
62372024-02-03T17:12:16.862Z 10 200-219 54 54 54 95 95 95 F F F
62382024-02-03T17:12:16.862Z 11 220-239 54 54 54 95 95 95 F F F
62392024-02-03T17:12:16.863Z 12 240-259 53 53 53 94 94 94 F F F
62402024-02-03T17:12:16.863Z 13 260-279 54 54 54 95 95 95 F F F
62412024-02-03T17:12:16.863Z 14 280-299 49 49 49 86 86 86 F F F
62422024-02-03T17:12:16.863Z 15 300-319 56 56 56 96 96 96 F F F
62432024-02-03T17:12:16.863Z 16 320-339 57 57 57 98 98 98 F F F
62442024-02-03T17:12:16.863Z 17 340-359 57 57 57 98 98 98 F F F
62452024-02-03T17:12:16.863Z 18 360-379 54 54 54 95 95 95 F F F
62462024-02-03T17:12:16.863Z 19 380-399 52 52 52 93 93 93 F F F
62472024-02-03T17:12:16.863Z 20 400-419 57 57 57 98 98 98 F F F
62482024-02-03T17:12:16.863Z 21 420-439 57 57 57 98 98 98 F F F
62492024-02-03T17:12:16.863Z 22 440-459 56 56 56 97 97 97 F F F
62502024-02-03T17:12:16.863Z 23 460-479 53 53 53 94 94 94 F F F
62512024-02-03T17:12:16.863Z 24 480-499 51 51 51 91 91 91 F F F
62522024-02-03T17:12:16.863Z 25 500-519 54 54 54 95 95 95 F F F
62532024-02-03T17:12:16.863Z 26 520-539 54 54 54 95 95 95 F F F
62542024-02-03T17:12:16.863Z 27 540-559 57 57 57 98 98 98 F F F
62552024-02-03T17:12:16.863Z 28 560-579 46 46 46 82 82 82 F F F
62562024-02-03T17:12:16.863Z 29 580-599 54 54 54 95 95 95 F F F
62572024-02-03T17:12:16.863ZMax gen: 57, Max flush: 98
62582024-02-03T17:12:16.863Z
62592024-02-03T17:12:16.863ZBegin loop 29
62602024-02-03T17:12:16.863ZDownstairs to restart: 1
62612024-02-03T17:12:16.894Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.442495408Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364}
62622024-02-03T17:12:16.895Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.443001962Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364}
62632024-02-03T17:12:16.903Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.443048995Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","Crucible runtime is spawned
62642024-02-03T17:12:16.903Zpid":1364}
62652024-02-03T17:12:16.903Z{"msg":"Crucible 0fab6cde-f856-48b7-bc72-44674401af4b has session id: 7efc5c5e-b17f-4150-a17d-c0bf3238fd53","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.443187659Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62662024-02-03T17:12:16.903Z{"msg":"Upstairs opts: Upstairs UUID: 0fab6cde-f856-48b7-bc72-44674401af4b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.443285006Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62672024-02-03T17:12:16.903Z{"msg":"Crucible stats registered with UUID: 0fab6cde-f856-48b7-bc72-44674401af4b","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.443364799Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62682024-02-03T17:12:16.903Z{"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.443406856Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364}
62692024-02-03T17:12:16.903Z{"msg":"0fab6cde-f856-48b7-bc72-44674401af4b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.443468899Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62702024-02-03T17:12:16.903Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.4435097Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"client":"0","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62712024-02-03T17:12:16.903Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.44361759Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"client":"0","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62722024-02-03T17:12:16.904Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.443759646Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"client":"1","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62732024-02-03T17:12:16.904Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.444010953Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"client":"1","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62742024-02-03T17:12:16.904Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.444107337Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"client":"2","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62752024-02-03T17:12:16.904Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.444154281Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"client":"2","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62762024-02-03T17:12:16.904Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.444297765Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"":"io task","client":"2","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62772024-02-03T17:12:16.904Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.444350066Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"":"io task","client":"0","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62782024-02-03T17:12:16.904Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.44443585Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"":"io task","client":"1","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62792024-02-03T17:12:16.904Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.444479843Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"":"io task","client":"2","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62802024-02-03T17:12:16.904Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.44460065Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"":"io task","client":"0","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62812024-02-03T17:12:16.904Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.444832137Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"":"io task","client":"1","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62822024-02-03T17:12:16.904Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.445324564Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"client":"2","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62832024-02-03T17:12:16.904Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.445401331Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"client":"2","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62842024-02-03T17:12:16.904Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.445493571Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"client":"0","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62852024-02-03T17:12:16.904Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.445540846Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"client":"0","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62862024-02-03T17:12:16.904Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.445783237Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"client":"1","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62872024-02-03T17:12:16.904Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.445938186Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"client":"1","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62882024-02-03T17:12:16.904Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.446373809Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"client":"2","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62892024-02-03T17:12:16.904Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.446638523Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"client":"0","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62902024-02-03T17:12:16.904Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.447165704Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"client":"1","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62912024-02-03T17:12:16.904Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.447468288Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"client":"2","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62922024-02-03T17:12:16.904Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.447537973Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"client":"2","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62932024-02-03T17:12:16.904Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.447564953Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62942024-02-03T17:12:16.904Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.447628612Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"client":"0","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62952024-02-03T17:12:16.904Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.447754212Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"client":"0","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62962024-02-03T17:12:16.905Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.44782898Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62972024-02-03T17:12:16.905Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.447903271Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"client":"1","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62982024-02-03T17:12:16.905Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.447948486Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"client":"1","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
62992024-02-03T17:12:16.905Z{"msg":"[0]R flush_numbers[0..12]: [98, 98, 92, 95, 92, 98, 98, 97, 97, 95, 95, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.447989749Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"":"downstairs","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
63002024-02-03T17:12:16.905Z{"msg":"[0]R generation[0..12]: [57, 57, 52, 54, 52, 57, 57, 56, 56, 54, 54, 54]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.448081741Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"":"downstairs","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
63012024-02-03T17:12:16.905Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.44812852Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"":"downstairs","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
63022024-02-03T17:12:16.905Z{"msg":"[1]R flush_numbers[0..12]: [98, 98, 92, 95, 92, 98, 98, 97, 97, 95, 95, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.448196085Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"":"downstairs","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
63032024-02-03T17:12:16.905Z{"msg":"[1]R generation[0..12]: [57, 57, 52, 54, 52, 57, 57, 56, 56, 54, 54, 54]","v":0,"name":"Wait for a query_work_queue command to finish before sending IO
63042024-02-03T17:12:16.905Zcrucible","level":30,"time":"2024-02-03T17:12:16.448243939Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"":"downstairs","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
63052024-02-03T17:12:16.905Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.448320636Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"":"downstairs","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
63062024-02-03T17:12:16.905Z{"msg":"[2]R flush_numbers[0..12]: [98, 98, 92, 95, 92, 98, 98, 97, 97, 95, 95, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.44836619Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"":"downstairs","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
63072024-02-03T17:12:16.905Z{"msg":"[2]R generation[0..12]: [57, 57, 52, 54, 52, 57, 57, 56, 56, 54, 54, 54]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.448434196Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"":"downstairs","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
63082024-02-03T17:12:16.905Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.44847995Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"":"downstairs","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
63092024-02-03T17:12:16.905Z{"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.448544513Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"":"downstairs","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
63102024-02-03T17:12:16.905Z{"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.448586809Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"":"downstairs","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
63112024-02-03T17:12:16.905Z{"msg":"Next flush: 99","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.448652806Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"":"downstairs","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
63122024-02-03T17:12:16.905Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.448694843Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"":"downstairs","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
63132024-02-03T17:12:16.905Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.448761737Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
63142024-02-03T17:12:16.905Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.448801262Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"client":"0","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
63152024-02-03T17:12:16.905Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.448839649Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"client":"1","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
63162024-02-03T17:12:16.905Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.44888061Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"client":"2","":"downstairs","session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
63172024-02-03T17:12:16.905Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
63182024-02-03T17:12:16.905Z,"time":"2024-02-03T17:12:16.449015435Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
63192024-02-03T17:12:16.909Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.449163243Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
63202024-02-03T17:12:16.909Z{"msg":"0fab6cde-f856-48b7-bc72-44674401af4b is now active with session: 7efc5c5e-b17f-4150-a17d-c0bf3238fd53","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.449231501Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
63212024-02-03T17:12:16.910Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.449271535Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364,"session_id":"7efc5c5e-b17f-4150-a17d-c0bf3238fd53"}
63222024-02-03T17:12:16.910Z{"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:16.449335825Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364}
63232024-02-03T17:12:16.910ZLoading write count information from file "/tmp/test_repair_verify.data"
63242024-02-03T17:12:16.910ZRun Repair workload
63252024-02-03T17:12:16.910Z01/30 Write block 51 len 40960 data: 4 4 2 2 2 2 2 3 3 2
63262024-02-03T17:12:16.910Z02/30 Read block 350 len 12288
63272024-02-03T17:12:16.910Z03/30 Write block 31 len 20480 data: 6 6 5 6 5
63282024-02-03T17:12:16.910Z04/30 Write block 157 len 32768 data: 4 5 4 6 6 8 8 8
63292024-02-03T17:12:16.910Z05/30 Flush
63302024-02-03T17:12:16.913Z06/30 Write block 21 len 12288 data: 6 9 9
63312024-02-03T17:12:16.913Z07/30 Read block 35 len 8192
63322024-02-03T17:12:16.916Z08/30 Read block 312 len 36864
63332024-02-03T17:12:16.916Z09/30 Read block 446 len 16384
63342024-02-03T17:12:16.919Z10/30 Read block 253 len 32768
63352024-02-03T17:12:16.919Z11/30 Read block 156 len 20480
63362024-02-03T17:12:16.922Z12/30 Write block 13 len 40960 data: 4 5 3 3 5 6 7 5 7 10
63372024-02-03T17:12:16.922Z13/30 Read block 329 len 16384
63382024-02-03T17:12:16.926Z14/30 Write block 231 len 28672 data: 5 6 4 5 5 4 4
63392024-02-03T17:12:16.929Z15/30 Read block 109 len 8192
63402024-02-03T17:12:16.932Z16/30 Write block 265 len 40960 data: 9 9 10 11 8 8 7 7 7 6
63412024-02-03T17:12:16.932Z17/30 Read block 441 len 24576
63422024-02-03T17:12:16.935Z18/30 Read block 579 len 32768
63432024-02-03T17:12:16.935Z19/30 Flush
63442024-02-03T17:12:16.939Z20/30 Write block 188 len 40960 data: 10 10 11 10 9 6 6 6 5 5
63452024-02-03T17:12:16.942Z21/30 Flush
63462024-02-03T17:12:16.945Z22/30 Write block 164 len 4096 data: 9
63472024-02-03T17:12:16.945Z23/30 Write block 278 len 8192 data: 6 7
63482024-02-03T17:12:16.948Z24/30 Write block 125 len 28672 data: 4 4 5 4 5 5 4
63492024-02-03T17:12:16.948Z25/30 Write block 33 len 8192 data: 6 7
63502024-02-03T17:12:16.948Z26/30 Read block 290 len 8192
63512024-02-03T17:12:16.951Z27/30 Read block 247 len 12288
63522024-02-03T17:12:16.951Z28/30 Read block 138 len 20480
63532024-02-03T17:12:16.951Z29/30 Read block 223 len 24576
63542024-02-03T17:12:16.954Z30/30 Read block 475 len 4096
63552024-02-03T17:12:16.954Z----------------------------------------------------------------
63562024-02-03T17:12:16.954Z Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:25
63572024-02-03T17:12:16.955ZStates: Active Active Active
63582024-02-03T17:12:16.955ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
63592024-02-03T17:12:16.955Z 6 Acked 1005 Write 3 Done Done Done false
63602024-02-03T17:12:16.955Z 7 Acked 1006 Read 2 Done Done Done false
63612024-02-03T17:12:16.955Z 8 Acked 1007 Read 9 Done Sent Done false
63622024-02-03T17:12:16.955Z 9 Acked 1008 Read 4 Done Sent Done false
63632024-02-03T17:12:16.955Z 10 Acked 1009 Read 8 Done Sent Done false
63642024-02-03T17:12:16.955Z 11 Acked 1010 Read 5 Done Sent Done false
63652024-02-03T17:12:16.955Z 12 Acked 1011 Write 10 Done Sent Done false
63662024-02-03T17:12:16.955Z 13 Acked 1012 Read 4 Done Sent Done false
63672024-02-03T17:12:16.955Z 14 Acked 1013 Write 7 Done Sent Done false
63682024-02-03T17:12:16.955Z 15 Acked 1014 Read 2 Done Sent Done false
63692024-02-03T17:12:16.955Z 16 Acked 1015 Write 10 Done Sent Done false
63702024-02-03T17:12:16.955Z 17 Acked 1016 Read 6 Done Sent Done false
63712024-02-03T17:12:16.955Z 18 Acked 1017 Read 8 Done Sent Done false
63722024-02-03T17:12:16.955Z 19 Acked 1018 Flush 0 Done Sent Done false
63732024-02-03T17:12:16.955Z 20 Acked 1019 Write 10 Done Sent Done false
63742024-02-03T17:12:16.955Z 21 Acked 1020 Flush 0 Done Sent Done false
63752024-02-03T17:12:16.955Z 22 Acked 1021 Write 1 Done Sent Done false
63762024-02-03T17:12:16.955Z 23 Acked 1022 Write 2 Done Sent Done false
63772024-02-03T17:12:16.955Z 24 Acked 1023 Write 7 Done Sent Done false
63782024-02-03T17:12:16.955Z 25 Acked 1024 Write 2 Done Sent Done false
63792024-02-03T17:12:16.955Z 26 Acked 1025 Read 2 Done Sent Done false
63802024-02-03T17:12:16.955Z 27 Acked 1026 Read 3 Done Sent Done false
63812024-02-03T17:12:16.956Z 28 Acked 1027 Read 5 Done Sent Done false
63822024-02-03T17:12:16.956Z 29 Acked 1028 Read 6 Done Sent Done false
63832024-02-03T17:12:16.956Z 30 Acked 1029 Read 1 Sent Sent Done false
63842024-02-03T17:12:16.956Z STATES DS:0 DS:1 DS:2 TOTAL
63852024-02-03T17:12:16.956Z New 0 0 0 0
63862024-02-03T17:12:16.956Z Sent 1 23 0 24
63872024-02-03T17:12:16.956Z Done 24 2 25 51
63882024-02-03T17:12:16.956Z Skipped 0 0 0 0
63892024-02-03T17:12:16.956Z Error 0 0 0 0
63902024-02-03T17:12:16.956ZLast Flush: 1020 1004 1020
63912024-02-03T17:12:16.956ZDownstairs last five completed: 1004 1003 1002 1001 1000
63922024-02-03T17:12:16.956ZUpstairs last five completed: 30 29 28 27 26
63932024-02-03T17:12:16.956Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:12:16.504945927Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1364}
63942024-02-03T17:12:16.956ZWrote out file "/tmp/test_repair_verify.data"
63952024-02-03T17:12:16.961Z
63962024-02-03T17:12:16.967ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
63972024-02-03T17:12:16.996ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
63982024-02-03T17:12:16.996Z 0 000-019 58 57 58 100 98 100 F F F <---
63992024-02-03T17:12:16.996Z 1 020-039 58 58 58 100 99 100 T T T <---
64002024-02-03T17:12:16.996Z 2 040-059 58 58 58 99 99 99 F F F
64012024-02-03T17:12:16.996Z 3 060-079 58 58 58 99 99 99 F F F
64022024-02-03T17:12:16.996Z 4 080-099 52 52 52 92 92 92 F F F
64032024-02-03T17:12:16.996Z 5 100-119 57 57 57 98 98 98 F F F
64042024-02-03T17:12:16.996Z 6 120-139 57 57 57 98 98 98 T F T <---
64052024-02-03T17:12:16.996Z 7 140-159 58 58 58 99 99 99 F F F
64062024-02-03T17:12:16.996Z 8 160-179 58 58 58 99 99 99 T F T <---
64072024-02-03T17:12:17.004Z 9 180-199 58 54 58 101 95 101 F F F <---
64082024-02-03T17:12:17.004Z 10 200-219 54 54 54 95 95 95 F F F
64092024-02-03T17:12:17.004Z 11 220-239 58 54 58 100 95 100 F F F <---
64102024-02-03T17:12:17.004Z 12 240-259 53 53 53 94 94 94 F F F
64112024-02-03T17:12:17.004Z 13 260-279 58 54 58 100 95 100 T F T <---
64122024-02-03T17:12:17.004Z 14 280-299 49 49 49 86 86 86 F F F
64132024-02-03T17:12:17.004Z 15 300-319 56 56 56 96 96 96 F F F
64142024-02-03T17:12:17.004Z 16 320-339 57 57 57 98 98 98 F F F
64152024-02-03T17:12:17.004Z 17 340-359 57 57 57 98 98 98 F F F
64162024-02-03T17:12:17.004Z 18 360-379 54 54 54 95 95 95 F F F
64172024-02-03T17:12:17.004Z 19 380-399 52 52 52 93 93 93 F F F
64182024-02-03T17:12:17.005Z 20 400-419 57 57 57 98 98 98 F F F
64192024-02-03T17:12:17.005Z 21 420-439 57 57 57 98 98 98 F F F
64202024-02-03T17:12:17.005Z 22 440-459 56 56 56 97 97 97 F F F
64212024-02-03T17:12:17.005Z 23 460-479 53 53 53 94 94 94 F F F
64222024-02-03T17:12:17.005Z 24 480-499 51 51 51 91 91 91 F F F
64232024-02-03T17:12:17.005Z 25 500-519 54 54 54 95 95 95 F F F
64242024-02-03T17:12:17.005Z 26 520-539 54 54 54 95 95 95 F F F
64252024-02-03T17:12:17.005Z 27 540-559 57 57 57 98 98 98 F F F
64262024-02-03T17:12:17.005Z 28 560-579 46 46 46 82 82 82 F F F
64272024-02-03T17:12:17.005Z 29 580-599 54 54 54 95 95 95 F F F
64282024-02-03T17:12:17.005ZMax gen: 58, Max flush: 101
64292024-02-03T17:12:17.005ZError: Difference in extent metadata found!
64302024-02-03T17:12:17.005ZA Difference in extent metadata is expected here
64312024-02-03T17:12:17.005ZOn loop 29
64322024-02-03T17:12:17.005Z
64332024-02-03T17:12:17.005ZVerifying data now
64342024-02-03T17:12:21.700ZNo change after verify
64352024-02-03T17:12:21.700ZLoop: 29 Downstairs dump after verify (and repair):
64362024-02-03T17:12:21.732ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
64372024-02-03T17:12:21.732Z 0 000-019 58 58 58 100 100 100 F F F
64382024-02-03T17:12:21.743Z 1 020-039 59 59 59 102 102 102 F F F
64392024-02-03T17:12:21.743Z 2 040-059 58 58 58 99 99 99 F F F
64402024-02-03T17:12:21.743Z 3 060-079 58 58 58 99 99 99 F F F
64412024-02-03T17:12:21.743Z 4 080-099 52 52 52 92 92 92 F F F
64422024-02-03T17:12:21.744Z 5 100-119 57 57 57 98 98 98 F F F
64432024-02-03T17:12:21.744Z 6 120-139 59 59 59 102 102 102 F F F
64442024-02-03T17:12:21.744Z 7 140-159 58 58 58 99 99 99 F F F
64452024-02-03T17:12:21.744Z 8 160-179 59 59 59 102 102 102 F F F
64462024-02-03T17:12:21.744Z 9 180-199 58 58 58 101 101 101 F F F
64472024-02-03T17:12:21.744Z 10 200-219 54 54 54 95 95 95 F F F
64482024-02-03T17:12:21.744Z 11 220-239 58 58 58 100 100 100 F F F
64492024-02-03T17:12:21.744Z 12 240-259 53 53 53 94 94 94 F F F
64502024-02-03T17:12:21.744Z 13 260-279 59 59 59 102 102 102 F F F
64512024-02-03T17:12:21.744Z 14 280-299 49 49 49 86 86 86 F F F
64522024-02-03T17:12:21.744Z 15 300-319 56 56 56 96 96 96 F F F
64532024-02-03T17:12:21.744Z 16 320-339 57 57 57 98 98 98 F F F
64542024-02-03T17:12:21.744Z 17 340-359 57 57 57 98 98 98 F F F
64552024-02-03T17:12:21.744Z 18 360-379 54 54 54 95 95 95 F F F
64562024-02-03T17:12:21.744Z 19 380-399 52 52 52 93 93 93 F F F
64572024-02-03T17:12:21.745Z 20 400-419 57 57 57 98 98 98 F F F
64582024-02-03T17:12:21.745Z 21 420-439 57 57 57 98 98 98 F F F
64592024-02-03T17:12:21.745Z 22 440-459 56 56 56 97 97 97 F F F
64602024-02-03T17:12:21.745Z 23 460-479 53 53 53 94 94 94 F F F
64612024-02-03T17:12:21.745Z 24 480-499 51 51 51 91 91 91 F F F
64622024-02-03T17:12:21.745Z 25 500-519 54 54 54 95 95 95 F F F
64632024-02-03T17:12:21.745Z 26 520-539 54 54 54 95 95 95 F F F
64642024-02-03T17:12:21.745Z 27 540-559 57 57 57 98 98 98 F F F
64652024-02-03T17:12:21.745Z 28 560-579 46 46 46 82 82 82 F F F
64662024-02-03T17:12:21.745Z 29 580-599 54 54 54 95 95 95 F F F
64672024-02-03T17:12:21.745ZMax gen: 59, Max flush: 102
64682024-02-03T17:12:21.746Z
64692024-02-03T17:12:21.746ZBegin loop 30
64702024-02-03T17:12:21.746ZDownstairs to restart: 1
64712024-02-03T17:12:21.761Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.308615749Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372}
64722024-02-03T17:12:21.761Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0Crucible runtime is spawned
64732024-02-03T17:12:21.771Z\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.309378608Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372}
64742024-02-03T17:12:21.772Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.309476629Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372}
64752024-02-03T17:12:21.772Z{"msg":"Crucible 5ac25af4-3193-499d-804b-a6a55fa59eb1 has session id: e32e16d0-772a-4723-b36e-e6d49dc5dada","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.309519702Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
64762024-02-03T17:12:21.772Z{"msg":"Upstairs opts: Upstairs UUID: 5ac25af4-3193-499d-804b-a6a55fa59eb1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.30957755Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
64772024-02-03T17:12:21.772Z{"msg":"Crucible stats registered with UUID: 5ac25af4-3193-499d-804b-a6a55fa59eb1","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.309659536Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
64782024-02-03T17:12:21.773Z{"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.309743628Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372}
64792024-02-03T17:12:21.773Z{"msg":"5ac25af4-3193-499d-804b-a6a55fa59eb1 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.309783848Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
64802024-02-03T17:12:21.773Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.309826169Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"client":"0","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
64812024-02-03T17:12:21.773Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.309868937Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"client":"0","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
64822024-02-03T17:12:21.774Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.310012617Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"client":"1","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
64832024-02-03T17:12:21.774Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.310158617Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"client":"1","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
64842024-02-03T17:12:21.774Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.310222303Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"client":"2","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
64852024-02-03T17:12:21.774Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.31027085Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"client":"2","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
64862024-02-03T17:12:21.774Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.310384911Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"":"io task","client":"2","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
64872024-02-03T17:12:21.775Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.310433594Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"":"io task","client":"0","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
64882024-02-03T17:12:21.775Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.310469685Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"":"io task","client":"1","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
64892024-02-03T17:12:21.775Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.310523743Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"":"io task","client":"1","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
64902024-02-03T17:12:21.775Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.310549806Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"":"io task","client":"2","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
64912024-02-03T17:12:21.776Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.310598044Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"":"io task","client":"0","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
64922024-02-03T17:12:21.776Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.311938797Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"client":"2","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
64932024-02-03T17:12:21.776Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.312024887Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"client":"2","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
64942024-02-03T17:12:21.776Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.312159606Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"client":"1","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
64952024-02-03T17:12:21.776Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.312228998Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"client":"1","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
64962024-02-03T17:12:21.776Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.312270734Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"client":"0","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
64972024-02-03T17:12:21.777Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.312318728Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"client":"0","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
64982024-02-03T17:12:21.777Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.313089394Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"client":"2","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
64992024-02-03T17:12:21.777Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.313165966Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"client":"0","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65002024-02-03T17:12:21.777Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.313309744Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"client":"1","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65012024-02-03T17:12:21.777Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.31402506Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"client":"2","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65022024-02-03T17:12:21.777Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.31408444Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"client":"2","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65032024-02-03T17:12:21.777Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.314146624Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65042024-02-03T17:12:21.777Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.314171412Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"client":"1","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65052024-02-03T17:12:21.778Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.314195901Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"client":"1","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65062024-02-03T17:12:21.778Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.314244673Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65072024-02-03T17:12:21.778Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.314379667Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"client":"0","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65082024-02-03T17:12:21.778Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.314440097Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"client":"0","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65092024-02-03T17:12:21.778Z{"msg":"[0]R flush_numbers[0..12]: [100, 102, 99, 99, 92, 98, 102, 99, 102, 101, 95, 100]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.314468101Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"":"downstairs","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65102024-02-03T17:12:21.778Z{"msg":"[0]R generation[0..12]: [58, 59, 58, 58, 52, 57, 59, 58, 59, 58, 54, 58]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.314491625Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"":"downstairs","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65112024-02-03T17:12:21.779Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.314517106Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"":"downstairs","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65122024-02-03T17:12:21.779Z{"msg":"[1]R flush_numbers[0..12]: [100, 102, 99, 99, 92, 98, 102, 99, 102, 101, 95, 100]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.314540825Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"":"downstairs","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65132024-02-03T17:12:21.779Z{"msg":"[1]R generation[0..12]: [58, 59, 58, 58, 52, 57, 59, 58, 59, 58, 54, 58]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.314564023Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"":"downstairs","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65142024-02-03T17:12:21.779Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.314612397Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"":"downstairs","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65152024-02-03T17:12:21.779Z{"msg":"[2]R flush_numbers[0..12]: [100, 102, 99, 99, 92, 98, 102, 99, 102, 101, 95, 100]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.314638136Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"":"downstairs","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65162024-02-03T17:12:21.779Z{"msg":"[2]R generation[0..12]: [58, 59, 58, 58, 52, 57, 59, 58, 59, 58, 54, 58]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.314661288Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"":"downstairs","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65172024-02-03T17:12:21.779Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.314709351Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"":"downstairs","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65182024-02-03T17:12:21.780Z{"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.314733397Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"":"downstairs","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65192024-02-03T17:12:21.780Z{"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.3147562Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal",Wait for a query_work_queue command to finish before sending IO
65202024-02-03T17:12:21.780Z"pid":1372,"":"downstairs","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65212024-02-03T17:12:21.780Z{"msg":"Next flush: 103","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.314783533Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"":"downstairs","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65222024-02-03T17:12:21.780Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.314805197Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"":"downstairs","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65232024-02-03T17:12:21.780Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.314827239Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65242024-02-03T17:12:21.781Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.314877907Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"client":"0","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65252024-02-03T17:12:21.781Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.314902161Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"client":"1","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65262024-02-03T17:12:21.781Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.314924033Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"client":"2","":"downstairs","session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65272024-02-03T17:12:21.781Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.314971826Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65282024-02-03T17:12:21.781Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.314994661Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65292024-02-03T17:12:21.781Z{"msg":"5ac25af4-3193-499d-804b-a6a55fa59eb1 is now active with session: e32e16d0-772a-4723-b36e-e6d49dc5dada","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.31501625Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65302024-02-03T17:12:21.781Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.31503751Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372,"session_id":"e32e16d0-772a-4723-b36e-e6d49dc5dada"}
65312024-02-03T17:12:21.781Z{"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:21.315058162Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372}
65322024-02-03T17:12:21.782ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
65332024-02-03T17:12:21.782ZLoading write count information from file "/tmp/test_repair_verify.data"
65342024-02-03T17:12:21.782ZRun Repair workload
65352024-02-03T17:12:21.789Z01/30 Write block 215 len 28672 data: 4 5 4 3 3 3 4
65362024-02-03T17:12:21.789Z02/30 Read block 398 len 8192
65372024-02-03T17:12:21.790Z03/30 Read block 9 len 28672
65382024-02-03T17:12:21.790Z04/30 Flush
65392024-02-03T17:12:21.790Z05/30 Write block 312 len 24576 data: 6 7 6 6 6 7
65402024-02-03T17:12:21.790Z06/30 Read block 261 len 20480
65412024-02-03T17:12:21.790Z07/30 Read block 113 len 32768
65422024-02-03T17:12:21.790Z08/30 Read block 528 len 4096
65432024-02-03T17:12:21.791Z09/30 Read block 233 len 16384
65442024-02-03T17:12:21.791Z10/30 Read block 165 len 12288
65452024-02-03T17:12:21.791Z11/30 Read block 296 len 32768
65462024-02-03T17:12:21.791Z12/30 Read block 41 len 40960
65472024-02-03T17:12:21.791Z13/30 Write block 466 len 32768 data: 6 5 4 4 4 3 4 4
65482024-02-03T17:12:21.803Z14/30 Read block 458 len 28672
65492024-02-03T17:12:21.803Z15/30 Read block 380 len 4096
65502024-02-03T17:12:21.803Z16/30 Write block 156 len 20480 data: 6 5 6 5 7
65512024-02-03T17:12:21.803Z17/30 Read block 78 len 40960
65522024-02-03T17:12:21.803Z18/30 Write block 39 len 4096 data: 7
65532024-02-03T17:12:21.803Z19/30 Read block 288 len 32768
65542024-02-03T17:12:21.803Z20/30 Read block 346 len 32768
65552024-02-03T17:12:21.803Z21/30 Write block 269 len 36864 data: 9 9 8 8 8 7 8 7 7
65562024-02-03T17:12:21.803Z22/30 Read block 478 len 16384
65572024-02-03T17:12:21.809Z23/30 Read block 150 len 8192
65582024-02-03T17:12:21.809Z24/30 Read block 205 len 28672
65592024-02-03T17:12:21.810Z25/30 Read block 374 len 16384
65602024-02-03T17:12:21.810Z26/30 Write block 279 len 40960 data: 8 7 7 8 9 7 7 6 6 4
65612024-02-03T17:12:21.818Z27/30 Write block 370 len 40960 data: 8 5 6 6 6 5 5 6 5 7
65622024-02-03T17:12:21.818Z28/30 Read block 73 len 20480
65632024-02-03T17:12:21.818Z29/30 Write block 321 len 20480 data: 8 9 10 8 7
65642024-02-03T17:12:21.818Z30/30 Read block 86 len 36864
65652024-02-03T17:12:21.818Z----------------------------------------------------------------
65662024-02-03T17:12:21.818Z Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:26
65672024-02-03T17:12:21.818ZStates: Active Active Active
65682024-02-03T17:12:21.818ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
65692024-02-03T17:12:21.819Z 5 Acked 1004 Write 6 Done Done Done false
65702024-02-03T17:12:21.819Z 6 Acked 1005 Read 5 Done Sent Done false
65712024-02-03T17:12:21.819Z 7 Acked 1006 Read 8 Done Sent Done false
65722024-02-03T17:12:21.819Z 8 Acked 1007 Read 1 Done Sent Done false
65732024-02-03T17:12:21.819Z 9 Acked 1008 Read 4 Done Sent Done false
65742024-02-03T17:12:21.819Z 10 Acked 1009 Read 3 Done Sent Done false
65752024-02-03T17:12:21.819Z 11 Acked 1010 Read 8 Done Sent Done false
65762024-02-03T17:12:21.819Z 12 Acked 1011 Read 10 Done Sent Done false
65772024-02-03T17:12:21.819Z 13 Acked 1012 Write 8 Done Sent Done false
65782024-02-03T17:12:21.819Z 14 Acked 1013 Read 7 Done Sent Done false
65792024-02-03T17:12:21.819Z 15 Acked 1014 Read 1 Done Sent Done false
65802024-02-03T17:12:21.819Z 16 Acked 1015 Write 5 Done Sent Done false
65812024-02-03T17:12:21.819Z 17 Acked 1016 Read 10 Done Sent Done false
65822024-02-03T17:12:21.820Z 18 Acked 1017 Write 1 Done Sent Done false
65832024-02-03T17:12:21.820Z 19 Acked 1018 Read 8 Done Sent Done false
65842024-02-03T17:12:21.820Z 20 Acked 1019 Read 8 Done Sent Done false
65852024-02-03T17:12:21.820Z 21 Acked 1020 Write 9 Done Sent Done false
65862024-02-03T17:12:21.820Z 22 Acked 1021 Read 4 Done Sent Done false
65872024-02-03T17:12:21.820Z 23 Acked 1022 Read 2 Done Sent Done false
65882024-02-03T17:12:21.820Z 24 Acked 1023 Read 7 Done Sent Done false
65892024-02-03T17:12:21.820Z 25 Acked 1024 Read 4 Done Sent Done false
65902024-02-03T17:12:21.821Z 26 Acked 1025 Write 10 Done Sent Done false
65912024-02-03T17:12:21.821Z 27 Acked 1026 Write 10 Done Sent Done false
65922024-02-03T17:12:21.821Z 28 Acked 1027 Read 5 Done Sent Done false
65932024-02-03T17:12:21.821Z 29 Acked 1028 Write 5 Done Sent Done false
65942024-02-03T17:12:21.821Z 30 Acked 1029 Read 9 Done Sent Done false
65952024-02-03T17:12:21.821Z STATES DS:0 DS:1 DS:2 TOTAL
65962024-02-03T17:12:21.821Z New 0 0 0 0
65972024-02-03T17:12:21.821Z Sent 0 25 0 25
65982024-02-03T17:12:21.822Z Done 26 1 26 53
65992024-02-03T17:12:21.822Z Skipped 0 0 0 0
66002024-02-03T17:12:21.822Z Error 0 0 0 0
66012024-02-03T17:12:21.822ZLast Flush: 1003 1003 1003
66022024-02-03T17:12:21.822ZDownstairs last five completed: 1003 1002 1001 1000
66032024-02-03T17:12:21.822ZUpstairs last five completed: 30 29 28 27 26
66042024-02-03T17:12:21.831Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:12:21.366314381Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1372}
66052024-02-03T17:12:21.831ZWrote out file "/tmp/test_repair_verify.data"
66062024-02-03T17:12:21.832Z
66072024-02-03T17:12:21.845ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
66082024-02-03T17:12:21.859ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
66092024-02-03T17:12:21.859Z 0 000-019 58 58 58 100 100 100 F F F
66102024-02-03T17:12:21.860Z 1 020-039 59 59 59 102 102 102 T F T <---
66112024-02-03T17:12:21.870Z 2 040-059 58 58 58 99 99 99 F F F
66122024-02-03T17:12:21.870Z 3 060-079 58 58 58 99 99 99 F F F
66132024-02-03T17:12:21.870Z 4 080-099 52 52 52 92 92 92 F F F
66142024-02-03T17:12:21.870Z 5 100-119 57 57 57 98 98 98 F F F
66152024-02-03T17:12:21.870Z 6 120-139 59 59 59 102 102 102 F F F
66162024-02-03T17:12:21.870Z 7 140-159 58 58 58 99 99 99 T F T <---
66172024-02-03T17:12:21.870Z 8 160-179 59 59 59 102 102 102 T F T <---
66182024-02-03T17:12:21.870Z 9 180-199 58 58 58 101 101 101 F F F
66192024-02-03T17:12:21.870Z 10 200-219 60 60 60 103 103 103 F F F
66202024-02-03T17:12:21.870Z 11 220-239 60 60 60 103 103 103 F F F
66212024-02-03T17:12:21.871Z 12 240-259 53 53 53 94 94 94 F F F
66222024-02-03T17:12:21.871Z 13 260-279 59 59 59 102 102 102 T F T <---
66232024-02-03T17:12:21.871Z 14 280-299 49 49 49 86 86 86 T F T <---
66242024-02-03T17:12:21.871Z 15 300-319 56 56 56 96 96 96 T T T
66252024-02-03T17:12:21.871Z 16 320-339 57 57 57 98 98 98 T F T <---
66262024-02-03T17:12:21.871Z 17 340-359 57 57 57 98 98 98 F F F
66272024-02-03T17:12:21.871Z 18 360-379 54 54 54 95 95 95 T F T <---
66282024-02-03T17:12:21.871Z 19 380-399 52 52 52 93 93 93 F F F
66292024-02-03T17:12:21.871Z 20 400-419 57 57 57 98 98 98 F F F
66302024-02-03T17:12:21.871Z 21 420-439 57 57 57 98 98 98 F F F
66312024-02-03T17:12:21.871Z 22 440-459 56 56 56 97 97 97 F F F
66322024-02-03T17:12:21.872Z 23 460-479 53 53 53 94 94 94 T F T <---
66332024-02-03T17:12:21.872Z 24 480-499 51 51 51 91 91 91 F F F
66342024-02-03T17:12:21.872Z 25 500-519 54 54 54 95 95 95 F F F
66352024-02-03T17:12:21.872Z 26 520-539 54 54 54 95 95 95 F F F
66362024-02-03T17:12:21.872Z 27 540-559 57 57 57 98 98 98 F F F
66372024-02-03T17:12:21.872Z 28 560-579 46 46 46 82 82 82 F F F
66382024-02-03T17:12:21.872Z 29 580-599 54 54 54 95 95 95 F F F
66392024-02-03T17:12:21.872ZMax gen: 60, Max flush: 103
66402024-02-03T17:12:21.872ZError: Difference in extent metadata found!
66412024-02-03T17:12:21.872ZA Difference in extent metadata is expected here
66422024-02-03T17:12:21.873ZOn loop 30
66432024-02-03T17:12:21.873Z
66442024-02-03T17:12:21.879ZVerifying data now
66452024-02-03T17:12:26.725ZNo change after verify
66462024-02-03T17:12:26.725ZLoop: 30 Downstairs dump after verify (and repair):
66472024-02-03T17:12:26.748ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
66482024-02-03T17:12:26.749Z 0 000-019 58 58 58 100 100 100 F F F
66492024-02-03T17:12:26.749Z 1 020-039 61 61 61 104 104 104 F F F
66502024-02-03T17:12:26.749Z 2 040-059 58 58 58 99 99 99 F F F
66512024-02-03T17:12:26.749Z 3 060-079 58 58 58 99 99 99 F F F
66522024-02-03T17:12:26.749Z 4 080-099 52 52 52 92 92 92 F F F
66532024-02-03T17:12:26.749Z 5 100-119 57 57 57 98 98 98 F F F
66542024-02-03T17:12:26.749Z 6 120-139 59 59 59 102 102 102 F F F
66552024-02-03T17:12:26.752Z 7 140-159 61 61 61 104 104 104 F F F
66562024-02-03T17:12:26.752Z 8 160-179 61 61 61 104 104 104 F F F
66572024-02-03T17:12:26.752Z 9 180-199 58 58 58 101 101 101 F F F
66582024-02-03T17:12:26.752Z 10 200-219 60 60 60 103 103 103 F F F
66592024-02-03T17:12:26.753Z 11 220-239 60 60 60 103 103 103 F F F
66602024-02-03T17:12:26.753Z 12 240-259 53 53 53 94 94 94 F F F
66612024-02-03T17:12:26.753Z 13 260-279 61 61 61 104 104 104 F F F
66622024-02-03T17:12:26.753Z 14 280-299 61 61 61 104 104 104 F F F
66632024-02-03T17:12:26.753Z 15 300-319 61 61 61 104 104 104 F F F
66642024-02-03T17:12:26.753Z 16 320-339 61 61 61 104 104 104 F F F
66652024-02-03T17:12:26.753Z 17 340-359 57 57 57 98 98 98 F F F
66662024-02-03T17:12:26.753Z 18 360-379 61 61 61 104 104 104 F F F
66672024-02-03T17:12:26.753Z 19 380-399 52 52 52 93 93 93 F F F
66682024-02-03T17:12:26.753Z 20 400-419 57 57 57 98 98 98 F F F
66692024-02-03T17:12:26.753Z 21 420-439 57 57 57 98 98 98 F F F
66702024-02-03T17:12:26.753Z 22 440-459 56 56 56 97 97 97 F F F
66712024-02-03T17:12:26.753Z 23 460-479 61 61 61 104 104 104 F F F
66722024-02-03T17:12:26.753Z 24 480-499 51 51 51 91 91 91 F F F
66732024-02-03T17:12:26.753Z 25 500-519 54 54 54 95 95 95 F F F
66742024-02-03T17:12:26.753Z 26 520-539 54 54 54 95 95 95 F F F
66752024-02-03T17:12:26.753Z 27 540-559 57 57 57 98 98 98 F F F
66762024-02-03T17:12:26.753Z 28 560-579 46 46 46 82 82 82 F F F
66772024-02-03T17:12:26.753Z 29 580-599 54 54 54 95 95 95 F F F
66782024-02-03T17:12:26.753ZMax gen: 61, Max flush: 104
66792024-02-03T17:12:26.757Z
66802024-02-03T17:12:26.757ZBegin loop 31
66812024-02-03T17:12:26.757ZDownstairs to restart: 0
66822024-02-03T17:12:26.785Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.336137755Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380}
66832024-02-03T17:12:26.785Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.336837211Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380}
66842024-02-03T17:12:26.797Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.336885691Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380}
66852024-02-03T17:12:26.797Z{"Crucible runtime is spawned
66862024-02-03T17:12:26.797Zmsg":"Crucible 9f093563-9369-4e9f-9377-a4833d6a26b0 has session id: 0a7d0398-fdb2-4dad-8537-474c3ca4cedc","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.337199402Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
66872024-02-03T17:12:26.797Z{"msg":"Upstairs opts: Upstairs UUID: 9f093563-9369-4e9f-9377-a4833d6a26b0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.337340326Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
66882024-02-03T17:12:26.797Z{"msg":"Crucible stats registered with UUID: 9f093563-9369-4e9f-9377-a4833d6a26b0","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.337386939Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
66892024-02-03T17:12:26.797Z{"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.337462144Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380}
66902024-02-03T17:12:26.797Z{"msg":"9f093563-9369-4e9f-9377-a4833d6a26b0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.337669023Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
66912024-02-03T17:12:26.797Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.337727325Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"client":"0","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
66922024-02-03T17:12:26.797Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.337753818Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"client":"0","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
66932024-02-03T17:12:26.797Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.337776328Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"client":"1","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
66942024-02-03T17:12:26.797Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.338004552Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"client":"1","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
66952024-02-03T17:12:26.797Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.338107074Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"client":"2","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
66962024-02-03T17:12:26.797Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.338152518Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"client":"2","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
66972024-02-03T17:12:26.797Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.338196733Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"":"io task","client":"2","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
66982024-02-03T17:12:26.797Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.338240197Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"":"io task","client":"0","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
66992024-02-03T17:12:26.797Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.338283467Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"":"io task","client":"1","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67002024-02-03T17:12:26.797Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.338325356Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"":"io task","client":"2","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67012024-02-03T17:12:26.797Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.338522447Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"":"io task","client":"1","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67022024-02-03T17:12:26.797Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.338628639Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"":"io task","client":"0","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67032024-02-03T17:12:26.797Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.339812395Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"client":"1","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67042024-02-03T17:12:26.798Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.339959142Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"client":"1","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67052024-02-03T17:12:26.798Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.340015403Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"client":"2","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67062024-02-03T17:12:26.798Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.340123552Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"client":"2","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67072024-02-03T17:12:26.803Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.340645403Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"client":"0","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67082024-02-03T17:12:26.803Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.340762306Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"client":"0","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67092024-02-03T17:12:26.804Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.340808473Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"client":"1","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67102024-02-03T17:12:26.804Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.34113975Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"client":"2","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67112024-02-03T17:12:26.804Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.341644978Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"client":"1","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67122024-02-03T17:12:26.804Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.341740568Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"client":"1","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67132024-02-03T17:12:26.804Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.341786329Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67142024-02-03T17:12:26.804Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.341898422Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"client":"0","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67152024-02-03T17:12:26.804Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.341948001Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"client":"2","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67162024-02-03T17:12:26.804Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.341993947Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"client":"2","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67172024-02-03T17:12:26.804Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.342040949Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67182024-02-03T17:12:26.804Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.342628526Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"client":"0","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67192024-02-03T17:12:26.804Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.342709706Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"client":"0","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67202024-02-03T17:12:26.804Z{"msg":"[0]R flush_numbers[0..12]: [100, 104, 99, 99, 92, 98, 102, 104, 104, 101, 103, 103]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.34274105Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"":"downstairs","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67212024-02-03T17:12:26.804Z{"msg":"[0]R generation[0..12]: [58, 61, 58, 58, 52, 57, 59, 61, 61, 58, 60, 60]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.342770452Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"":"downstairs","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67222024-02-03T17:12:26.804Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.342835639Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"":"downstairs","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67232024-02-03T17:12:26.804Z{"msg":"[1]R flush_numbers[0..12]: [100, 104, 99, 99, 92, 98, 102, 104, 104, 101, 103, 103]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.342863976Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"":"downstairs","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67242024-02-03T17:12:26.804Z{"msg":"[1]R generation[0..12]: [58, 61, 58, 58, 52, 57, 59, 61, 61, 58, 60, 60]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.342892688Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"":"downstairs","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67252024-02-03T17:12:26.804Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.342956791Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"":"downstairs","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67262024-02-03T17:12:26.804Z{"msg":"[2]R flush_numbers[0..12]: [100, 104, 99, 99, 92, 98, 102, 104, 104, 101, 103, 103]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.342984883Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"":"downstairs","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67272024-02-03T17:12:26.804Z{Wait for a query_work_queue command to finish before sending IO
67282024-02-03T17:12:26.804Z"msg":"[2]R generation[0..12]: [58, 61, 58, 58, 52, 57, 59, 61, 61, 58, 60, 60]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.343092967Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"":"downstairs","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67292024-02-03T17:12:26.804Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.343165483Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"":"downstairs","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67302024-02-03T17:12:26.804Z{"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.343193006Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"":"downstairs","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67312024-02-03T17:12:26.804Z{"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.343222633Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"":"downstairs","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67322024-02-03T17:12:26.805Z{"msg":"Next flush: 105","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.34328211Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"":"downstairs","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67332024-02-03T17:12:26.805Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.34330794Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"":"downstairs","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67342024-02-03T17:12:26.805Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.343331425Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67352024-02-03T17:12:26.805Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.343393655Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"client":"0","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67362024-02-03T17:12:26.805Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.34342013Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"client":"1","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67372024-02-03T17:12:26.805Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.343442149Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"client":"2","":"downstairs","session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67382024-02-03T17:12:26.805Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.343508177Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67392024-02-03T17:12:26.805Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.343533432Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67402024-02-03T17:12:26.805Z{"msg":"9f093563-9369-4e9f-9377-a4833d6a26b0 is now active with session: 0a7d0398-fdb2-4dad-8537-474c3ca4cedc","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.343555014Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67412024-02-03T17:12:26.805Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.343577435Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380,"session_id":"0a7d0398-fdb2-4dad-8537-474c3ca4cedc"}
67422024-02-03T17:12:26.805Z{"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:26.343640864Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380}
67432024-02-03T17:12:26.805ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
67442024-02-03T17:12:26.805ZLoading write count information from file "/tmp/test_repair_verify.data"
67452024-02-03T17:12:26.805ZRun Repair workload
67462024-02-03T17:12:26.805Z01/30 Write block 368 len 4096 data: 7
67472024-02-03T17:12:26.805Z02/30 Write block 327 len 28672 data: 7 7 8 6 6 6 6
67482024-02-03T17:12:26.805Z03/30 Read block 505 len 40960
67492024-02-03T17:12:26.805Z04/30 Read block 451 len 16384
67502024-02-03T17:12:26.805Z05/30 Read block 119 len 28672
67512024-02-03T17:12:26.809Z06/30 Write block 344 len 32768 data: 4 4 3 4 7 7 5 5
67522024-02-03T17:12:26.809Z07/30 Flush
67532024-02-03T17:12:26.809Z08/30 Write block 192 len 16384 data: 10 7 7 7
67542024-02-03T17:12:26.809Z09/30 Read block 282 len 20480
67552024-02-03T17:12:26.812Z10/30 Write block 446 len 12288 data: 8 8 8
67562024-02-03T17:12:26.812Z11/30 Read block 152 len 40960
67572024-02-03T17:12:26.815Z12/30 Write block 506 len 40960 data: 5 5 5 7 7 8 7 7 6 6
67582024-02-03T17:12:26.815Z13/30 Write block 376 len 12288 data: 6 7 6
67592024-02-03T17:12:26.815Z14/30 Read block 11 len 8192
67602024-02-03T17:12:26.821Z15/30 Read block 187 len 40960
67612024-02-03T17:12:26.821Z16/30 Flush
67622024-02-03T17:12:26.825Z17/30 Write block 51 len 40960 data: 5 5 3 3 3 3 3 4 4 3
67632024-02-03T17:12:26.825Z18/30 Write block 184 len 28672 data: 6 9 7 8 11 11 12
67642024-02-03T17:12:26.831Z19/30 Write block 460 len 36864 data: 3 3 3 5 5 6 7 6 5
67652024-02-03T17:12:26.831Z20/30 Write block 95 len 8192 data: 5 5
67662024-02-03T17:12:26.831Z21/30 Flush
67672024-02-03T17:12:26.831Z22/30 Write block 165 len 40960 data: 9 11 11 7 6 6 5 3 5 5
67682024-02-03T17:12:26.834Z23/30 Read block 368 len 32768
67692024-02-03T17:12:26.834Z24/30 Read block 95 len 4096
67702024-02-03T17:12:26.839Z25/30 Read block 53 len 24576
67712024-02-03T17:12:26.839Z26/30 Read block 163 len 4096
67722024-02-03T17:12:26.839Z27/30 Write block 588 len 24576 data: 7 6 6 5 6 6
67732024-02-03T17:12:26.840Z28/30 Read block 440 len 36864
67742024-02-03T17:12:26.843Z29/30 Write block 180 len 24576 data: 4 4 4 6 7 10
67752024-02-03T17:12:26.843Z30/30 Write block 234 len 12288 data: 6 6 5
67762024-02-03T17:12:26.843Z----------------------------------------------------------------
67772024-02-03T17:12:26.843Z Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:30
67782024-02-03T17:12:26.843ZStates: Active Active Active
67792024-02-03T17:12:26.843ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
67802024-02-03T17:12:26.844Z 1 Acked 1000 Write 1 Done Done Done false
67812024-02-03T17:12:26.844Z 2 Acked 1001 Write 7 Sent Done Done false
67822024-02-03T17:12:26.844Z 3 Acked 1002 Read 10 Sent Done Done false
67832024-02-03T17:12:26.844Z 4 Acked 1003 Read 4 Sent Done Done false
67842024-02-03T17:12:26.844Z 5 Acked 1004 Read 7 Sent Done Done false
67852024-02-03T17:12:26.844Z 6 Acked 1005 Write 8 Sent Done Done false
67862024-02-03T17:12:26.844Z 7 Acked 1006 Flush 0 Sent Done Done false
67872024-02-03T17:12:26.844Z 8 Acked 1007 Write 4 Sent Done Done false
67882024-02-03T17:12:26.844Z 9 Acked 1008 Read 5 Sent Done Done false
67892024-02-03T17:12:26.844Z 10 Acked 1009 Write 3 Sent Done Done false
67902024-02-03T17:12:26.844Z 11 Acked 1010 Read 10 Sent Done Done false
67912024-02-03T17:12:26.844Z 12 Acked 1011 Write 10 Sent Done Done false
67922024-02-03T17:12:26.844Z 13 Acked 1012 Write 3 Sent Done Done false
67932024-02-03T17:12:26.844Z 14 Acked 1013 Read 2 Sent Done Done false
67942024-02-03T17:12:26.844Z 15 Acked 1014 Read 10 Sent Done Done false
67952024-02-03T17:12:26.844Z 16 Acked 1015 Flush 0 Sent Done Done false
67962024-02-03T17:12:26.844Z 17 Acked 1016 Write 10 Sent Done Done false
67972024-02-03T17:12:26.844Z 18 Acked 1017 Write 7 Sent Done Done false
67982024-02-03T17:12:26.844Z 19 Acked 1018 Write 9 Sent Done Done false
67992024-02-03T17:12:26.844Z 20 Acked 1019 Write 2 Sent Done Done false
68002024-02-03T17:12:26.844Z 21 Acked 1020 Flush 0 Sent Done Done false
68012024-02-03T17:12:26.844Z 22 Acked 1021 Write 10 Sent Done Done false
68022024-02-03T17:12:26.844Z 23 Acked 1022 Read 8 Sent Done Done false
68032024-02-03T17:12:26.848Z 24 Acked 1023 Read 1 Sent Done Done false
68042024-02-03T17:12:26.848Z 25 Acked 1024 Read 6 Sent Done Done false
68052024-02-03T17:12:26.848Z 26 Acked 1025 Read 1 Sent Done Done false
68062024-02-03T17:12:26.848Z 27 Acked 1026 Write 6 Sent Done Done false
68072024-02-03T17:12:26.848Z 28 Acked 1027 Read 9 Sent Done Done false
68082024-02-03T17:12:26.848Z 29 Acked 1028 Write 6 Sent Sent Sent false
68092024-02-03T17:12:26.848Z 30 Acked 1029 Write 3 Sent Sent Sent false
68102024-02-03T17:12:26.848Z STATES DS:0 DS:1 DS:2 TOTAL
68112024-02-03T17:12:26.848Z New 0 0 0 0
68122024-02-03T17:12:26.848Z Sent 29 2 2 33
68132024-02-03T17:12:26.848Z Done 1 28 28 57
68142024-02-03T17:12:26.849Z Skipped 0 0 0 0
68152024-02-03T17:12:26.849Z Error 0 0 0 0
68162024-02-03T17:12:26.849ZLast Flush: 0 1020 1020
68172024-02-03T17:12:26.849ZDownstairs last five completed:
68182024-02-03T17:12:26.849ZUpstairs last five completed: 30 29 28 27 26
68192024-02-03T17:12:26.849Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:12:26.395232627Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1380}
68202024-02-03T17:12:26.849ZWrote out file "/tmp/test_repair_verify.data"
68212024-02-03T17:12:26.852Z
68222024-02-03T17:12:26.855ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
68232024-02-03T17:12:26.885ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
68242024-02-03T17:12:26.885Z 0 000-019 58 58 58 100 100 100 F F F
68252024-02-03T17:12:26.885Z 1 020-039 61 61 61 104 104 104 F F F
68262024-02-03T17:12:26.886Z 2 040-059 58 62 62 99 107 107 F F F <---
68272024-02-03T17:12:26.886Z 3 060-079 58 62 62 99 107 107 F F F <---
68282024-02-03T17:12:26.901Z 4 080-099 52 62 62 92 107 107 F F F <---
68292024-02-03T17:12:26.901Z 5 100-119 57 57 57 98 98 98 F F F
68302024-02-03T17:12:26.901Z 6 120-139 59 59 59 102 102 102 F F F
68312024-02-03T17:12:26.901Z 7 140-159 61 61 61 104 104 104 F F F
68322024-02-03T17:12:26.901Z 8 160-179 61 61 61 104 104 104 F T T <---
68332024-02-03T17:12:26.901Z 9 180-199 58 62 62 101 107 107 F T T <---
68342024-02-03T17:12:26.901Z 10 200-219 60 60 60 103 103 103 F F F
68352024-02-03T17:12:26.901Z 11 220-239 60 60 60 103 103 103 F T T <---
68362024-02-03T17:12:26.901Z 12 240-259 53 53 53 94 94 94 F F F
68372024-02-03T17:12:26.901Z 13 260-279 61 61 61 104 104 104 F F F
68382024-02-03T17:12:26.901Z 14 280-299 61 61 61 104 104 104 F F F
68392024-02-03T17:12:26.902Z 15 300-319 61 61 61 104 104 104 F F F
68402024-02-03T17:12:26.902Z 16 320-339 61 62 62 104 105 105 F F F <---
68412024-02-03T17:12:26.902Z 17 340-359 57 62 62 98 105 105 F F F <---
68422024-02-03T17:12:26.902Z 18 360-379 61 62 62 104 106 106 T F F <---
68432024-02-03T17:12:26.902Z 19 380-399 52 52 52 93 93 93 F F F
68442024-02-03T17:12:26.902Z 20 400-419 57 57 57 98 98 98 F F F
68452024-02-03T17:12:26.902Z 21 420-439 57 57 57 98 98 98 F F F
68462024-02-03T17:12:26.902Z 22 440-459 56 62 62 97 106 106 F F F <---
68472024-02-03T17:12:26.902Z 23 460-479 61 62 62 104 107 107 F F F <---
68482024-02-03T17:12:26.902Z 24 480-499 51 51 51 91 91 91 F F F
68492024-02-03T17:12:26.902Z 25 500-519 54 62 62 95 106 106 F F F <---
68502024-02-03T17:12:26.902Z 26 520-539 54 54 54 95 95 95 F F F
68512024-02-03T17:12:26.902Z 27 540-559 57 57 57 98 98 98 F F F
68522024-02-03T17:12:26.902Z 28 560-579 46 46 46 82 82 82 F F F
68532024-02-03T17:12:26.902Z 29 580-599 54 54 54 95 95 95 F T T <---
68542024-02-03T17:12:26.902ZMax gen: 62, Max flush: 107
68552024-02-03T17:12:26.902ZError: Difference in extent metadata found!
68562024-02-03T17:12:26.902ZA Difference in extent metadata is expected here
68572024-02-03T17:12:26.902ZOn loop 31
68582024-02-03T17:12:26.902Z
68592024-02-03T17:12:26.902ZVerifying data now
68602024-02-03T17:12:32.018ZNo change after verify
68612024-02-03T17:12:32.018ZLoop: 31 Downstairs dump after verify (and repair):
68622024-02-03T17:12:32.040ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
68632024-02-03T17:12:32.041Z 0 000-019 58 58 58 100 100 100 F F F
68642024-02-03T17:12:32.041Z 1 020-039 61 61 61 104 104 104 F F F
68652024-02-03T17:12:32.044Z 2 040-059 62 62 62 107 107 107 F F F
68662024-02-03T17:12:32.044Z 3 060-079 62 62 62 107 107 107 F F F
68672024-02-03T17:12:32.044Z 4 080-099 62 62 62 107 107 107 F F F
68682024-02-03T17:12:32.044Z 5 100-119 57 57 57 98 98 98 F F F
68692024-02-03T17:12:32.044Z 6 120-139 59 59 59 102 102 102 F F F
68702024-02-03T17:12:32.044Z 7 140-159 61 61 61 104 104 104 F F F
68712024-02-03T17:12:32.044Z 8 160-179 63 63 63 108 108 108 F F F
68722024-02-03T17:12:32.044Z 9 180-199 63 63 63 108 108 108 F F F
68732024-02-03T17:12:32.044Z 10 200-219 60 60 60 103 103 103 F F F
68742024-02-03T17:12:32.044Z 11 220-239 63 63 63 108 108 108 F F F
68752024-02-03T17:12:32.044Z 12 240-259 53 53 53 94 94 94 F F F
68762024-02-03T17:12:32.044Z 13 260-279 61 61 61 104 104 104 F F F
68772024-02-03T17:12:32.044Z 14 280-299 61 61 61 104 104 104 F F F
68782024-02-03T17:12:32.045Z 15 300-319 61 61 61 104 104 104 F F F
68792024-02-03T17:12:32.045Z 16 320-339 62 62 62 105 105 105 F F F
68802024-02-03T17:12:32.045Z 17 340-359 62 62 62 105 105 105 F F F
68812024-02-03T17:12:32.045Z 18 360-379 62 62 62 106 106 106 F F F
68822024-02-03T17:12:32.045Z 19 380-399 52 52 52 93 93 93 F F F
68832024-02-03T17:12:32.045Z 20 400-419 57 57 57 98 98 98 F F F
68842024-02-03T17:12:32.045Z 21 420-439 57 57 57 98 98 98 F F F
68852024-02-03T17:12:32.045Z 22 440-459 62 62 62 106 106 106 F F F
68862024-02-03T17:12:32.045Z 23 460-479 62 62 62 107 107 107 F F F
68872024-02-03T17:12:32.045Z 24 480-499 51 51 51 91 91 91 F F F
68882024-02-03T17:12:32.045Z 25 500-519 62 62 62 106 106 106 F F F
68892024-02-03T17:12:32.045Z 26 520-539 54 54 54 95 95 95 F F F
68902024-02-03T17:12:32.045Z 27 540-559 57 57 57 98 98 98 F F F
68912024-02-03T17:12:32.045Z 28 560-579 46 46 46 82 82 82 F F F
68922024-02-03T17:12:32.045Z 29 580-599 63 63 63 108 108 108 F F F
68932024-02-03T17:12:32.045ZMax gen: 63, Max flush: 108
68942024-02-03T17:12:32.048Z
68952024-02-03T17:12:32.048ZBegin loop 32
68962024-02-03T17:12:32.048ZDownstairs to restart: 2
68972024-02-03T17:12:32.078Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.629190459Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388}
68982024-02-03T17:12:32.078Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.62976137Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388}
68992024-02-03T17:12:32.084Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.629831499Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388}
69002024-02-03T17:12:32.084Z{"msg":"Crucible 350ede11-35ad-41dc-ac00-050374ed9269 has session id: eea09274-e5e6-4b68-9b30-1172c548e24f","v":0,"name":"crucible","Crucible runtime is spawned
69012024-02-03T17:12:32.084Zlevel":30,"time":"2024-02-03T17:12:31.630034155Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69022024-02-03T17:12:32.084Z{"msg":"Upstairs opts: Upstairs UUID: 350ede11-35ad-41dc-ac00-050374ed9269, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.630139947Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69032024-02-03T17:12:32.084Z{"msg":"Crucible stats registered with UUID: 350ede11-35ad-41dc-ac00-050374ed9269","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.630183677Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69042024-02-03T17:12:32.084Z{"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.630258549Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388}
69052024-02-03T17:12:32.084Z{"msg":"350ede11-35ad-41dc-ac00-050374ed9269 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.630297206Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69062024-02-03T17:12:32.084Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.630401819Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"client":"0","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69072024-02-03T17:12:32.084Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.630447271Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"client":"0","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69082024-02-03T17:12:32.084Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.630555613Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"client":"1","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69092024-02-03T17:12:32.084Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.630603779Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"client":"1","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69102024-02-03T17:12:32.084Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.630643654Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"client":"2","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69112024-02-03T17:12:32.084Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.630707065Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"client":"2","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69122024-02-03T17:12:32.084Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.630784932Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"":"io task","client":"0","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69132024-02-03T17:12:32.084Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.630833278Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"":"io task","client":"2","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69142024-02-03T17:12:32.084Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.630876098Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"":"io task","client":"1","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69152024-02-03T17:12:32.085Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.630917881Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"":"io task","client":"2","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69162024-02-03T17:12:32.085Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.630961875Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"":"io task","client":"1","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69172024-02-03T17:12:32.085Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.631147477Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"":"io task","client":"0","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69182024-02-03T17:12:32.085Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.632325532Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"client":"0","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69192024-02-03T17:12:32.085Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.63238766Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"client":"0","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69202024-02-03T17:12:32.085Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.63253542Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"client":"1","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69212024-02-03T17:12:32.085Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.632768316Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"client":"1","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69222024-02-03T17:12:32.085Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.632815053Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"client":"2","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69232024-02-03T17:12:32.085Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.632889891Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"client":"2","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69242024-02-03T17:12:32.085Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.633585861Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"client":"1","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69252024-02-03T17:12:32.085Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.633655577Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"client":"0","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69262024-02-03T17:12:32.085Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.633872574Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"client":"2","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69272024-02-03T17:12:32.092Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.634538284Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"client":"1","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69282024-02-03T17:12:32.092Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.634664727Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"client":"1","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69292024-02-03T17:12:32.092Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.634714301Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69302024-02-03T17:12:32.092Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.634759051Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"client":"0","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69312024-02-03T17:12:32.093Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.634805935Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"client":"0","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69322024-02-03T17:12:32.093Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.634893346Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69332024-02-03T17:12:32.093Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.634975428Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"client":"2","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69342024-02-03T17:12:32.093Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.635055801Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"client":"2","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69352024-02-03T17:12:32.093Z{"msg":"[0]R flush_numbers[0..12]: [100, 104, 107, 107, 107, 98, 102, 104, 108, 108, 103, 108]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.635138103Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"":"downstairs","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69362024-02-03T17:12:32.093Z{"msg":"[0]R generation[0..12]: [58, 61, 62, 62, 62, 57, 59, 61, 63, 63, 60, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.635223568Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"":"downstairs","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69372024-02-03T17:12:32.093Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.635304988Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"":"downstairs","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69382024-02-03T17:12:32.093Z{"msg":"[1]R flush_numbers[0..12]: [100, 104, 107, 107, 107, 98, 102, 104, 108, 108, 103, 108]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.635383606ZWait for a query_work_queue command to finish before sending IO
69392024-02-03T17:12:32.093Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"":"downstairs","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69402024-02-03T17:12:32.093Z{"msg":"[1]R generation[0..12]: [58, 61, 62, 62, 62, 57, 59, 61, 63, 63, 60, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.635473921Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"":"downstairs","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69412024-02-03T17:12:32.093Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.635561075Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"":"downstairs","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69422024-02-03T17:12:32.093Z{"msg":"[2]R flush_numbers[0..12]: [100, 104, 107, 107, 107, 98, 102, 104, 108, 108, 103, 108]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.635644783Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"":"downstairs","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69432024-02-03T17:12:32.093Z{"msg":"[2]R generation[0..12]: [58, 61, 62, 62, 62, 57, 59, 61, 63, 63, 60, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.635726448Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"":"downstairs","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69442024-02-03T17:12:32.093Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.635788948Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"":"downstairs","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69452024-02-03T17:12:32.093Z{"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.635824827Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"":"downstairs","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69462024-02-03T17:12:32.093Z{"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.635900592Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"":"downstairs","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69472024-02-03T17:12:32.093Z{"msg":"Next flush: 109","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.635935175Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"":"downstairs","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69482024-02-03T17:12:32.093Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.636013525Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"":"downstairs","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69492024-02-03T17:12:32.094Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.636086163Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69502024-02-03T17:12:32.094Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.636111745Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"client":"0","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69512024-02-03T17:12:32.094Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.636164032Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"client":"1","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69522024-02-03T17:12:32.094Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.636204933Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"client":"2","":"downstairs","session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69532024-02-03T17:12:32.094Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.636278306Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69542024-02-03T17:12:32.094Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.636304172Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69552024-02-03T17:12:32.094Z{"msg":"350ede11-35ad-41dc-ac00-050374ed9269 is now active with session: eea09274-e5e6-4b68-9b30-1172c548e24f","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.636352546Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69562024-02-03T17:12:32.094Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
69572024-02-03T17:12:32.094Z,"time":"2024-02-03T17:12:31.636387566Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388,"session_id":"eea09274-e5e6-4b68-9b30-1172c548e24f"}
69582024-02-03T17:12:32.094Z{"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:31.636537374Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388}
69592024-02-03T17:12:32.094ZLoading write count information from file "/tmp/test_repair_verify.data"
69602024-02-03T17:12:32.094ZRun Repair workload
69612024-02-03T17:12:32.094Z01/30 Write block 72 len 24576 data: 3 4 5 5 6 6
69622024-02-03T17:12:32.094Z02/30 Read block 232 len 8192
69632024-02-03T17:12:32.094Z03/30 Write block 460 len 8192 data: 4 4
69642024-02-03T17:12:32.094Z04/30 Read block 397 len 12288
69652024-02-03T17:12:32.099Z05/30 Read block 342 len 36864
69662024-02-03T17:12:32.099Z06/30 Read block 574 len 12288
69672024-02-03T17:12:32.099Z07/30 Read block 487 len 36864
69682024-02-03T17:12:32.099Z08/30 Flush
69692024-02-03T17:12:32.102Z09/30 Write block 598 len 8192 data: 2 2
69702024-02-03T17:12:32.102Z10/30 Write block 170 len 40960 data: 7 6 4 6 6 4 6 7 5 4
69712024-02-03T17:12:32.102Z11/30 Write block 54 len 12288 data: 4 4 4
69722024-02-03T17:12:32.105Z12/30 Read block 424 len 32768
69732024-02-03T17:12:32.109Z13/30 Write block 276 len 36864 data: 8 8 7 9 8 8 9 10 8
69742024-02-03T17:12:32.109Z14/30 Write block 588 len 4096 data: 8
69752024-02-03T17:12:32.109Z15/30 Write block 504 len 20480 data: 7 5 6 6 6
69762024-02-03T17:12:32.111Z16/30 Write block 202 len 24576 data: 5 5 4 6 6 5
69772024-02-03T17:12:32.112Z17/30 Write block 406 len 8192 data: 4 5
69782024-02-03T17:12:32.112Z18/30 Write block 442 len 36864 data: 10 8 7 7 9 9 9 6 4
69792024-02-03T17:12:32.118Z19/30 Write block 526 len 40960 data: 7 8 7 7 8 6 6 6 4 5
69802024-02-03T17:12:32.118Z20/30 Read block 23 len 24576
69812024-02-03T17:12:32.118Z21/30 Read block 279 len 8192
69822024-02-03T17:12:32.118Z22/30 Write block 155 len 16384 data: 5 7 6 7
69832024-02-03T17:12:32.125Z23/30 Read block 537 len 8192
69842024-02-03T17:12:32.125Z24/30 Read block 165 len 4096
69852024-02-03T17:12:32.125Z25/30 Read block 73 len 16384
69862024-02-03T17:12:32.130Z26/30 Read block 99 len 40960
69872024-02-03T17:12:32.130Z27/30 Write block 183 len 28672 data: 7 8 11 8 9 12 12
69882024-02-03T17:12:32.130Z28/30 Write block 421 len 40960 data: 7 5 4 4 4 4 5 5 8 8
69892024-02-03T17:12:32.130Z29/30 Write block 477 len 24576 data: 5 5 7 7 6 5
69902024-02-03T17:12:32.134Z30/30 Read block 170 len 24576
69912024-02-03T17:12:32.134Z----------------------------------------------------------------
69922024-02-03T17:12:32.134Z Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30
69932024-02-03T17:12:32.134ZStates: Active Active Active
69942024-02-03T17:12:32.134ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
69952024-02-03T17:12:32.134Z 1 Acked 1000 Write 6 Done Done Done false
69962024-02-03T17:12:32.134Z 2 Acked 1001 Read 2 Done Done Done false
69972024-02-03T17:12:32.134Z 3 Acked 1002 Write 2 Done Done Done false
69982024-02-03T17:12:32.134Z 4 Acked 1003 Read 3 Done Done Done false
69992024-02-03T17:12:32.134Z 5 Acked 1004 Read 9 Done Done Sent false
70002024-02-03T17:12:32.134Z 6 Acked 1005 Read 3 Done Done Sent false
70012024-02-03T17:12:32.134Z 7 Acked 1006 Read 9 Done Done Sent false
70022024-02-03T17:12:32.134Z 8 Acked 1007 Flush 0 Done Done Sent false
70032024-02-03T17:12:32.135Z 9 Acked 1008 Write 2 Done Done Sent false
70042024-02-03T17:12:32.135Z 10 Acked 1009 Write 10 Done Done Sent false
70052024-02-03T17:12:32.135Z 11 Acked 1010 Write 3 Done Done Sent false
70062024-02-03T17:12:32.135Z 12 Acked 1011 Read 8 Done Done Sent false
70072024-02-03T17:12:32.135Z 13 Acked 1012 Write 9 Done Done Sent false
70082024-02-03T17:12:32.135Z 14 Acked 1013 Write 1 Done Done Sent false
70092024-02-03T17:12:32.135Z 15 Acked 1014 Write 5 Done Done Sent false
70102024-02-03T17:12:32.135Z 16 Acked 1015 Write 6 Done Done Sent false
70112024-02-03T17:12:32.135Z 17 Acked 1016 Write 2 Done Done Sent false
70122024-02-03T17:12:32.135Z 18 Acked 1017 Write 9 Done Done Sent false
70132024-02-03T17:12:32.135Z 19 Acked 1018 Write 10 Done Done Sent false
70142024-02-03T17:12:32.135Z 20 Acked 1019 Read 6 Done Done Sent false
70152024-02-03T17:12:32.135Z 21 Acked 1020 Read 2 Done Done Sent false
70162024-02-03T17:12:32.135Z 22 Acked 1021 Write 4 Done Done Sent false
70172024-02-03T17:12:32.135Z 23 Acked 1022 Read 2 Done Done Sent false
70182024-02-03T17:12:32.135Z 24 Acked 1023 Read 1 Done Done Sent false
70192024-02-03T17:12:32.135Z 25 Acked 1024 Read 4 Done Done Sent false
70202024-02-03T17:12:32.135Z 26 Acked 1025 Read 10 Done Done Sent false
70212024-02-03T17:12:32.135Z 27 Acked 1026 Write 7 Done Done Sent false
70222024-02-03T17:12:32.136Z 28 Acked 1027 Write 10 Done Done Sent false
70232024-02-03T17:12:32.136Z 29 Acked 1028 Write 6 Done Done Sent false
70242024-02-03T17:12:32.136Z 30 Acked 1029 Read 6 Done Sent Sent false
70252024-02-03T17:12:32.136Z STATES DS:0 DS:1 DS:2 TOTAL
70262024-02-03T17:12:32.136Z New 0 0 0 0
70272024-02-03T17:12:32.136Z Sent 0 1 26 27
70282024-02-03T17:12:32.136Z Done 30 29 4 63
70292024-02-03T17:12:32.136Z Skipped 0 0 0 0
70302024-02-03T17:12:32.136Z Error 0 0 0 0
70312024-02-03T17:12:32.136ZLast Flush: 1007 1007 0
70322024-02-03T17:12:32.136ZDownstairs last five completed:
70332024-02-03T17:12:32.136ZUpstairs last five completed: 30 29 28 27 26
70342024-02-03T17:12:32.136Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:12:31.684002177Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1388}
70352024-02-03T17:12:32.136ZWrote out file "/tmp/test_repair_verify.data"
70362024-02-03T17:12:32.141Z
70372024-02-03T17:12:32.146ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
70382024-02-03T17:12:32.179ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70392024-02-03T17:12:32.179Z 0 000-019 58 58 58 100 100 100 F F F
70402024-02-03T17:12:32.179Z 1 020-039 61 61 61 104 104 104 F F F
70412024-02-03T17:12:32.179Z 2 040-059 62 62 62 107 107 107 T T F <---
70422024-02-03T17:12:32.183Z 3 060-079 64 64 62 109 109 107 F F T <---
70432024-02-03T17:12:32.183Z 4 080-099 62 62 62 107 107 107 F F F
70442024-02-03T17:12:32.183Z 5 100-119 57 57 57 98 98 98 F F F
70452024-02-03T17:12:32.183Z 6 120-139 59 59 59 102 102 102 F F F
70462024-02-03T17:12:32.183Z 7 140-159 61 61 61 104 104 104 T T F <---
70472024-02-03T17:12:32.183Z 8 160-179 63 63 63 108 108 108 T T F <---
70482024-02-03T17:12:32.183Z 9 180-199 63 63 63 108 108 108 T T F <---
70492024-02-03T17:12:32.183Z 10 200-219 60 60 60 103 103 103 T T F <---
70502024-02-03T17:12:32.183Z 11 220-239 63 63 63 108 108 108 F F F
70512024-02-03T17:12:32.183Z 12 240-259 53 53 53 94 94 94 F F F
70522024-02-03T17:12:32.183Z 13 260-279 61 61 61 104 104 104 T T F <---
70532024-02-03T17:12:32.183Z 14 280-299 61 61 61 104 104 104 T T F <---
70542024-02-03T17:12:32.183Z 15 300-319 61 61 61 104 104 104 F F F
70552024-02-03T17:12:32.183Z 16 320-339 62 62 62 105 105 105 F F F
70562024-02-03T17:12:32.183Z 17 340-359 62 62 62 105 105 105 F F F
70572024-02-03T17:12:32.183Z 18 360-379 62 62 62 106 106 106 F F F
70582024-02-03T17:12:32.183Z 19 380-399 52 52 52 93 93 93 F F F
70592024-02-03T17:12:32.183Z 20 400-419 57 57 57 98 98 98 T T F <---
70602024-02-03T17:12:32.184Z 21 420-439 57 57 57 98 98 98 T T F <---
70612024-02-03T17:12:32.184Z 22 440-459 62 62 62 106 106 106 T T F <---
70622024-02-03T17:12:32.184Z 23 460-479 64 64 62 109 109 107 T T T <---
70632024-02-03T17:12:32.184Z 24 480-499 51 51 51 91 91 91 T T F <---
70642024-02-03T17:12:32.184Z 25 500-519 62 62 62 106 106 106 T T F <---
70652024-02-03T17:12:32.184Z 26 520-539 54 54 54 95 95 95 T T F <---
70662024-02-03T17:12:32.184Z 27 540-559 57 57 57 98 98 98 F F F
70672024-02-03T17:12:32.184Z 28 560-579 46 46 46 82 82 82 F F F
70682024-02-03T17:12:32.184Z 29 580-599 63 63 63 108 108 108 T T F <---
70692024-02-03T17:12:32.184ZMax gen: 64, Max flush: 109
70702024-02-03T17:12:32.184ZError: Difference in extent metadata found!
70712024-02-03T17:12:32.189ZA Difference in extent metadata is expected here
70722024-02-03T17:12:32.189ZOn loop 32
70732024-02-03T17:12:32.189Z
70742024-02-03T17:12:32.189ZVerifying data now
70752024-02-03T17:12:38.591ZNo change after verify
70762024-02-03T17:12:38.591ZLoop: 32 Downstairs dump after verify (and repair):
70772024-02-03T17:12:38.617ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70782024-02-03T17:12:38.617Z 0 000-019 58 58 58 100 100 100 F F F
70792024-02-03T17:12:38.618Z 1 020-039 61 61 61 104 104 104 F F F
70802024-02-03T17:12:38.618Z 2 040-059 65 65 65 110 110 110 F F F
70812024-02-03T17:12:38.618Z 3 060-079 64 64 64 109 109 109 F F F
70822024-02-03T17:12:38.621Z 4 080-099 62 62 62 107 107 107 F F F
70832024-02-03T17:12:38.621Z 5 100-119 57 57 57 98 98 98 F F F
70842024-02-03T17:12:38.621Z 6 120-139 59 59 59 102 102 102 F F F
70852024-02-03T17:12:38.621Z 7 140-159 65 65 65 110 110 110 F F F
70862024-02-03T17:12:38.621Z 8 160-179 65 65 65 110 110 110 F F F
70872024-02-03T17:12:38.621Z 9 180-199 65 65 65 110 110 110 F F F
70882024-02-03T17:12:38.622Z 10 200-219 65 65 65 110 110 110 F F F
70892024-02-03T17:12:38.622Z 11 220-239 63 63 63 108 108 108 F F F
70902024-02-03T17:12:38.622Z 12 240-259 53 53 53 94 94 94 F F F
70912024-02-03T17:12:38.622Z 13 260-279 65 65 65 110 110 110 F F F
70922024-02-03T17:12:38.622Z 14 280-299 65 65 65 110 110 110 F F F
70932024-02-03T17:12:38.622Z 15 300-319 61 61 61 104 104 104 F F F
70942024-02-03T17:12:38.622Z 16 320-339 62 62 62 105 105 105 F F F
70952024-02-03T17:12:38.622Z 17 340-359 62 62 62 105 105 105 F F F
70962024-02-03T17:12:38.622Z 18 360-379 62 62 62 106 106 106 F F F
70972024-02-03T17:12:38.622Z 19 380-399 52 52 52 93 93 93 F F F
70982024-02-03T17:12:38.622Z 20 400-419 65 65 65 110 110 110 F F F
70992024-02-03T17:12:38.623Z 21 420-439 65 65 65 110 110 110 F F F
71002024-02-03T17:12:38.623Z 22 440-459 65 65 65 110 110 110 F F F
71012024-02-03T17:12:38.623Z 23 460-479 65 65 65 110 110 110 F F F
71022024-02-03T17:12:38.623Z 24 480-499 65 65 65 110 110 110 F F F
71032024-02-03T17:12:38.623Z 25 500-519 65 65 65 110 110 110 F F F
71042024-02-03T17:12:38.623Z 26 520-539 65 65 65 110 110 110 F F F
71052024-02-03T17:12:38.623Z 27 540-559 57 57 57 98 98 98 F F F
71062024-02-03T17:12:38.623Z 28 560-579 46 46 46 82 82 82 F F F
71072024-02-03T17:12:38.623Z 29 580-599 65 65 65 110 110 110 F F F
71082024-02-03T17:12:38.623ZMax gen: 65, Max flush: 110
71092024-02-03T17:12:38.623Z
71102024-02-03T17:12:38.623ZBegin loop 33
71112024-02-03T17:12:38.623ZDownstairs to restart: 2
71122024-02-03T17:12:38.653Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.202193814Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396}
71132024-02-03T17:12:38.654Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.202702724Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396}
71142024-02-03T17:12:38.654ZCrucible runtime is spawned
71152024-02-03T17:12:38.681Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.202803048Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396}
71162024-02-03T17:12:38.681Z{"msg":"Crucible 683822a2-65a2-4c94-a4be-c36080551ae6 has session id: 79463b78-2332-4c60-b564-d28c4b1a2eac","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.202876639Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71172024-02-03T17:12:38.682Z{"msg":"Upstairs opts: Upstairs UUID: 683822a2-65a2-4c94-a4be-c36080551ae6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.202939871Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71182024-02-03T17:12:38.682Z{"msg":"Crucible stats registered with UUID: 683822a2-65a2-4c94-a4be-c36080551ae6","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.203005955Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71192024-02-03T17:12:38.682Z{"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.203046779Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396}
71202024-02-03T17:12:38.682Z{"msg":"683822a2-65a2-4c94-a4be-c36080551ae6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.203175122Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71212024-02-03T17:12:38.682Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.203249416Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"client":"0","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71222024-02-03T17:12:38.682Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.203280722Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"client":"0","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71232024-02-03T17:12:38.682Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.203359352Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"client":"1","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71242024-02-03T17:12:38.682Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.203405952Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"client":"1","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71252024-02-03T17:12:38.682Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.203445864Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"client":"2","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71262024-02-03T17:12:38.682Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.20348504Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"client":"2","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71272024-02-03T17:12:38.682Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.203564507Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"":"io task","client":"0","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71282024-02-03T17:12:38.682Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.20360781Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"":"io task","client":"2","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71292024-02-03T17:12:38.683Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.20364949Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"":"io task","client":"1","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71302024-02-03T17:12:38.683Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.203935773Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"":"io task","client":"2","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71312024-02-03T17:12:38.683Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.204565693Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"":"io task","client":"1","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71322024-02-03T17:12:38.683Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.2046811Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"":"io task","client":"0","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71332024-02-03T17:12:38.683Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.205360758Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"client":"0","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71342024-02-03T17:12:38.683Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.205433317Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"client":"0","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71352024-02-03T17:12:38.683Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.205605804Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"client":"1","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71362024-02-03T17:12:38.683Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.20569502Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"client":"1","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71372024-02-03T17:12:38.683Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.205873273Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"client":"2","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71382024-02-03T17:12:38.683Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.20594217Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"client":"2","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71392024-02-03T17:12:38.683Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.206649309Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"client":"1","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71402024-02-03T17:12:38.683Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.20672474Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"client":"0","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71412024-02-03T17:12:38.684Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.206765023Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"client":"2","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71422024-02-03T17:12:38.684Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.207561969Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"client":"1","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71432024-02-03T17:12:38.684Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.207655717Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"client":"1","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71442024-02-03T17:12:38.684Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.20770517Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71452024-02-03T17:12:38.684Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.207766053Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"client":"2","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71462024-02-03T17:12:38.684Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.2077922Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"client":"2","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71472024-02-03T17:12:38.684Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.207839893Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71482024-02-03T17:12:38.684Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.20786262Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"client":"0","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71492024-02-03T17:12:38.684Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.207910071Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"client":"0","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71502024-02-03T17:12:38.684Z{"msg":"[0]R flush_numbers[0..12]: [100, 104, 110, 109, 107, 98, 102, 110, 110, 110, 110, 108]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.207935988Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"":"downstairs","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71512024-02-03T17:12:38.684Z{"msg":"[0]R generation[0..12]: [58, 61, 65, 64, 62, 57, 59, 65, 65, 65, 65, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.207983776Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"":"downstairs","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71522024-02-03T17:12:38.684Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.208010312Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"":"downstairs","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71532024-02-03T17:12:38.684Z{"msg":"[1]R flush_numbers[0..12]: [100, 104, 110, 109, 107, 98, 102, 110, 110, 110, 110, 108]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.208058976Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"":"downstairs","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71542024-02-03T17:12:38.684Z{"msg":"[1]R generation[0..12]: [58, 61, 65, 64, 62, 57, 59, 65, 65, 65, 65, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.208083972Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"":"downstairs","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71552024-02-03T17:12:38.685Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.208107579Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"":"downstairs","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71562024-02-03T17:12:38.685Z{"msg":"[2]R flush_numbers[0..12]: [100, 104, 110, 109, 107, 98, 102, 110, 110, 110, 110, 108]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.208130624Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"":"downstairs","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71572024-02-03T17:12:38.685Z{"msg":"[2]R generation[0..12]: [58, 61, 65, 64, 62, 57, 59, 65, 65, 65, 65, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.208152941Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"":"downstairs","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71582024-02-03T17:12:38.685Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
71592024-02-03T17:12:38.685Z,"time":"2024-02-03T17:12:38.208226117Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"":"downstairs","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71602024-02-03T17:12:38.685Z{"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.20825175Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"":"downstairs","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71612024-02-03T17:12:38.685Z{"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.208321291Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"":"downstairs","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71622024-02-03T17:12:38.685Z{"msg":"Next flush: 111","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.208345022Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"":"downstairs","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71632024-02-03T17:12:38.686Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.208366445Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"":"downstairs","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71642024-02-03T17:12:38.686Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.208388441Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71652024-02-03T17:12:38.686Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.208408332Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"client":"0","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71662024-02-03T17:12:38.686Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.208430011Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"client":"1","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71672024-02-03T17:12:38.686Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.208451621Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"client":"2","":"downstairs","session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71682024-02-03T17:12:38.686Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.208475352Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71692024-02-03T17:12:38.686Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.208498757Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71702024-02-03T17:12:38.686Z{"msg":"683822a2-65a2-4c94-a4be-c36080551ae6 is now active with session: 79463b78-2332-4c60-b564-d28c4b1a2eac","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.208520093Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71712024-02-03T17:12:38.686Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.208539703Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396,"session_id":"79463b78-2332-4c60-b564-d28c4b1a2eac"}
71722024-02-03T17:12:38.686Z{"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:38.208559841Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396}
71732024-02-03T17:12:38.686ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
71742024-02-03T17:12:38.686ZLoading write count information from file "/tmp/test_repair_verify.data"
71752024-02-03T17:12:38.686ZRun Repair workload
71762024-02-03T17:12:38.686Z01/30 Flush
71772024-02-03T17:12:38.686Z02/30 Write block 293 len 28672 data: 7 8 7 7 6 5 6
71782024-02-03T17:12:38.694Z03/30 Write block 24 len 16384 data: 9 8 8 8
71792024-02-03T17:12:38.694Z04/30 Write block 45 len 8192 data: 3 3
71802024-02-03T17:12:38.694Z05/30 Read block 259 len 16384
71812024-02-03T17:12:38.694Z06/30 Read block 474 len 16384
71822024-02-03T17:12:38.694Z07/30 Write block 456 len 20480 data: 2 2 2 3 5
71832024-02-03T17:12:38.694Z08/30 Flush
71842024-02-03T17:12:38.694Z09/30 Write block 148 len 28672 data: 5 5 4 5 4 4 4
71852024-02-03T17:12:38.694Z10/30 Read block 119 len 8192
71862024-02-03T17:12:38.695Z11/30 Flush
71872024-02-03T17:12:38.695Z12/30 Write block 1 len 40960 data: 2 4 5 5 5 6 5 6 7 7
71882024-02-03T17:12:38.695Z13/30 Read block 103 len 16384
71892024-02-03T17:12:38.695Z14/30 Write block 470 len 28672 data: 5 4 5 5 6 6 5
71902024-02-03T17:12:38.695Z15/30 Read block 273 len 4096
71912024-02-03T17:12:38.695Z16/30 Write block 477 len 24576 data: 6 6 8 8 7 6
71922024-02-03T17:12:38.695Z17/30 Read block 118 len 8192
71932024-02-03T17:12:38.695Z18/30 Write block 467 len 16384 data: 7 6 5 6
71942024-02-03T17:12:38.695Z19/30 Read block 91 len 24576
71952024-02-03T17:12:38.695Z20/30 Flush
71962024-02-03T17:12:38.702Z21/30 Write block 86 len 36864 data: 8 7 6 7 6 5 5 7 7
71972024-02-03T17:12:38.702Z22/30 Write block 352 len 4096 data: 5
71982024-02-03T17:12:38.702Z23/30 Read block 313 len 8192
71992024-02-03T17:12:38.702Z24/30 Read block 139 len 16384
72002024-02-03T17:12:38.702Z25/30 Write block 13 len 16384 data: 5 6 4 4
72012024-02-03T17:12:38.702Z26/30 Flush
72022024-02-03T17:12:38.712Z27/30 Write block 515 len 36864 data: 7 6 6 4 4 5 4 4 5
72032024-02-03T17:12:38.712Z28/30 Write block 537 len 8192 data: 3 3
72042024-02-03T17:12:38.712Z29/30 Write block 96 len 20480 data: 6 6 5 7 7
72052024-02-03T17:12:38.712Z30/30 Write block 44 len 24576 data: 3 4 4 4 4 4
72062024-02-03T17:12:38.712Z----------------------------------------------------------------
72072024-02-03T17:12:38.712Z Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:29
72082024-02-03T17:12:38.712ZStates: Active Active Active
72092024-02-03T17:12:38.712ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
72102024-02-03T17:12:38.712Z 2 Acked 1001 Write 7 Done Done Sent false
72112024-02-03T17:12:38.712Z 3 Acked 1002 Write 4 Done Done Sent false
72122024-02-03T17:12:38.712Z 4 Acked 1003 Write 2 Done Done Sent false
72132024-02-03T17:12:38.713Z 5 Acked 1004 Read 4 Done Done Sent false
72142024-02-03T17:12:38.713Z 6 Acked 1005 Read 4 Done Done Sent false
72152024-02-03T17:12:38.713Z 7 Acked 1006 Write 5 Done Done Sent false
72162024-02-03T17:12:38.713Z 8 Acked 1007 Flush 0 Done Done Sent false
72172024-02-03T17:12:38.713Z 9 Acked 1008 Write 7 Done Done Sent false
72182024-02-03T17:12:38.713Z 10 Acked 1009 Read 2 Done Done Sent false
72192024-02-03T17:12:38.713Z 11 Acked 1010 Flush 0 Done Done Sent false
72202024-02-03T17:12:38.713Z 12 Acked 1011 Write 10 Done Done Sent false
72212024-02-03T17:12:38.713Z 13 Acked 1012 Read 4 Done Done Sent false
72222024-02-03T17:12:38.713Z 14 Acked 1013 Write 7 Done Done Sent false
72232024-02-03T17:12:38.713Z 15 Acked 1014 Read 1 Done Done Sent false
72242024-02-03T17:12:38.713Z 16 Acked 1015 Write 6 Done Done Sent false
72252024-02-03T17:12:38.713Z 17 Acked 1016 Read 2 Done Done Sent false
72262024-02-03T17:12:38.714Z 18 Acked 1017 Write 4 Done Done Sent false
72272024-02-03T17:12:38.714Z 19 Acked 1018 Read 6 Done Done Sent false
72282024-02-03T17:12:38.714Z 20 Acked 1019 Flush 0 Done Done Sent false
72292024-02-03T17:12:38.714Z 21 Acked 1020 Write 9 Done Done Sent false
72302024-02-03T17:12:38.714Z 22 Acked 1021 Write 1 Done Done Sent false
72312024-02-03T17:12:38.714Z 23 Acked 1022 Read 2 Done Done Sent false
72322024-02-03T17:12:38.714Z 24 Acked 1023 Read 4 Done Done Sent false
72332024-02-03T17:12:38.714Z 25 Acked 1024 Write 4 Done Done Sent false
72342024-02-03T17:12:38.714Z 26 Acked 1025 Flush 0 Done Done Sent false
72352024-02-03T17:12:38.714Z 27 Acked 1026 Write 9 Done Done Sent false
72362024-02-03T17:12:38.714Z 28 Acked 1027 Write 2 Done Done Sent false
72372024-02-03T17:12:38.714Z 29 Acked 1028 Write 5 Done Done Sent false
72382024-02-03T17:12:38.715Z 30 Acked 1029 Write 6 Sent Sent Sent false
72392024-02-03T17:12:38.715Z STATES DS:0 DS:1 DS:2 TOTAL
72402024-02-03T17:12:38.715Z New 0 0 0 0
72412024-02-03T17:12:38.715Z Sent 1 1 29 31
72422024-02-03T17:12:38.715Z Done 28 28 0 56
72432024-02-03T17:12:38.715Z Skipped 0 0 0 0
72442024-02-03T17:12:38.715Z Error 0 0 0 0
72452024-02-03T17:12:38.715ZLast Flush: 1025 1025 1000
72462024-02-03T17:12:38.715ZDownstairs last five completed: 1000
72472024-02-03T17:12:38.715ZUpstairs last five completed: 30 29 28 27 26
72482024-02-03T17:12:38.715Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:12:38.25828577Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1396}
72492024-02-03T17:12:38.720ZWrote out file "/tmp/test_repair_verify.data"
72502024-02-03T17:12:38.720Z
72512024-02-03T17:12:38.725ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
72522024-02-03T17:12:38.746ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
72532024-02-03T17:12:38.746Z 0 000-019 66 66 58 115 115 100 F F F <---
72542024-02-03T17:12:38.746Z 1 020-039 66 66 61 112 112 104 F F F <---
72552024-02-03T17:12:38.746Z 2 040-059 66 66 65 112 112 110 T T F <---
72562024-02-03T17:12:38.750Z 3 060-079 64 64 64 109 109 109 F F F
72572024-02-03T17:12:38.750Z 4 080-099 66 66 62 115 115 107 T T F <---
72582024-02-03T17:12:38.750Z 5 100-119 57 57 57 98 98 98 T T F <---
72592024-02-03T17:12:38.750Z 6 120-139 59 59 59 102 102 102 F F F
72602024-02-03T17:12:38.750Z 7 140-159 66 66 65 113 113 110 F F F <---
72612024-02-03T17:12:38.750Z 8 160-179 65 65 65 110 110 110 F F F
72622024-02-03T17:12:38.750Z 9 180-199 65 65 65 110 110 110 F F F
72632024-02-03T17:12:38.750Z 10 200-219 65 65 65 110 110 110 F F F
72642024-02-03T17:12:38.750Z 11 220-239 63 63 63 108 108 108 F F F
72652024-02-03T17:12:38.750Z 12 240-259 53 53 53 94 94 94 F F F
72662024-02-03T17:12:38.750Z 13 260-279 65 65 65 110 110 110 F F F
72672024-02-03T17:12:38.750Z 14 280-299 66 66 65 112 112 110 F F F <---
72682024-02-03T17:12:38.750Z 15 300-319 61 61 61 104 104 104 F F F
72692024-02-03T17:12:38.750Z 16 320-339 62 62 62 105 105 105 F F F
72702024-02-03T17:12:38.750Z 17 340-359 66 66 62 115 115 105 F F F <---
72712024-02-03T17:12:38.750Z 18 360-379 62 62 62 106 106 106 F F F
72722024-02-03T17:12:38.750Z 19 380-399 52 52 52 93 93 93 F F F
72732024-02-03T17:12:38.750Z 20 400-419 65 65 65 110 110 110 F F F
72742024-02-03T17:12:38.751Z 21 420-439 65 65 65 110 110 110 F F F
72752024-02-03T17:12:38.751Z 22 440-459 66 66 65 112 112 110 F F F <---
72762024-02-03T17:12:38.751Z 23 460-479 66 66 65 114 114 110 F F F <---
72772024-02-03T17:12:38.751Z 24 480-499 66 66 65 114 114 110 F F F <---
72782024-02-03T17:12:38.751Z 25 500-519 65 65 65 110 110 110 T T F <---
72792024-02-03T17:12:38.751Z 26 520-539 65 65 65 110 110 110 T T F <---
72802024-02-03T17:12:38.751Z 27 540-559 57 57 57 98 98 98 F F F
72812024-02-03T17:12:38.751Z 28 560-579 46 46 46 82 82 82 F F F
72822024-02-03T17:12:38.751Z 29 580-599 65 65 65 110 110 110 F F F
72832024-02-03T17:12:38.751ZMax gen: 66, Max flush: 115
72842024-02-03T17:12:38.751ZError: Difference in extent metadata found!
72852024-02-03T17:12:38.757ZA Difference in extent metadata is expected here
72862024-02-03T17:12:38.757ZOn loop 33
72872024-02-03T17:12:38.757Z
72882024-02-03T17:12:38.757ZVerifying data now
72892024-02-03T17:12:43.894ZNo change after verify
72902024-02-03T17:12:43.894ZLoop: 33 Downstairs dump after verify (and repair):
72912024-02-03T17:12:43.915ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
72922024-02-03T17:12:43.915Z 0 000-019 66 66 66 115 115 115 F F F
72932024-02-03T17:12:43.915Z 1 020-039 66 66 66 112 112 112 F F F
72942024-02-03T17:12:43.915Z 2 040-059 67 67 67 116 116 116 F F F
72952024-02-03T17:12:43.919Z 3 060-079 64 64 64 109 109 109 F F F
72962024-02-03T17:12:43.919Z 4 080-099 67 67 67 116 116 116 F F F
72972024-02-03T17:12:43.919Z 5 100-119 67 67 67 116 116 116 F F F
72982024-02-03T17:12:43.919Z 6 120-139 59 59 59 102 102 102 F F F
72992024-02-03T17:12:43.919Z 7 140-159 66 66 66 113 113 113 F F F
73002024-02-03T17:12:43.919Z 8 160-179 65 65 65 110 110 110 F F F
73012024-02-03T17:12:43.919Z 9 180-199 65 65 65 110 110 110 F F F
73022024-02-03T17:12:43.919Z 10 200-219 65 65 65 110 110 110 F F F
73032024-02-03T17:12:43.919Z 11 220-239 63 63 63 108 108 108 F F F
73042024-02-03T17:12:43.919Z 12 240-259 53 53 53 94 94 94 F F F
73052024-02-03T17:12:43.919Z 13 260-279 65 65 65 110 110 110 F F F
73062024-02-03T17:12:43.919Z 14 280-299 66 66 66 112 112 112 F F F
73072024-02-03T17:12:43.919Z 15 300-319 61 61 61 104 104 104 F F F
73082024-02-03T17:12:43.919Z 16 320-339 62 62 62 105 105 105 F F F
73092024-02-03T17:12:43.919Z 17 340-359 66 66 66 115 115 115 F F F
73102024-02-03T17:12:43.919Z 18 360-379 62 62 62 106 106 106 F F F
73112024-02-03T17:12:43.919Z 19 380-399 52 52 52 93 93 93 F F F
73122024-02-03T17:12:43.919Z 20 400-419 65 65 65 110 110 110 F F F
73132024-02-03T17:12:43.919Z 21 420-439 65 65 65 110 110 110 F F F
73142024-02-03T17:12:43.919Z 22 440-459 66 66 66 112 112 112 F F F
73152024-02-03T17:12:43.919Z 23 460-479 66 66 66 114 114 114 F F F
73162024-02-03T17:12:43.920Z 24 480-499 66 66 66 114 114 114 F F F
73172024-02-03T17:12:43.920Z 25 500-519 67 67 67 116 116 116 F F F
73182024-02-03T17:12:43.920Z 26 520-539 67 67 67 116 116 116 F F F
73192024-02-03T17:12:43.920Z 27 540-559 57 57 57 98 98 98 F F F
73202024-02-03T17:12:43.920Z 28 560-579 46 46 46 82 82 82 F F F
73212024-02-03T17:12:43.920Z 29 580-599 65 65 65 110 110 110 F F F
73222024-02-03T17:12:43.920ZMax gen: 67, Max flush: 116
73232024-02-03T17:12:43.923Z
73242024-02-03T17:12:43.923ZBegin loop 34
73252024-02-03T17:12:43.923ZDownstairs to restart: 1
73262024-02-03T17:12:43.952Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.505144425Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404}
73272024-02-03T17:12:43.952Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.505688149Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404}
73282024-02-03T17:12:43.952Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.505770604Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404}
73292024-02-03T17:12:43.957Z{"msg":"Crucible fbadb26c-42f3-42c8-80e2-e5f0043d6993 has session id: 2abd3733-30b9-4bf8-aa9f-a6ea062e0d59","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.505842586Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404Crucible runtime is spawned
73302024-02-03T17:12:43.957Z,"session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73312024-02-03T17:12:43.957Z{"msg":"Upstairs opts: Upstairs UUID: fbadb26c-42f3-42c8-80e2-e5f0043d6993, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.506057445Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73322024-02-03T17:12:43.957Z{"msg":"Crucible stats registered with UUID: fbadb26c-42f3-42c8-80e2-e5f0043d6993","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.506093076Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73332024-02-03T17:12:43.957Z{"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.50614253Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404}
73342024-02-03T17:12:43.957Z{"msg":"fbadb26c-42f3-42c8-80e2-e5f0043d6993 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.506268354Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73352024-02-03T17:12:43.957Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.506327613Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"client":"0","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73362024-02-03T17:12:43.957Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.506354708Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"client":"0","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73372024-02-03T17:12:43.957Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.506377239Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"client":"1","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73382024-02-03T17:12:43.957Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.506399592Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"client":"1","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73392024-02-03T17:12:43.957Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.506433119Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"client":"2","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73402024-02-03T17:12:43.957Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.506508725Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"client":"2","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73412024-02-03T17:12:43.957Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.506557597Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"":"io task","client":"2","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73422024-02-03T17:12:43.957Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.506599881Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"":"io task","client":"1","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73432024-02-03T17:12:43.957Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.506721992Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"":"io task","client":"0","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73442024-02-03T17:12:43.957Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.506893522Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"":"io task","client":"2","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73452024-02-03T17:12:43.957Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.506947884Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"":"io task","client":"1","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73462024-02-03T17:12:43.957Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.507061049Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"":"io task","client":"0","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73472024-02-03T17:12:43.957Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.50840523Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"client":"2","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73482024-02-03T17:12:43.957Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.508580618Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"client":"2","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73492024-02-03T17:12:43.957Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.508817893Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"client":"0","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73502024-02-03T17:12:43.957Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.508924256Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"client":"0","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73512024-02-03T17:12:43.958Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.508954243Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"client":"1","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73522024-02-03T17:12:43.958Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.508978588Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"client":"1","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73532024-02-03T17:12:43.963Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.50957926Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"client":"0","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73542024-02-03T17:12:43.963Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.509815213Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"client":"2","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73552024-02-03T17:12:43.963Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.510288785Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"client":"1","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73562024-02-03T17:12:43.963Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.510669488Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"client":"2","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73572024-02-03T17:12:43.963Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.510760154Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"client":"2","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73582024-02-03T17:12:43.963Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.510786342Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73592024-02-03T17:12:43.963Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.510954777Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"client":"0","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73602024-02-03T17:12:43.963Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.511035746Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"client":"0","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73612024-02-03T17:12:43.963Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.511061827Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73622024-02-03T17:12:43.963Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.511321573Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"client":"1","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73632024-02-03T17:12:43.963Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.511411846Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"client":"1","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73642024-02-03T17:12:43.963Z{"msg":"[0]R flush_numbers[0..12]: [115, 112, 116, 109, 116, 116, 102, 113, 110, 110, 110, 108]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.511440734Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"":"downstairs","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73652024-02-03T17:12:43.964Z{"msg":"[0]R generation[0..12]: [66, 66, 67, 64, 67, 67, 59, 66, 65, 65, 65, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.511464587Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"":"downstairs","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73662024-02-03T17:12:43.964Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.511488624Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"":"downstairs","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73672024-02-03T17:12:43.964Z{"msg":"[1]R flush_numbers[0..12]: [115, 112, 116, 109, 116, 116, 102, 113, 110, 110, 110, 108]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.511538086Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"":"downstairs","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73682024-02-03T17:12:43.964Z{"msg":"[1]R generation[0..12]: [66, 66, 67, 64, 67, 67, 59, 66, 65, 65, 65, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.511564459Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"":"downstairs","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73692024-02-03T17:12:43.964Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.511614251Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"":"downstairs","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73702024-02-03T17:12:43.964Z{"msg":"[2]R flush_numbers[0..12]: [115, 112, 116, 109, 116, 116, 102, 113, 110, 110, 110, 108]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.511640375Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"":"downstairs","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73712024-02-03T17:12:43.964Z{"msg":"[2]R generation[0..12]: [66, 66, 67, 64, 67, 67, 59, 66, 65, 65, 65, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.511663308Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal",Wait for a query_work_queue command to finish before sending IO
73722024-02-03T17:12:43.964Z"pid":1404,"":"downstairs","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73732024-02-03T17:12:43.964Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.511691578Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"":"downstairs","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73742024-02-03T17:12:43.964Z{"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.511713176Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"":"downstairs","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73752024-02-03T17:12:43.964Z{"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.511735137Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"":"downstairs","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73762024-02-03T17:12:43.964Z{"msg":"Next flush: 117","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.511756237Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"":"downstairs","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73772024-02-03T17:12:43.964Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.51180453Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"":"downstairs","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73782024-02-03T17:12:43.964Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.511829848Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73792024-02-03T17:12:43.964Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.511850163Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"client":"0","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73802024-02-03T17:12:43.964Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.511898224Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"client":"1","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73812024-02-03T17:12:43.964Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.511947581Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"client":"2","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73822024-02-03T17:12:43.964Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.511972279Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73832024-02-03T17:12:43.964Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.511992751Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73842024-02-03T17:12:43.964Z{"msg":"fbadb26c-42f3-42c8-80e2-e5f0043d6993 is now active with session: 2abd3733-30b9-4bf8-aa9f-a6ea062e0d59","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.512014277Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73852024-02-03T17:12:43.964Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.512033619Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
73862024-02-03T17:12:43.964Z{"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:43.51205382Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404}
73872024-02-03T17:12:43.964ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
73882024-02-03T17:12:43.964ZLoading write count information from file "/tmp/test_repair_verify.data"
73892024-02-03T17:12:43.965ZRun Repair workload
73902024-02-03T17:12:43.965Z01/30 Write block 227 len 8192 data: 3 5
73912024-02-03T17:12:43.968Z02/30 Write block 337 len 8192 data: 5 7
73922024-02-03T17:12:43.968Z03/30 Read block 597 len 4096
73932024-02-03T17:12:43.968Z04/30 Read block 180 len 24576
73942024-02-03T17:12:43.968Z05/30 Write block 409 len 28672 data: 4 3 4 6 6 7 7
73952024-02-03T17:12:43.971Z06/30 Write block 488 len 40960 data: 5 5 4 5 4 4 4 4 5 4
73962024-02-03T17:12:43.971Z07/30 Write block 310 len 40960 data: 8 6 7 8 7 7 7 8 5 8
73972024-02-03T17:12:43.974Z08/30 Write block 205 len 36864 data: 7 7 6 5 5 6 5 5 6
73982024-02-03T17:12:43.988Z09/30 Write block 151 len 28672 data: 6 5 5 5 6 8 7
73992024-02-03T17:12:43.988Z10/30 Write block 530 len 28672 data: 9 7 7 7 5 6 4
74002024-02-03T17:12:43.988Z11/30 Read block 54 len 20480
74012024-02-03T17:12:43.988Z12/30 Write block 159 len 24576 data: 6 8 7 9 9 10
74022024-02-03T17:12:43.988Z13/30 Write block 138 len 36864 data: 3 3 3 5 5 5 5 4 4
74032024-02-03T17:12:43.988Z14/30 Write block 398 len 8192 data: 7 7
74042024-02-03T17:12:43.988Z15/30 Write block 399 len 36864 data: 8 6 5 4 4 4 4 5 6
74052024-02-03T17:12:43.988Z16/30 Write block 262 len 20480 data: 6 7 8 10 10
74062024-02-03T17:12:43.989Z17/30 Flush
74072024-02-03T17:12:43.992Z18/30 Write block 50 len 24576 data: 5 6 6 4 5 5
74082024-02-03T17:12:43.995Z19/30 Read block 526 len 32768
74092024-02-03T17:12:43.995Z20/30 Read block 3 len 8192
74102024-02-03T17:12:44.007Z21/30 Read block 566 len 16384
74112024-02-03T17:12:44.007Z22/30 Write block 546 len 16384 data: 6 9 7 8
74122024-02-03T17:12:44.007Z23/30 Read block 404 len 40960
74132024-02-03T17:12:44.010Z24/30 Flush
74142024-02-03T17:12:44.010Z25/30 Write block 75 len 32768 data: 6 7 7 6 7 6 9 8
74152024-02-03T17:12:44.011Z26/30 Write block 203 len 12288 data: 6 5 8
74162024-02-03T17:12:44.011Z27/30 Write block 263 len 40960 data: 8 9 11 11 11 12 10 10 9 9
74172024-02-03T17:12:44.011Z28/30 Read block 567 len 12288
74182024-02-03T17:12:44.014Z29/30 Read block 566 len 4096
74192024-02-03T17:12:44.014Z30/30 Write block 511 len 28672 data: 9 8 8 7 8 7 7
74202024-02-03T17:12:44.014Z----------------------------------------------------------------
74212024-02-03T17:12:44.014Z Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:30
74222024-02-03T17:12:44.014ZStates: Active Active Active
74232024-02-03T17:12:44.014ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
74242024-02-03T17:12:44.014Z 1 Acked 1000 Write 2 Done Sent Done false
74252024-02-03T17:12:44.014Z 2 Acked 1001 Write 2 Done Sent Done false
74262024-02-03T17:12:44.014Z 3 Acked 1002 Read 1 Done Sent Done false
74272024-02-03T17:12:44.014Z 4 Acked 1003 Read 6 Done Sent Done false
74282024-02-03T17:12:44.014Z 5 Acked 1004 Write 7 Done Sent Done false
74292024-02-03T17:12:44.014Z 6 Acked 1005 Write 10 Done Sent Done false
74302024-02-03T17:12:44.014Z 7 Acked 1006 Write 10 Done Sent Done false
74312024-02-03T17:12:44.015Z 8 Acked 1007 Write 9 Done Sent Done false
74322024-02-03T17:12:44.015Z 9 Acked 1008 Write 7 Done Sent Done false
74332024-02-03T17:12:44.015Z 10 Acked 1009 Write 7 Done Sent Done false
74342024-02-03T17:12:44.015Z 11 Acked 1010 Read 5 Done Sent Done false
74352024-02-03T17:12:44.015Z 12 Acked 1011 Write 6 Done Sent Done false
74362024-02-03T17:12:44.015Z 13 Acked 1012 Write 9 Done Sent Done false
74372024-02-03T17:12:44.015Z 14 Acked 1013 Write 2 Done Sent Done false
74382024-02-03T17:12:44.015Z 15 Acked 1014 Write 9 Done Sent Done false
74392024-02-03T17:12:44.015Z 16 Acked 1015 Write 5 Done Sent Done false
74402024-02-03T17:12:44.015Z 17 Acked 1016 Flush 0 Done Sent Done false
74412024-02-03T17:12:44.015Z 18 Acked 1017 Write 6 Done Sent Done false
74422024-02-03T17:12:44.015Z 19 Acked 1018 Read 8 Done Sent Done false
74432024-02-03T17:12:44.015Z 20 Acked 1019 Read 2 Done Sent Done false
74442024-02-03T17:12:44.015Z 21 Acked 1020 Read 4 Done Sent Done false
74452024-02-03T17:12:44.015Z 22 Acked 1021 Write 4 Done Sent Done false
74462024-02-03T17:12:44.015Z 23 Acked 1022 Read 10 Done Sent Done false
74472024-02-03T17:12:44.015Z 24 Acked 1023 Flush 0 Done Sent Done false
74482024-02-03T17:12:44.015Z 25 Acked 1024 Write 8 Done Sent Done false
74492024-02-03T17:12:44.015Z 26 Acked 1025 Write 3 Done Sent Done false
74502024-02-03T17:12:44.015Z 27 Acked 1026 Write 10 Done Sent Done false
74512024-02-03T17:12:44.015Z 28 Acked 1027 Read 3 Done Sent Done false
74522024-02-03T17:12:44.015Z 29 Acked 1028 Read 1 Done Sent Done false
74532024-02-03T17:12:44.015Z 30 Acked 1029 Write 7 Sent Sent Sent false
74542024-02-03T17:12:44.015Z STATES DS:0 DS:1 DS:2 TOTAL
74552024-02-03T17:12:44.015Z New 0 0 0 0
74562024-02-03T17:12:44.016Z Sent 1 30 1 32
74572024-02-03T17:12:44.016Z Done 29 0 29 58
74582024-02-03T17:12:44.016Z Skipped 0 0 0 0
74592024-02-03T17:12:44.016Z Error 0 0 0 0
74602024-02-03T17:12:44.016ZLast Flush: 1023 0 1023
74612024-02-03T17:12:44.016ZDownstairs last five completed:
74622024-02-03T17:12:44.016ZUpstairs last five completed: 30 29 28 27 26
74632024-02-03T17:12:44.019Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:12:43.567468583Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404}
74642024-02-03T17:12:44.019ZWrote out file "/tmp/test_repair_verify.data"
74652024-02-03T17:12:44.019Z{"msg":"client response queue closed unexpectedly: channel closed; is the program exiting?","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:12:43.56826358Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1404,"":"io task","client":"0","":"downstairs","session_id":"2abd3733-30b9-4bf8-aa9f-a6ea062e0d59"}
74662024-02-03T17:12:44.022Z
74672024-02-03T17:12:44.028ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
74682024-02-03T17:12:44.053ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
74692024-02-03T17:12:44.053Z 0 000-019 66 66 66 115 115 115 F F F
74702024-02-03T17:12:44.053Z 1 020-039 66 66 66 112 112 112 F F F
74712024-02-03T17:12:44.053Z 2 040-059 68 67 68 118 116 118 F F F <---
74722024-02-03T17:12:44.056Z 3 060-079 64 64 64 109 109 109 T F T <---
74732024-02-03T17:12:44.056Z 4 080-099 67 67 67 116 116 116 T F T <---
74742024-02-03T17:12:44.056Z 5 100-119 67 67 67 116 116 116 F F F
74752024-02-03T17:12:44.056Z 6 120-139 68 59 68 117 102 117 F F F <---
74762024-02-03T17:12:44.056Z 7 140-159 68 66 68 117 113 117 F F F <---
74772024-02-03T17:12:44.056Z 8 160-179 68 65 68 117 110 117 F F F <---
74782024-02-03T17:12:44.056Z 9 180-199 65 65 65 110 110 110 F F F
74792024-02-03T17:12:44.056Z 10 200-219 68 65 68 117 110 117 T F T <---
74802024-02-03T17:12:44.056Z 11 220-239 68 63 68 117 108 117 F F F <---
74812024-02-03T17:12:44.056Z 12 240-259 53 53 53 94 94 94 F F F
74822024-02-03T17:12:44.057Z 13 260-279 68 65 68 117 110 117 T F T <---
74832024-02-03T17:12:44.057Z 14 280-299 66 66 66 112 112 112 F F F
74842024-02-03T17:12:44.057Z 15 300-319 68 61 68 117 104 117 F F F <---
74852024-02-03T17:12:44.057Z 16 320-339 68 62 68 117 105 117 F F F <---
74862024-02-03T17:12:44.057Z 17 340-359 66 66 66 115 115 115 F F F
74872024-02-03T17:12:44.057Z 18 360-379 62 62 62 106 106 106 F F F
74882024-02-03T17:12:44.057Z 19 380-399 68 52 68 117 93 117 F F F <---
74892024-02-03T17:12:44.057Z 20 400-419 68 65 68 117 110 117 F F F <---
74902024-02-03T17:12:44.057Z 21 420-439 65 65 65 110 110 110 F F F
74912024-02-03T17:12:44.057Z 22 440-459 66 66 66 112 112 112 F F F
74922024-02-03T17:12:44.057Z 23 460-479 66 66 66 114 114 114 F F F
74932024-02-03T17:12:44.057Z 24 480-499 68 66 68 117 114 117 F F F <---
74942024-02-03T17:12:44.057Z 25 500-519 67 67 67 116 116 116 T F T <---
74952024-02-03T17:12:44.057Z 26 520-539 68 67 68 117 116 117 F F F <---
74962024-02-03T17:12:44.057Z 27 540-559 68 57 68 118 98 118 F F F <---
74972024-02-03T17:12:44.057Z 28 560-579 46 46 46 82 82 82 F F F
74982024-02-03T17:12:44.057Z 29 580-599 65 65 65 110 110 110 F F F
74992024-02-03T17:12:44.057ZMax gen: 68, Max flush: 118
75002024-02-03T17:12:44.057ZError: Difference in extent metadata found!
75012024-02-03T17:12:44.060ZA Difference in extent metadata is expected here
75022024-02-03T17:12:44.060ZOn loop 34
75032024-02-03T17:12:44.061Z
75042024-02-03T17:12:44.066ZVerifying data now
75052024-02-03T17:12:49.481ZNo change after verify
75062024-02-03T17:12:49.481ZLoop: 34 Downstairs dump after verify (and repair):
75072024-02-03T17:12:49.501ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
75082024-02-03T17:12:49.502Z 0 000-019 66 66 66 115 115 115 F F F
75092024-02-03T17:12:49.502Z 1 020-039 66 66 66 112 112 112 F F F
75102024-02-03T17:12:49.502Z 2 040-059 68 68 68 118 118 118 F F F
75112024-02-03T17:12:49.502Z 3 060-079 69 69 69 119 119 119 F F F
75122024-02-03T17:12:49.506Z 4 080-099 69 69 69 119 119 119 F F F
75132024-02-03T17:12:49.506Z 5 100-119 67 67 67 116 116 116 F F F
75142024-02-03T17:12:49.506Z 6 120-139 68 68 68 117 117 117 F F F
75152024-02-03T17:12:49.506Z 7 140-159 68 68 68 117 117 117 F F F
75162024-02-03T17:12:49.506Z 8 160-179 68 68 68 117 117 117 F F F
75172024-02-03T17:12:49.506Z 9 180-199 65 65 65 110 110 110 F F F
75182024-02-03T17:12:49.506Z 10 200-219 69 69 69 119 119 119 F F F
75192024-02-03T17:12:49.506Z 11 220-239 68 68 68 117 117 117 F F F
75202024-02-03T17:12:49.506Z 12 240-259 53 53 53 94 94 94 F F F
75212024-02-03T17:12:49.506Z 13 260-279 69 69 69 119 119 119 F F F
75222024-02-03T17:12:49.506Z 14 280-299 66 66 66 112 112 112 F F F
75232024-02-03T17:12:49.506Z 15 300-319 68 68 68 117 117 117 F F F
75242024-02-03T17:12:49.506Z 16 320-339 68 68 68 117 117 117 F F F
75252024-02-03T17:12:49.506Z 17 340-359 66 66 66 115 115 115 F F F
75262024-02-03T17:12:49.506Z 18 360-379 62 62 62 106 106 106 F F F
75272024-02-03T17:12:49.506Z 19 380-399 68 68 68 117 117 117 F F F
75282024-02-03T17:12:49.506Z 20 400-419 68 68 68 117 117 117 F F F
75292024-02-03T17:12:49.506Z 21 420-439 65 65 65 110 110 110 F F F
75302024-02-03T17:12:49.507Z 22 440-459 66 66 66 112 112 112 F F F
75312024-02-03T17:12:49.507Z 23 460-479 66 66 66 114 114 114 F F F
75322024-02-03T17:12:49.507Z 24 480-499 68 68 68 117 117 117 F F F
75332024-02-03T17:12:49.507Z 25 500-519 69 69 69 119 119 119 F F F
75342024-02-03T17:12:49.507Z 26 520-539 68 68 68 117 117 117 F F F
75352024-02-03T17:12:49.507Z 27 540-559 68 68 68 118 118 118 F F F
75362024-02-03T17:12:49.507Z 28 560-579 46 46 46 82 82 82 F F F
75372024-02-03T17:12:49.507Z 29 580-599 65 65 65 110 110 110 F F F
75382024-02-03T17:12:49.507ZMax gen: 69, Max flush: 119
75392024-02-03T17:12:49.511Z
75402024-02-03T17:12:49.511ZBegin loop 35
75412024-02-03T17:12:49.511ZDownstairs to restart: 1
75422024-02-03T17:12:49.545Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.096151849Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412}
75432024-02-03T17:12:49.545Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.096682179Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412}
75442024-02-03T17:12:49.545Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.096718356Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412}
75452024-02-03T17:12:49.545ZCrucible runtime is spawned
75462024-02-03T17:12:49.551Z{"msg":"Crucible afeec0cd-1b38-4bcf-8b13-06e4aeb0dac9 has session id: 36836525-310b-4291-8d7d-f0a1bdbde1a7","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.096872933Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75472024-02-03T17:12:49.551Z{"msg":"Upstairs opts: Upstairs UUID: afeec0cd-1b38-4bcf-8b13-06e4aeb0dac9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.096982105Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75482024-02-03T17:12:49.552Z{"msg":"Crucible stats registered with UUID: afeec0cd-1b38-4bcf-8b13-06e4aeb0dac9","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.097063798Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75492024-02-03T17:12:49.552Z{"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.097092619Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412}
75502024-02-03T17:12:49.552Z{"msg":"afeec0cd-1b38-4bcf-8b13-06e4aeb0dac9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.097138617Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75512024-02-03T17:12:49.552Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.097161981Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"client":"0","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75522024-02-03T17:12:49.552Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.097229634Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"client":"0","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75532024-02-03T17:12:49.552Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.097254636Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"client":"1","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75542024-02-03T17:12:49.552Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.097277752Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"client":"1","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75552024-02-03T17:12:49.552Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.097299489Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"client":"2","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75562024-02-03T17:12:49.552Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.097348335Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"client":"2","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75572024-02-03T17:12:49.552Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.097372831Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"":"io task","client":"2","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75582024-02-03T17:12:49.552Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.097396222Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"":"io task","client":"0","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75592024-02-03T17:12:49.552Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.097449543Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"":"io task","client":"1","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75602024-02-03T17:12:49.552Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.097579386Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"":"io task","client":"2","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75612024-02-03T17:12:49.552Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.097817885Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"":"io task","client":"1","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75622024-02-03T17:12:49.552Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.097947828Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"":"io task","client":"0","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75632024-02-03T17:12:49.552Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.098720115Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"client":"2","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75642024-02-03T17:12:49.552Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.098790011Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"client":"2","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75652024-02-03T17:12:49.552Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.098838924Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"client":"0","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75662024-02-03T17:12:49.552Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.098921473Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"client":"0","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75672024-02-03T17:12:49.552Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.099093384Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"client":"1","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75682024-02-03T17:12:49.553Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.09914914Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"client":"1","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75692024-02-03T17:12:49.553Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.100088956Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"client":"0","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75702024-02-03T17:12:49.553Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.100181722Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"client":"2","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75712024-02-03T17:12:49.553Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.100383902Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"client":"1","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75722024-02-03T17:12:49.553Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.101160615Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"client":"0","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75732024-02-03T17:12:49.553Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.101238399Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"client":"0","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75742024-02-03T17:12:49.553Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.101266367Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75752024-02-03T17:12:49.553Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.10137417Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"client":"2","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75762024-02-03T17:12:49.553Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.101433686Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"client":"2","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75772024-02-03T17:12:49.553Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.101459029Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75782024-02-03T17:12:49.553Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.101583804Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"client":"1","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75792024-02-03T17:12:49.553Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.101641626Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"client":"1","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75802024-02-03T17:12:49.553Z{"msg":"[0]R flush_numbers[0..12]: [115, 112, 118, 119, 119, 116, 117, 117, 117, 110, 119, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.101668171Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"":"downstairs","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75812024-02-03T17:12:49.553Z{"msg":"[0]R generation[0..12]: [66, 66, 68, 69, 69, 67, 68, 68, 68, 65, 69, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.101717213Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"":"downstairs","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75822024-02-03T17:12:49.553Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.101743096Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"":"downstairs","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75832024-02-03T17:12:49.553Z{"msg":"[1]R flush_numbers[0..12]: [115, 112, 118, 119, 119, 116, 117, 117, 117, 110, 119, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.101792109Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"":"downstairs","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75842024-02-03T17:12:49.553Z{"msg":"[1]R generation[0..12]: [66, 66, 68, 69, 69, 67, 68, 68, 68, 65, 69, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.101816981Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"":"downstairs","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75852024-02-03T17:12:49.553Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.101840728Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"":"downstairs","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75862024-02-03T17:12:49.553Z{"msg":"[2]R flush_numbers[0..12]: [115, 112, 118, 119, 119, 116, 117, 117, 117, 110, 119, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.101889358Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"":"downstairs","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75872024-02-03T17:12:49.553Z{"msg":"[2]R generation[0..12]: [66, 66, 68, 69, 69, 67, 68, 68, 68, 65, 69, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.101914044Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"":"downstairs","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75882024-02-03T17:12:49.553ZWait for a query_work_queue command to finish before sending IO
75892024-02-03T17:12:49.553Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.101965287Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"":"downstairs","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75902024-02-03T17:12:49.553Z{"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.101989108Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"":"downstairs","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75912024-02-03T17:12:49.553Z{"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.102037128Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"":"downstairs","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75922024-02-03T17:12:49.554Z{"msg":"Next flush: 120","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.102059942Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"":"downstairs","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75932024-02-03T17:12:49.554Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.102081196Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"":"downstairs","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75942024-02-03T17:12:49.554Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.102130818Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75952024-02-03T17:12:49.554Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.102152654Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"client":"0","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75962024-02-03T17:12:49.554Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.10217413Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"client":"1","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75972024-02-03T17:12:49.554Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.102221076Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"client":"2","":"downstairs","session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75982024-02-03T17:12:49.554Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.102244697Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
75992024-02-03T17:12:49.554Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.102272483Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
76002024-02-03T17:12:49.558Z{"msg":"afeec0cd-1b38-4bcf-8b13-06e4aeb0dac9 is now active with session: 36836525-310b-4291-8d7d-f0a1bdbde1a7","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.102311379Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
76012024-02-03T17:12:49.558Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.102393867Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412,"session_id":"36836525-310b-4291-8d7d-f0a1bdbde1a7"}
76022024-02-03T17:12:49.558Z{"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:49.102436014Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412}
76032024-02-03T17:12:49.558ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
76042024-02-03T17:12:49.558ZLoading write count information from file "/tmp/test_repair_verify.data"
76052024-02-03T17:12:49.558ZRun Repair workload
76062024-02-03T17:12:49.558Z01/30 Write block 121 len 16384 data: 5 6 5 4
76072024-02-03T17:12:49.558Z02/30 Read block 505 len 40960
76082024-02-03T17:12:49.558Z03/30 Read block 593 len 16384
76092024-02-03T17:12:49.558Z04/30 Read block 33 len 32768
76102024-02-03T17:12:49.558Z05/30 Read block 304 len 8192
76112024-02-03T17:12:49.562Z06/30 Write block 151 len 32768 data: 7 6 6 6 7 9 8 8
76122024-02-03T17:12:49.562Z07/30 Write block 562 len 40960 data: 6 6 7 6 6 5 8 6 6 5
76132024-02-03T17:12:49.562Z08/30 Write block 427 len 24576 data: 6 6 9 9 7 8
76142024-02-03T17:12:49.566Z09/30 Read block 30 len 20480
76152024-02-03T17:12:49.569Z10/30 Write block 56 len 32768 data: 5 4 5 5 4 2 3 3
76162024-02-03T17:12:49.569Z11/30 Read block 70 len 16384
76172024-02-03T17:12:49.572Z12/30 Write block 238 len 36864 data: 2 2 2 4 4 4 5 5 7
76182024-02-03T17:12:49.572Z13/30 Read block 434 len 20480
76192024-02-03T17:12:49.576Z14/30 Write block 569 len 24576 data: 7 7 6 5 6 6
76202024-02-03T17:12:49.576Z15/30 Read block 503 len 20480
76212024-02-03T17:12:49.579Z16/30 Write block 210 len 40960 data: 7 6 6 7 4 5 6 5 4 4
76222024-02-03T17:12:49.579Z17/30 Write block 278 len 24576 data: 8 10 9 9 10 11
76232024-02-03T17:12:49.579Z18/30 Read block 254 len 16384
76242024-02-03T17:12:49.589Z19/30 Write block 89 len 40960 data: 8 7 6 6 8 8 6 7 7 6
76252024-02-03T17:12:49.589Z20/30 Write block 591 len 32768 data: 6 7 7 6 5 6 4 3
76262024-02-03T17:12:49.592Z21/30 Read block 326 len 36864
76272024-02-03T17:12:49.592Z22/30 Read block 537 len 20480
76282024-02-03T17:12:49.592Z23/30 Read block 153 len 28672
76292024-02-03T17:12:49.592Z24/30 Read block 399 len 20480
76302024-02-03T17:12:49.592Z25/30 Read block 124 len 24576
76312024-02-03T17:12:49.596Z26/30 Read block 357 len 24576
76322024-02-03T17:12:49.596Z27/30 Read block 467 len 12288
76332024-02-03T17:12:49.600Z28/30 Write block 131 len 12288 data: 5 4 4
76342024-02-03T17:12:49.600Z29/30 Write block 128 len 28672 data: 5 6 6 6 5 5 4
76352024-02-03T17:12:49.600Z30/30 Write block 469 len 12288 data: 6 7 5
76362024-02-03T17:12:49.603Z----------------------------------------------------------------
76372024-02-03T17:12:49.603Z Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:30
76382024-02-03T17:12:49.604ZStates: Active Active Active
76392024-02-03T17:12:49.604ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
76402024-02-03T17:12:49.604Z 1 Acked 1000 Write 4 Done Sent Done false
76412024-02-03T17:12:49.604Z 2 Acked 1001 Read 10 Done Sent Done false
76422024-02-03T17:12:49.604Z 3 Acked 1002 Read 4 Done Sent Done false
76432024-02-03T17:12:49.604Z 4 Acked 1003 Read 8 Done Sent Done false
76442024-02-03T17:12:49.604Z 5 Acked 1004 Read 2 Done Sent Done false
76452024-02-03T17:12:49.604Z 6 Acked 1005 Write 8 Done Sent Done false
76462024-02-03T17:12:49.604Z 7 Acked 1006 Write 10 Done Sent Done false
76472024-02-03T17:12:49.604Z 8 Acked 1007 Write 6 Done Sent Done false
76482024-02-03T17:12:49.604Z 9 Acked 1008 Read 5 Done Sent Done false
76492024-02-03T17:12:49.604Z 10 Acked 1009 Write 8 Done Sent Done false
76502024-02-03T17:12:49.604Z 11 Acked 1010 Read 4 Done Sent Done false
76512024-02-03T17:12:49.605Z 12 Acked 1011 Write 9 Done Sent Done false
76522024-02-03T17:12:49.605Z 13 Acked 1012 Read 5 Done Sent Done false
76532024-02-03T17:12:49.605Z 14 Acked 1013 Write 6 Done Sent Done false
76542024-02-03T17:12:49.605Z 15 Acked 1014 Read 5 Done Sent Done false
76552024-02-03T17:12:49.605Z 16 Acked 1015 Write 10 Done Sent Done false
76562024-02-03T17:12:49.605Z 17 Acked 1016 Write 6 Done Sent Done false
76572024-02-03T17:12:49.605Z 18 Acked 1017 Read 4 Done Sent Done false
76582024-02-03T17:12:49.605Z 19 Acked 1018 Write 10 Done Sent Done false
76592024-02-03T17:12:49.605Z 20 Acked 1019 Write 8 Done Sent Done false
76602024-02-03T17:12:49.605Z 21 Acked 1020 Read 9 Done Sent Done false
76612024-02-03T17:12:49.605Z 22 Acked 1021 Read 5 Done Sent Done false
76622024-02-03T17:12:49.605Z 23 Acked 1022 Read 7 Done Sent Done false
76632024-02-03T17:12:49.605Z 24 Acked 1023 Read 5 Done Sent Done false
76642024-02-03T17:12:49.606Z 25 Acked 1024 Read 6 Done Sent Done false
76652024-02-03T17:12:49.606Z 26 Acked 1025 Read 6 Done Sent Done false
76662024-02-03T17:12:49.606Z 27 Acked 1026 Read 3 Done Sent Done false
76672024-02-03T17:12:49.606Z 28 Acked 1027 Write 3 Done Sent Done false
76682024-02-03T17:12:49.606Z 29 Acked 1028 Write 7 Sent Sent Done false
76692024-02-03T17:12:49.606Z 30 Acked 1029 Write 3 Sent Sent Sent false
76702024-02-03T17:12:49.606Z STATES DS:0 DS:1 DS:2 TOTAL
76712024-02-03T17:12:49.606Z New 0 0 0 0
76722024-02-03T17:12:49.606Z Sent 2 30 1 33
76732024-02-03T17:12:49.606Z Done 28 0 29 57
76742024-02-03T17:12:49.606Z Skipped 0 0 0 0
76752024-02-03T17:12:49.606Z Error 0 0 0 0
76762024-02-03T17:12:49.606ZLast Flush: 0 0 0
76772024-02-03T17:12:49.607ZDownstairs last five completed:
76782024-02-03T17:12:49.607ZUpstairs last five completed: 30 29 28 27 26
76792024-02-03T17:12:49.607Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:12:49.154544917Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1412}
76802024-02-03T17:12:49.607ZWrote out file "/tmp/test_repair_verify.data"
76812024-02-03T17:12:49.610Z
76822024-02-03T17:12:49.616ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
76832024-02-03T17:12:49.647ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
76842024-02-03T17:12:49.647Z 0 000-019 66 66 66 115 115 115 F F F
76852024-02-03T17:12:49.647Z 1 020-039 66 66 66 112 112 112 F F F
76862024-02-03T17:12:49.647Z 2 040-059 68 68 68 118 118 118 T F T <---
76872024-02-03T17:12:49.647Z 3 060-079 69 69 69 119 119 119 T F T <---
76882024-02-03T17:12:49.654Z 4 080-099 69 69 69 119 119 119 T F T <---
76892024-02-03T17:12:49.655Z 5 100-119 67 67 67 116 116 116 F F F
76902024-02-03T17:12:49.655Z 6 120-139 68 68 68 117 117 117 T F T <---
76912024-02-03T17:12:49.655Z 7 140-159 68 68 68 117 117 117 T F T <---
76922024-02-03T17:12:49.656Z 8 160-179 68 68 68 117 117 117 F F F
76932024-02-03T17:12:49.656Z 9 180-199 65 65 65 110 110 110 F F F
76942024-02-03T17:12:49.656Z 10 200-219 69 69 69 119 119 119 T F T <---
76952024-02-03T17:12:49.656Z 11 220-239 68 68 68 117 117 117 T F T <---
76962024-02-03T17:12:49.656Z 12 240-259 53 53 53 94 94 94 T F T <---
76972024-02-03T17:12:49.656Z 13 260-279 69 69 69 119 119 119 T F T <---
76982024-02-03T17:12:49.656Z 14 280-299 66 66 66 112 112 112 T F T <---
76992024-02-03T17:12:49.657Z 15 300-319 68 68 68 117 117 117 F F F
77002024-02-03T17:12:49.657Z 16 320-339 68 68 68 117 117 117 F F F
77012024-02-03T17:12:49.657Z 17 340-359 66 66 66 115 115 115 F F F
77022024-02-03T17:12:49.657Z 18 360-379 62 62 62 106 106 106 F F F
77032024-02-03T17:12:49.657Z 19 380-399 68 68 68 117 117 117 F F F
77042024-02-03T17:12:49.658Z 20 400-419 68 68 68 117 117 117 F F F
77052024-02-03T17:12:49.658Z 21 420-439 65 65 65 110 110 110 T F T <---
77062024-02-03T17:12:49.658Z 22 440-459 66 66 66 112 112 112 F F F
77072024-02-03T17:12:49.658Z 23 460-479 66 66 66 114 114 114 T F T <---
77082024-02-03T17:12:49.658Z 24 480-499 68 68 68 117 117 117 F F F
77092024-02-03T17:12:49.658Z 25 500-519 69 69 69 119 119 119 F F F
77102024-02-03T17:12:49.658Z 26 520-539 68 68 68 117 117 117 F F F
77112024-02-03T17:12:49.659Z 27 540-559 68 68 68 118 118 118 F F F
77122024-02-03T17:12:49.659Z 28 560-579 46 46 46 82 82 82 T F T <---
77132024-02-03T17:12:49.659Z 29 580-599 65 65 65 110 110 110 T F T <---
77142024-02-03T17:12:49.660ZMax gen: 69, Max flush: 119
77152024-02-03T17:12:49.660ZError: Difference in extent metadata found!
77162024-02-03T17:12:49.672ZA Difference in extent metadata is expected here
77172024-02-03T17:12:49.672ZOn loop 35
77182024-02-03T17:12:49.672Z
77192024-02-03T17:12:49.672ZVerifying data now
77202024-02-03T17:12:54.882ZNo change after verify
77212024-02-03T17:12:54.882ZLoop: 35 Downstairs dump after verify (and repair):
77222024-02-03T17:12:54.905ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
77232024-02-03T17:12:54.905Z 0 000-019 66 66 66 115 115 115 F F F
77242024-02-03T17:12:54.905Z 1 020-039 66 66 66 112 112 112 F F F
77252024-02-03T17:12:54.905Z 2 040-059 70 70 70 120 120 120 F F F
77262024-02-03T17:12:54.905Z 3 060-079 70 70 70 120 120 120 F F F
77272024-02-03T17:12:54.908Z 4 080-099 70 70 70 120 120 120 F F F
77282024-02-03T17:12:54.908Z 5 100-119 67 67 67 116 116 116 F F F
77292024-02-03T17:12:54.908Z 6 120-139 70 70 70 120 120 120 F F F
77302024-02-03T17:12:54.908Z 7 140-159 70 70 70 120 120 120 F F F
77312024-02-03T17:12:54.908Z 8 160-179 68 68 68 117 117 117 F F F
77322024-02-03T17:12:54.908Z 9 180-199 65 65 65 110 110 110 F F F
77332024-02-03T17:12:54.908Z 10 200-219 70 70 70 120 120 120 F F F
77342024-02-03T17:12:54.908Z 11 220-239 70 70 70 120 120 120 F F F
77352024-02-03T17:12:54.908Z 12 240-259 70 70 70 120 120 120 F F F
77362024-02-03T17:12:54.908Z 13 260-279 70 70 70 120 120 120 F F F
77372024-02-03T17:12:54.908Z 14 280-299 70 70 70 120 120 120 F F F
77382024-02-03T17:12:54.908Z 15 300-319 68 68 68 117 117 117 F F F
77392024-02-03T17:12:54.909Z 16 320-339 68 68 68 117 117 117 F F F
77402024-02-03T17:12:54.909Z 17 340-359 66 66 66 115 115 115 F F F
77412024-02-03T17:12:54.909Z 18 360-379 62 62 62 106 106 106 F F F
77422024-02-03T17:12:54.909Z 19 380-399 68 68 68 117 117 117 F F F
77432024-02-03T17:12:54.909Z 20 400-419 68 68 68 117 117 117 F F F
77442024-02-03T17:12:54.909Z 21 420-439 70 70 70 120 120 120 F F F
77452024-02-03T17:12:54.909Z 22 440-459 66 66 66 112 112 112 F F F
77462024-02-03T17:12:54.909Z 23 460-479 70 70 70 120 120 120 F F F
77472024-02-03T17:12:54.909Z 24 480-499 68 68 68 117 117 117 F F F
77482024-02-03T17:12:54.909Z 25 500-519 69 69 69 119 119 119 F F F
77492024-02-03T17:12:54.909Z 26 520-539 68 68 68 117 117 117 F F F
77502024-02-03T17:12:54.909Z 27 540-559 68 68 68 118 118 118 F F F
77512024-02-03T17:12:54.909Z 28 560-579 70 70 70 120 120 120 F F F
77522024-02-03T17:12:54.909Z 29 580-599 70 70 70 120 120 120 F F F
77532024-02-03T17:12:54.909ZMax gen: 70, Max flush: 120
77542024-02-03T17:12:54.912Z
77552024-02-03T17:12:54.912ZBegin loop 36
77562024-02-03T17:12:54.912ZDownstairs to restart: 0
77572024-02-03T17:12:54.942Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.494166268Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420}
77582024-02-03T17:12:54.942Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.494715163Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420}
77592024-02-03T17:12:54.946Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.494756218Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420}
77602024-02-03T17:12:54.946Z{"msg":"Crucible e7a9d0d4-74f2-4430-beab-572e755bc595 has session id: a2834cf1-fd7c-47ec-a97d-b87214e658ad","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.494886947Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77612024-02-03T17:12:54.946Z{"msg":"Upstairs opts: Upstairs UUID: e7a9d0d4-74f2-4430-beab-572e755bc595, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.494947225Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77622024-02-03T17:12:54.946Z{"msg":"Crucible stats registered with UUID: e7a9d0d4-74f2-4430-beab-572e755bc595","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.495012184Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77632024-02-03T17:12:54.946ZCrucible runtime is spawned
77642024-02-03T17:12:54.946Z{"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.495226452Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420}
77652024-02-03T17:12:54.946Z{"msg":"e7a9d0d4-74f2-4430-beab-572e755bc595 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.495535748Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77662024-02-03T17:12:54.946Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.495624336Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"client":"0","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77672024-02-03T17:12:54.947Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.495672129Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"client":"0","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77682024-02-03T17:12:54.947Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.495755087Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"client":"1","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77692024-02-03T17:12:54.947Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.495800039Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"client":"1","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77702024-02-03T17:12:54.947Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.495838334Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"client":"2","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77712024-02-03T17:12:54.947Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.495878646Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"client":"2","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77722024-02-03T17:12:54.947Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.495920231Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"":"io task","client":"0","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77732024-02-03T17:12:54.947Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.496070774Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"":"io task","client":"2","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77742024-02-03T17:12:54.947Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.496189841Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"":"io task","client":"1","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77752024-02-03T17:12:54.947Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.496313727Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"":"io task","client":"2","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77762024-02-03T17:12:54.947Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.496411933Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"":"io task","client":"0","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77772024-02-03T17:12:54.947Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.496458776Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"":"io task","client":"1","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77782024-02-03T17:12:54.947Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.497275791Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"client":"1","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77792024-02-03T17:12:54.947Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.497441625Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"client":"1","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77802024-02-03T17:12:54.947Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.497582284Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"client":"2","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77812024-02-03T17:12:54.947Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.497631845Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"client":"2","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77822024-02-03T17:12:54.947Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.498374674Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"client":"0","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77832024-02-03T17:12:54.947Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.498444632Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"client":"0","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77842024-02-03T17:12:54.947Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.498490564Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"client":"2","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77852024-02-03T17:12:54.947Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.498538543Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"client":"1","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77862024-02-03T17:12:54.952Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.499522506Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"client":"1","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77872024-02-03T17:12:54.952Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.499620769Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"client":"1","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77882024-02-03T17:12:54.952Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.499711678Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77892024-02-03T17:12:54.952Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.499757544Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"client":"2","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77902024-02-03T17:12:54.952Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.499839392Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"client":"2","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77912024-02-03T17:12:54.952Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.499869339Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77922024-02-03T17:12:54.952Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.499891396Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"client":"0","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77932024-02-03T17:12:54.952Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.500493213Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"client":"0","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77942024-02-03T17:12:54.952Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.500568472Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"client":"0","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77952024-02-03T17:12:54.952Z{"msg":"[0]R flush_numbers[0..12]: [115, 112, 120, 120, 120, 116, 120, 120, 117, 110, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.500597428Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"":"downstairs","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77962024-02-03T17:12:54.953Z{"msg":"[0]R generation[0..12]: [66, 66, 70, 70, 70, 67, 70, 70, 68, 65, 70, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.500621083Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"":"downstairs","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77972024-02-03T17:12:54.953Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.500645406Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"":"downstairs","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77982024-02-03T17:12:54.953Z{"msg":"[1]R flush_numbers[0..12]: [115, 112, 120, 120, 120, 116, 120, 120, 117, 110, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.500693639Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"":"downstairs","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
77992024-02-03T17:12:54.953Z{"msg":"[1]R generation[0..12]: [66, 66, 70, 70, 70, 67, 70, 70, 68, 65, 70, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.500719104Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"":"downstairs","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
78002024-02-03T17:12:54.953Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.500767784Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"":"downstairs","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
78012024-02-03T17:12:54.953Z{"msg":"[2]R flush_numbers[0..12]: [115, 112, 120, 120, 120, 116, 120, 120, 117, 110, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.500793254Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"":"downstairs","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
78022024-02-03T17:12:54.953Z{"msg":"[2]R generation[0..12]: [66, 66, 70, 70, 70, 67, 70, 70, 68, 65, 70, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.500817719Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"":"downstairs","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
78032024-02-03T17:12:54.953Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"timeWait for a query_work_queue command to finish before sending IO
78042024-02-03T17:12:54.953Z":"2024-02-03T17:12:54.500841421Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"":"downstairs","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
78052024-02-03T17:12:54.953Z{"msg":"Max found gen is 71","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.500892886Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"":"downstairs","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
78062024-02-03T17:12:54.953Z{"msg":"Generation requested: 72 >= found:71","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.501037226Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"":"downstairs","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
78072024-02-03T17:12:54.953Z{"msg":"Next flush: 121","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.501069673Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"":"downstairs","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
78082024-02-03T17:12:54.953Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.501092671Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"":"downstairs","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
78092024-02-03T17:12:54.953Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.50111439Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
78102024-02-03T17:12:54.953Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.501136183Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"client":"0","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
78112024-02-03T17:12:54.953Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.501157833Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"client":"1","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
78122024-02-03T17:12:54.953Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.501179317Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"client":"2","":"downstairs","session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
78132024-02-03T17:12:54.953Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.501201171Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
78142024-02-03T17:12:54.953Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.501247585Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
78152024-02-03T17:12:54.953Z{"msg":"e7a9d0d4-74f2-4430-beab-572e755bc595 is now active with session: a2834cf1-fd7c-47ec-a97d-b87214e658ad","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.501271523Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
78162024-02-03T17:12:54.953Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.501291636Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420,"session_id":"a2834cf1-fd7c-47ec-a97d-b87214e658ad"}
78172024-02-03T17:12:54.953Z{"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:54.501338141Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420}
78182024-02-03T17:12:54.953ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
78192024-02-03T17:12:54.953ZLoading write count information from file "/tmp/test_repair_verify.data"
78202024-02-03T17:12:54.953ZRun Repair workload
78212024-02-03T17:12:54.954Z01/30 Write block 410 len 32768 data: 4 5 7 7 8 8 7 9
78222024-02-03T17:12:54.957Z02/30 Read block 589 len 8192
78232024-02-03T17:12:54.957Z03/30 Read block 242 len 28672
78242024-02-03T17:12:54.957Z04/30 Read block 590 len 4096
78252024-02-03T17:12:54.957Z05/30 Flush
78262024-02-03T17:12:54.962Z06/30 Write block 398 len 36864 data: 8 9 7 6 5 5 5 5 6
78272024-02-03T17:12:54.962Z07/30 Read block 371 len 36864
78282024-02-03T17:12:54.965Z08/30 Write block 546 len 8192 data: 7 10
78292024-02-03T17:12:54.965Z09/30 Write block 289 len 32768 data: 5 6 7 6 8 9 8 8
78302024-02-03T17:12:54.965Z10/30 Write block 478 len 12288 data: 7 9 9
78312024-02-03T17:12:54.968Z11/30 Read block 145 len 36864
78322024-02-03T17:12:54.968Z12/30 Read block 5 len 32768
78332024-02-03T17:12:54.973Z13/30 Write block 462 len 20480 data: 4 6 6 7 8
78342024-02-03T17:12:54.973Z14/30 Write block 481 len 36864 data: 8 7 6 5 4 5 3 6 6
78352024-02-03T17:12:54.977Z15/30 Read block 391 len 4096
78362024-02-03T17:12:54.977Z16/30 Read block 551 len 32768
78372024-02-03T17:12:54.977Z17/30 Write block 187 len 40960 data: 10 13 13 13 11 11 8 8 8 6
78382024-02-03T17:12:54.980Z18/30 Write block 591 len 28672 data: 7 8 8 7 6 7 5
78392024-02-03T17:12:54.980Z19/30 Read block 219 len 16384
78402024-02-03T17:12:54.980Z20/30 Read block 43 len 20480
78412024-02-03T17:12:54.983Z21/30 Read block 365 len 4096
78422024-02-03T17:12:54.983Z22/30 Read block 79 len 32768
78432024-02-03T17:12:54.986Z23/30 Read block 348 len 36864
78442024-02-03T17:12:54.986Z24/30 Read block 302 len 24576
78452024-02-03T17:12:54.989Z25/30 Write block 278 len 20480 data: 9 11 10 10 11
78462024-02-03T17:12:54.989Z26/30 Read block 312 len 24576
78472024-02-03T17:12:54.993Z27/30 Write block 457 len 28672 data: 3 3 4 6 5 5 7
78482024-02-03T17:12:54.996Z28/30 Write block 491 len 20480 data: 6 5 5 5 5
78492024-02-03T17:12:54.996Z29/30 Read block 395 len 40960
78502024-02-03T17:12:54.996Z30/30 Write block 482 len 32768 data: 8 7 6 5 6 4 7 7
78512024-02-03T17:12:55.000Z----------------------------------------------------------------
78522024-02-03T17:12:55.000Z Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30
78532024-02-03T17:12:55.000ZStates: Active Active Active
78542024-02-03T17:12:55.000ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
78552024-02-03T17:12:55.000Z 1 Acked 1000 Write 8 Sent Done Done false
78562024-02-03T17:12:55.000Z 2 Acked 1001 Read 2 Sent Done Done false
78572024-02-03T17:12:55.000Z 3 Acked 1002 Read 7 Sent Done Done false
78582024-02-03T17:12:55.000Z 4 Acked 1003 Read 1 Sent Done Done false
78592024-02-03T17:12:55.000Z 5 Acked 1004 Flush 0 Sent Done Done false
78602024-02-03T17:12:55.000Z 6 Acked 1005 Write 9 Sent Done Done false
78612024-02-03T17:12:55.001Z 7 Acked 1006 Read 9 Sent Done Done false
78622024-02-03T17:12:55.001Z 8 Acked 1007 Write 2 Sent Done Done false
78632024-02-03T17:12:55.001Z 9 Acked 1008 Write 8 Sent Done Done false
78642024-02-03T17:12:55.001Z 10 Acked 1009 Write 3 Sent Done Done false
78652024-02-03T17:12:55.001Z 11 Acked 1010 Read 9 Sent Done Done false
78662024-02-03T17:12:55.001Z 12 Acked 1011 Read 8 Sent Done Done false
78672024-02-03T17:12:55.001Z 13 Acked 1012 Write 5 Sent Done Done false
78682024-02-03T17:12:55.001Z 14 Acked 1013 Write 9 Sent Done Done false
78692024-02-03T17:12:55.001Z 15 Acked 1014 Read 1 Sent Done Done false
78702024-02-03T17:12:55.001Z 16 Acked 1015 Read 8 Sent Done Done false
78712024-02-03T17:12:55.001Z 17 Acked 1016 Write 10 Sent Done Done false
78722024-02-03T17:12:55.001Z 18 Acked 1017 Write 7 Sent Done Done false
78732024-02-03T17:12:55.002Z 19 Acked 1018 Read 4 Sent Done Done false
78742024-02-03T17:12:55.002Z 20 Acked 1019 Read 5 Sent Done Done false
78752024-02-03T17:12:55.002Z 21 Acked 1020 Read 1 Sent Done Done false
78762024-02-03T17:12:55.002Z 22 Acked 1021 Read 8 Sent Done Done false
78772024-02-03T17:12:55.002Z 23 Acked 1022 Read 9 Sent Done Done false
78782024-02-03T17:12:55.002Z 24 Acked 1023 Read 6 Sent Done Done false
78792024-02-03T17:12:55.002Z 25 Acked 1024 Write 5 Sent Done Done false
78802024-02-03T17:12:55.002Z 26 Acked 1025 Read 6 Sent Done Done false
78812024-02-03T17:12:55.002Z 27 Acked 1026 Write 7 Sent Done Done false
78822024-02-03T17:12:55.002Z 28 Acked 1027 Write 5 Sent Done Done false
78832024-02-03T17:12:55.002Z 29 Acked 1028 Read 10 Sent Done Done false
78842024-02-03T17:12:55.002Z 30 Acked 1029 Write 8 Sent Sent Sent false
78852024-02-03T17:12:55.002Z STATES DS:0 DS:1 DS:2 TOTAL
78862024-02-03T17:12:55.002Z New 0 0 0 0
78872024-02-03T17:12:55.002Z Sent 30 1 1 32
78882024-02-03T17:12:55.002Z Done 0 29 29 58
78892024-02-03T17:12:55.002Z Skipped 0 0 0 0
78902024-02-03T17:12:55.002Z Error 0 0 0 0
78912024-02-03T17:12:55.002ZLast Flush: 0 1004 1004
78922024-02-03T17:12:55.003ZDownstairs last five completed:
78932024-02-03T17:12:55.003ZUpstairs last five completed: 30 29 28 27 26
78942024-02-03T17:12:55.003Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:12:54.551607539Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1420}
78952024-02-03T17:12:55.003ZWrote out file "/tmp/test_repair_verify.data"
78962024-02-03T17:12:55.007Z
78972024-02-03T17:12:55.011ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
78982024-02-03T17:12:55.039ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
78992024-02-03T17:12:55.039Z 0 000-019 66 66 66 115 115 115 F F F
79002024-02-03T17:12:55.039Z 1 020-039 66 66 66 112 112 112 F F F
79012024-02-03T17:12:55.045Z 2 040-059 70 70 70 120 120 120 F F F
79022024-02-03T17:12:55.045Z 3 060-079 70 70 70 120 120 120 F F F
79032024-02-03T17:12:55.045Z 4 080-099 70 70 70 120 120 120 F F F
79042024-02-03T17:12:55.045Z 5 100-119 67 67 67 116 116 116 F F F
79052024-02-03T17:12:55.046Z 6 120-139 70 70 70 120 120 120 F F F
79062024-02-03T17:12:55.046Z 7 140-159 70 70 70 120 120 120 F F F
79072024-02-03T17:12:55.046Z 8 160-179 68 68 68 117 117 117 F F F
79082024-02-03T17:12:55.046Z 9 180-199 65 65 65 110 110 110 F T T <---
79092024-02-03T17:12:55.046Z 10 200-219 70 70 70 120 120 120 F F F
79102024-02-03T17:12:55.046Z 11 220-239 70 70 70 120 120 120 F F F
79112024-02-03T17:12:55.046Z 12 240-259 70 70 70 120 120 120 F F F
79122024-02-03T17:12:55.046Z 13 260-279 70 70 70 120 120 120 F T T <---
79132024-02-03T17:12:55.046Z 14 280-299 70 70 70 120 120 120 F T T <---
79142024-02-03T17:12:55.046Z 15 300-319 68 68 68 117 117 117 F F F
79152024-02-03T17:12:55.046Z 16 320-339 68 68 68 117 117 117 F F F
79162024-02-03T17:12:55.046Z 17 340-359 66 66 66 115 115 115 F F F
79172024-02-03T17:12:55.046Z 18 360-379 62 62 62 106 106 106 F F F
79182024-02-03T17:12:55.046Z 19 380-399 68 68 68 117 117 117 F T T <---
79192024-02-03T17:12:55.046Z 20 400-419 68 72 72 117 121 121 F T T <---
79202024-02-03T17:12:55.046Z 21 420-439 70 70 70 120 120 120 F F F
79212024-02-03T17:12:55.046Z 22 440-459 66 66 66 112 112 112 F T T <---
79222024-02-03T17:12:55.046Z 23 460-479 70 70 70 120 120 120 F T T <---
79232024-02-03T17:12:55.046Z 24 480-499 68 68 68 117 117 117 F T T <---
79242024-02-03T17:12:55.046Z 25 500-519 69 69 69 119 119 119 F F F
79252024-02-03T17:12:55.046Z 26 520-539 68 68 68 117 117 117 F F F
79262024-02-03T17:12:55.046Z 27 540-559 68 68 68 118 118 118 F T T <---
79272024-02-03T17:12:55.046Z 28 560-579 70 70 70 120 120 120 F F F
79282024-02-03T17:12:55.046Z 29 580-599 70 70 70 120 120 120 F T T <---
79292024-02-03T17:12:55.046ZMax gen: 72, Max flush: 121
79302024-02-03T17:12:55.047ZError: Difference in extent metadata found!
79312024-02-03T17:12:55.050ZA Difference in extent metadata is expected here
79322024-02-03T17:12:55.050ZOn loop 36
79332024-02-03T17:12:55.050Z
79342024-02-03T17:12:55.050ZVerifying data now
79352024-02-03T17:12:59.982ZNo change after verify
79362024-02-03T17:12:59.982ZLoop: 36 Downstairs dump after verify (and repair):
79372024-02-03T17:13:00.004ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
79382024-02-03T17:13:00.004Z 0 000-019 66 66 66 115 115 115 F F F
79392024-02-03T17:13:00.004Z 1 020-039 66 66 66 112 112 112 F F F
79402024-02-03T17:13:00.004Z 2 040-059 70 70 70 120 120 120 F F F
79412024-02-03T17:13:00.010Z 3 060-079 70 70 70 120 120 120 F F F
79422024-02-03T17:13:00.010Z 4 080-099 70 70 70 120 120 120 F F F
79432024-02-03T17:13:00.010Z 5 100-119 67 67 67 116 116 116 F F F
79442024-02-03T17:13:00.010Z 6 120-139 70 70 70 120 120 120 F F F
79452024-02-03T17:13:00.010Z 7 140-159 70 70 70 120 120 120 F F F
79462024-02-03T17:13:00.010Z 8 160-179 68 68 68 117 117 117 F F F
79472024-02-03T17:13:00.010Z 9 180-199 73 73 73 122 122 122 F F F
79482024-02-03T17:13:00.010Z 10 200-219 70 70 70 120 120 120 F F F
79492024-02-03T17:13:00.010Z 11 220-239 70 70 70 120 120 120 F F F
79502024-02-03T17:13:00.010Z 12 240-259 70 70 70 120 120 120 F F F
79512024-02-03T17:13:00.011Z 13 260-279 73 73 73 122 122 122 F F F
79522024-02-03T17:13:00.011Z 14 280-299 73 73 73 122 122 122 F F F
79532024-02-03T17:13:00.011Z 15 300-319 68 68 68 117 117 117 F F F
79542024-02-03T17:13:00.011Z 16 320-339 68 68 68 117 117 117 F F F
79552024-02-03T17:13:00.011Z 17 340-359 66 66 66 115 115 115 F F F
79562024-02-03T17:13:00.011Z 18 360-379 62 62 62 106 106 106 F F F
79572024-02-03T17:13:00.011Z 19 380-399 73 73 73 122 122 122 F F F
79582024-02-03T17:13:00.011Z 20 400-419 73 73 73 122 122 122 F F F
79592024-02-03T17:13:00.011Z 21 420-439 70 70 70 120 120 120 F F F
79602024-02-03T17:13:00.011Z 22 440-459 73 73 73 122 122 122 F F F
79612024-02-03T17:13:00.011Z 23 460-479 73 73 73 122 122 122 F F F
79622024-02-03T17:13:00.011Z 24 480-499 73 73 73 122 122 122 F F F
79632024-02-03T17:13:00.011Z 25 500-519 69 69 69 119 119 119 F F F
79642024-02-03T17:13:00.011Z 26 520-539 68 68 68 117 117 117 F F F
79652024-02-03T17:13:00.012Z 27 540-559 73 73 73 122 122 122 F F F
79662024-02-03T17:13:00.012Z 28 560-579 70 70 70 120 120 120 F F F
79672024-02-03T17:13:00.012Z 29 580-599 73 73 73 122 122 122 F F F
79682024-02-03T17:13:00.012ZMax gen: 73, Max flush: 122
79692024-02-03T17:13:00.016Z
79702024-02-03T17:13:00.016ZBegin loop 37
79712024-02-03T17:13:00.016ZDownstairs to restart: 1
79722024-02-03T17:13:00.041Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.594643221Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428}
79732024-02-03T17:13:00.041Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.595120223Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428}
79742024-02-03T17:13:00.041Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.595204259Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428}
79752024-02-03T17:13:00.047Z{"msg":"Crucible 503e80a9-7367-47d7-a7d7-e1656c08bf2f has session id: a8565a69-806c-4878-9ab8-64b28b1d8bfd","v":0,"name":"crucible","level":30Crucible runtime is spawned
79762024-02-03T17:13:00.047Z,"time":"2024-02-03T17:12:59.595246027Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
79772024-02-03T17:13:00.047Z{"msg":"Upstairs opts: Upstairs UUID: 503e80a9-7367-47d7-a7d7-e1656c08bf2f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.595444351Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
79782024-02-03T17:13:00.047Z{"msg":"Crucible stats registered with UUID: 503e80a9-7367-47d7-a7d7-e1656c08bf2f","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.595519951Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
79792024-02-03T17:13:00.047Z{"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.595562766Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428}
79802024-02-03T17:13:00.047Z{"msg":"503e80a9-7367-47d7-a7d7-e1656c08bf2f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.595629168Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
79812024-02-03T17:13:00.047Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.595670201Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"client":"0","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
79822024-02-03T17:13:00.047Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.595711198Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"client":"0","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
79832024-02-03T17:13:00.047Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.59581781Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"client":"1","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
79842024-02-03T17:13:00.047Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.595864342Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"client":"1","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
79852024-02-03T17:13:00.047Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.59596886Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"client":"2","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
79862024-02-03T17:13:00.048Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.596013143Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"client":"2","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
79872024-02-03T17:13:00.048Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.596099133Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"":"io task","client":"2","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
79882024-02-03T17:13:00.048Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.596147468Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"":"io task","client":"1","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
79892024-02-03T17:13:00.048Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.596191543Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"":"io task","client":"1","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
79902024-02-03T17:13:00.048Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.596261385Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"":"io task","client":"2","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
79912024-02-03T17:13:00.048Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.596296939Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"":"io task","client":"0","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
79922024-02-03T17:13:00.048Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.596372544Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"":"io task","client":"0","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
79932024-02-03T17:13:00.048Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.597431316Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"client":"0","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
79942024-02-03T17:13:00.048Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.597552699Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"client":"0","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
79952024-02-03T17:13:00.048Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.597610324Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"client":"2","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
79962024-02-03T17:13:00.048Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.597734477Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"client":"2","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
79972024-02-03T17:13:00.048Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.598417653Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"client":"1","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
79982024-02-03T17:13:00.048Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.598609893Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"client":"1","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
79992024-02-03T17:13:00.048Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.598684775Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"client":"0","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80002024-02-03T17:13:00.048Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.598715015Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"client":"2","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80012024-02-03T17:13:00.053Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.599533736Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"client":"0","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80022024-02-03T17:13:00.054Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.599641864Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"client":"0","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80032024-02-03T17:13:00.054Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.599717003Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80042024-02-03T17:13:00.054Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.599742655Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"client":"1","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80052024-02-03T17:13:00.054Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.599871171Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"client":"2","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80062024-02-03T17:13:00.054Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.599931066Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"client":"2","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80072024-02-03T17:13:00.054Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.599957576Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80082024-02-03T17:13:00.054Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.600526327Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"client":"1","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80092024-02-03T17:13:00.054Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.600586201Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"client":"1","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80102024-02-03T17:13:00.054Z{"msg":"[0]R flush_numbers[0..12]: [115, 112, 120, 120, 120, 116, 120, 120, 117, 122, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.60061432Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"":"downstairs","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80112024-02-03T17:13:00.054Z{"msg":"[0]R generation[0..12]: [66, 66, 70, 70, 70, 67, 70, 70, 68, 73, 70, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.600637809Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"":"downstairs","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80122024-02-03T17:13:00.054Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.600805748Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"":"downstairs","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80132024-02-03T17:13:00.054Z{"msg":"[1]R flush_numbers[0..12]: [115, 112, 120, 120, 120, 116, 120, 120, 117, 122, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.600868182Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"":"downstairs","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80142024-02-03T17:13:00.054Z{"msg":"[1]R generation[0..12]: [66, 66, 70, 70, 70, 67, 70, 70, 68, 73, 70, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.600975857Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"":"downstairs","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80152024-02-03T17:13:00.054Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
80162024-02-03T17:13:00.054Z,"time":"2024-02-03T17:12:59.60102698Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"":"downstairs","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80172024-02-03T17:13:00.054Z{"msg":"[2]R flush_numbers[0..12]: [115, 112, 120, 120, 120, 116, 120, 120, 117, 122, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.601054664Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"":"downstairs","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80182024-02-03T17:13:00.054Z{"msg":"[2]R generation[0..12]: [66, 66, 70, 70, 70, 67, 70, 70, 68, 73, 70, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.601077687Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"":"downstairs","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80192024-02-03T17:13:00.054Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.601101328Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"":"downstairs","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80202024-02-03T17:13:00.054Z{"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.601151153Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"":"downstairs","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80212024-02-03T17:13:00.054Z{"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.601175831Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"":"downstairs","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80222024-02-03T17:13:00.054Z{"msg":"Next flush: 123","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.601197079Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"":"downstairs","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80232024-02-03T17:13:00.054Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.601218828Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"":"downstairs","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80242024-02-03T17:13:00.055Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.60124087Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80252024-02-03T17:13:00.055Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.601261078Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"client":"0","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80262024-02-03T17:13:00.055Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.601282939Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"client":"1","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80272024-02-03T17:13:00.055Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.601304485Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"client":"2","":"downstairs","session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80282024-02-03T17:13:00.055Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.601326095Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80292024-02-03T17:13:00.055Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.601371358Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80302024-02-03T17:13:00.055Z{"msg":"503e80a9-7367-47d7-a7d7-e1656c08bf2f is now active with session: a8565a69-806c-4878-9ab8-64b28b1d8bfd","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.601395664Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80312024-02-03T17:13:00.055Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.601415501Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428,"session_id":"a8565a69-806c-4878-9ab8-64b28b1d8bfd"}
80322024-02-03T17:13:00.055Z{"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:12:59.601460494Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428}
80332024-02-03T17:13:00.055ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
80342024-02-03T17:13:00.055ZLoading write count information from file "/tmp/test_repair_verify.data"
80352024-02-03T17:13:00.055ZRun Repair workload
80362024-02-03T17:13:00.055Z01/30 Flush
80372024-02-03T17:13:00.055Z02/30 Write block 107 len 8192 data: 7 4
80382024-02-03T17:13:00.055Z03/30 Write block 169 len 28672 data: 7 8 7 5 7 7 5
80392024-02-03T17:13:00.059Z04/30 Read block 179 len 24576
80402024-02-03T17:13:00.059Z05/30 Read block 56 len 40960
80412024-02-03T17:13:00.059Z06/30 Flush
80422024-02-03T17:13:00.065Z07/30 Write block 297 len 24576 data: 7 6 7 4 6 6
80432024-02-03T17:13:00.065Z08/30 Read block 268 len 12288
80442024-02-03T17:13:00.065Z09/30 Read block 130 len 8192
80452024-02-03T17:13:00.065Z10/30 Write block 282 len 20480 data: 12 12 9 8 7
80462024-02-03T17:13:00.068Z11/30 Write block 522 len 16384 data: 5 6 5 7
80472024-02-03T17:13:00.069Z12/30 Write block 447 len 32768 data: 10 10 7 5 4 4 4 4
80482024-02-03T17:13:00.069Z13/30 Write block 503 len 28672 data: 6 8 6 7 7 7 8
80492024-02-03T17:13:00.072Z14/30 Read block 246 len 32768
80502024-02-03T17:13:00.072Z15/30 Read block 68 len 20480
80512024-02-03T17:13:00.075Z16/30 Write block 179 len 36864 data: 5 5 5 5 8 9 12 9 11
80522024-02-03T17:13:00.075Z17/30 Flush
80532024-02-03T17:13:00.079Z18/30 Write block 96 len 12288 data: 8 8 7
80542024-02-03T17:13:00.083Z19/30 Write block 389 len 32768 data: 3 7 8 10 11 10 9 8
80552024-02-03T17:13:00.083Z20/30 Read block 419 len 12288
80562024-02-03T17:13:00.086Z21/30 Read block 323 len 20480
80572024-02-03T17:13:00.086Z22/30 Write block 309 len 24576 data: 7 9 7 8 9 8
80582024-02-03T17:13:00.092Z23/30 Read block 17 len 32768
80592024-02-03T17:13:00.092Z24/30 Write block 75 len 8192 data: 7 8
80602024-02-03T17:13:00.092Z25/30 Read block 81 len 32768
80612024-02-03T17:13:00.093Z26/30 Flush
80622024-02-03T17:13:00.096Z27/30 Write block 248 len 40960 data: 8 8 7 5 5 4 3 3 4 4
80632024-02-03T17:13:00.099Z28/30 Write block 23 len 40960 data: 10 10 9 9 9 9 7 6 7 7
80642024-02-03T17:13:00.099Z29/30 Write block 36 len 40960 data: 7 7 7 8 6 4 5 3 4 5
80652024-02-03T17:13:00.104Z30/30 Read block 483 len 28672
80662024-02-03T17:13:00.104Z----------------------------------------------------------------
80672024-02-03T17:13:00.104Z Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:29
80682024-02-03T17:13:00.104ZStates: Active Active Active
80692024-02-03T17:13:00.105ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
80702024-02-03T17:13:00.105Z 2 Acked 1001 Write 2 Done Done Done false
80712024-02-03T17:13:00.105Z 3 Acked 1002 Write 7 Done Sent Done false
80722024-02-03T17:13:00.105Z 4 Acked 1003 Read 6 Done Sent Done false
80732024-02-03T17:13:00.105Z 5 Acked 1004 Read 10 Done Sent Done false
80742024-02-03T17:13:00.105Z 6 Acked 1005 Flush 0 Done Sent Done false
80752024-02-03T17:13:00.105Z 7 Acked 1006 Write 6 Done Sent Done false
80762024-02-03T17:13:00.105Z 8 Acked 1007 Read 3 Done Sent Done false
80772024-02-03T17:13:00.105Z 9 Acked 1008 Read 2 Done Sent Done false
80782024-02-03T17:13:00.105Z 10 Acked 1009 Write 5 Done Sent Done false
80792024-02-03T17:13:00.105Z 11 Acked 1010 Write 4 Done Sent Done false
80802024-02-03T17:13:00.105Z 12 Acked 1011 Write 8 Done Sent Done false
80812024-02-03T17:13:00.105Z 13 Acked 1012 Write 7 Done Sent Done false
80822024-02-03T17:13:00.106Z 14 Acked 1013 Read 8 Done Sent Done false
80832024-02-03T17:13:00.106Z 15 Acked 1014 Read 5 Done Sent Done false
80842024-02-03T17:13:00.106Z 16 Acked 1015 Write 9 Done Sent Done false
80852024-02-03T17:13:00.106Z 17 Acked 1016 Flush 0 Done Sent Done false
80862024-02-03T17:13:00.106Z 18 Acked 1017 Write 3 Done Sent Done false
80872024-02-03T17:13:00.106Z 19 Acked 1018 Write 8 Done Sent Done false
80882024-02-03T17:13:00.106Z 20 Acked 1019 Read 3 Done Sent Done false
80892024-02-03T17:13:00.106Z 21 Acked 1020 Read 5 Done Sent Done false
80902024-02-03T17:13:00.106Z 22 Acked 1021 Write 6 Done Sent Done false
80912024-02-03T17:13:00.107Z 23 Acked 1022 Read 8 Done Sent Done false
80922024-02-03T17:13:00.107Z 24 Acked 1023 Write 2 Done Sent Done false
80932024-02-03T17:13:00.107Z 25 Acked 1024 Read 8 Done Sent Done false
80942024-02-03T17:13:00.107Z 26 Acked 1025 Flush 0 Done Sent Done false
80952024-02-03T17:13:00.107Z 27 Acked 1026 Write 10 Done Sent Done false
80962024-02-03T17:13:00.107Z 28 Acked 1027 Write 10 Done Sent Done false
80972024-02-03T17:13:00.107Z 29 Acked 1028 Write 10 Done Sent Done false
80982024-02-03T17:13:00.107Z 30 Acked 1029 Read 7 Done Sent Sent false
80992024-02-03T17:13:00.107Z STATES DS:0 DS:1 DS:2 TOTAL
81002024-02-03T17:13:00.107Z New 0 0 0 0
81012024-02-03T17:13:00.107Z Sent 0 28 1 29
81022024-02-03T17:13:00.108Z Done 29 1 28 58
81032024-02-03T17:13:00.108Z Skipped 0 0 0 0
81042024-02-03T17:13:00.108Z Error 0 0 0 0
81052024-02-03T17:13:00.108ZLast Flush: 1025 1000 1025
81062024-02-03T17:13:00.108ZDownstairs last five completed: 1000
81072024-02-03T17:13:00.108ZUpstairs last five completed: 30 29 28 27 26
81082024-02-03T17:13:00.108Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:12:59.658120467Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1428}
81092024-02-03T17:13:00.108ZWrote out file "/tmp/test_repair_verify.data"
81102024-02-03T17:13:00.112Z
81112024-02-03T17:13:00.117ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
81122024-02-03T17:13:00.144ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
81132024-02-03T17:13:00.144Z 0 000-019 66 66 66 115 115 115 F F F
81142024-02-03T17:13:00.148Z 1 020-039 66 66 66 112 112 112 T F T <---
81152024-02-03T17:13:00.148Z 2 040-059 70 70 70 120 120 120 T F T <---
81162024-02-03T17:13:00.148Z 3 060-079 74 70 74 126 120 126 F F F <---
81172024-02-03T17:13:00.148Z 4 080-099 74 70 74 126 120 126 F F F <---
81182024-02-03T17:13:00.148Z 5 100-119 74 67 74 124 116 124 F T F <---
81192024-02-03T17:13:00.149Z 6 120-139 70 70 70 120 120 120 F F F
81202024-02-03T17:13:00.149Z 7 140-159 70 70 70 120 120 120 F F F
81212024-02-03T17:13:00.149Z 8 160-179 74 68 74 125 117 125 F F F <---
81222024-02-03T17:13:00.149Z 9 180-199 74 73 74 125 122 125 F F F <---
81232024-02-03T17:13:00.149Z 10 200-219 70 70 70 120 120 120 F F F
81242024-02-03T17:13:00.149Z 11 220-239 70 70 70 120 120 120 F F F
81252024-02-03T17:13:00.149Z 12 240-259 70 70 70 120 120 120 T F T <---
81262024-02-03T17:13:00.149Z 13 260-279 73 73 73 122 122 122 F F F
81272024-02-03T17:13:00.149Z 14 280-299 74 73 74 125 122 125 F F F <---
81282024-02-03T17:13:00.149Z 15 300-319 74 68 74 126 117 126 F F F <---
81292024-02-03T17:13:00.149Z 16 320-339 68 68 68 117 117 117 F F F
81302024-02-03T17:13:00.149Z 17 340-359 66 66 66 115 115 115 F F F
81312024-02-03T17:13:00.149Z 18 360-379 62 62 62 106 106 106 F F F
81322024-02-03T17:13:00.150Z 19 380-399 74 73 74 126 122 126 F F F <---
81332024-02-03T17:13:00.150Z 20 400-419 73 73 73 122 122 122 F F F
81342024-02-03T17:13:00.150Z 21 420-439 70 70 70 120 120 120 F F F
81352024-02-03T17:13:00.150Z 22 440-459 74 73 74 125 122 125 F F F <---
81362024-02-03T17:13:00.150Z 23 460-479 73 73 73 122 122 122 F F F
81372024-02-03T17:13:00.150Z 24 480-499 73 73 73 122 122 122 F F F
81382024-02-03T17:13:00.150Z 25 500-519 74 69 74 125 119 125 F F F <---
81392024-02-03T17:13:00.150Z 26 520-539 74 68 74 125 117 125 F F F <---
81402024-02-03T17:13:00.150Z 27 540-559 73 73 73 122 122 122 F F F
81412024-02-03T17:13:00.150Z 28 560-579 70 70 70 120 120 120 F F F
81422024-02-03T17:13:00.150Z 29 580-599 73 73 73 122 122 122 F F F
81432024-02-03T17:13:00.150ZMax gen: 74, Max flush: 126
81442024-02-03T17:13:00.150ZError: Difference in extent metadata found!
81452024-02-03T17:13:00.151ZA Difference in extent metadata is expected here
81462024-02-03T17:13:00.151ZOn loop 37
81472024-02-03T17:13:00.151Z
81482024-02-03T17:13:00.154ZVerifying data now
81492024-02-03T17:13:05.351ZNo change after verify
81502024-02-03T17:13:05.352ZLoop: 37 Downstairs dump after verify (and repair):
81512024-02-03T17:13:05.374ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
81522024-02-03T17:13:05.374Z 0 000-019 66 66 66 115 115 115 F F F
81532024-02-03T17:13:05.374Z 1 020-039 75 75 75 127 127 127 F F F
81542024-02-03T17:13:05.375Z 2 040-059 75 75 75 127 127 127 F F F
81552024-02-03T17:13:05.375Z 3 060-079 74 74 74 126 126 126 F F F
81562024-02-03T17:13:05.378Z 4 080-099 74 74 74 126 126 126 F F F
81572024-02-03T17:13:05.378Z 5 100-119 74 74 74 124 124 124 F F F
81582024-02-03T17:13:05.378Z 6 120-139 70 70 70 120 120 120 F F F
81592024-02-03T17:13:05.378Z 7 140-159 70 70 70 120 120 120 F F F
81602024-02-03T17:13:05.378Z 8 160-179 74 74 74 125 125 125 F F F
81612024-02-03T17:13:05.378Z 9 180-199 74 74 74 125 125 125 F F F
81622024-02-03T17:13:05.378Z 10 200-219 70 70 70 120 120 120 F F F
81632024-02-03T17:13:05.378Z 11 220-239 70 70 70 120 120 120 F F F
81642024-02-03T17:13:05.379Z 12 240-259 75 75 75 127 127 127 F F F
81652024-02-03T17:13:05.379Z 13 260-279 73 73 73 122 122 122 F F F
81662024-02-03T17:13:05.379Z 14 280-299 74 74 74 125 125 125 F F F
81672024-02-03T17:13:05.379Z 15 300-319 74 74 74 126 126 126 F F F
81682024-02-03T17:13:05.379Z 16 320-339 68 68 68 117 117 117 F F F
81692024-02-03T17:13:05.379Z 17 340-359 66 66 66 115 115 115 F F F
81702024-02-03T17:13:05.379Z 18 360-379 62 62 62 106 106 106 F F F
81712024-02-03T17:13:05.379Z 19 380-399 74 74 74 126 126 126 F F F
81722024-02-03T17:13:05.379Z 20 400-419 73 73 73 122 122 122 F F F
81732024-02-03T17:13:05.379Z 21 420-439 70 70 70 120 120 120 F F F
81742024-02-03T17:13:05.379Z 22 440-459 74 74 74 125 125 125 F F F
81752024-02-03T17:13:05.379Z 23 460-479 73 73 73 122 122 122 F F F
81762024-02-03T17:13:05.379Z 24 480-499 73 73 73 122 122 122 F F F
81772024-02-03T17:13:05.379Z 25 500-519 74 74 74 125 125 125 F F F
81782024-02-03T17:13:05.379Z 26 520-539 74 74 74 125 125 125 F F F
81792024-02-03T17:13:05.379Z 27 540-559 73 73 73 122 122 122 F F F
81802024-02-03T17:13:05.379Z 28 560-579 70 70 70 120 120 120 F F F
81812024-02-03T17:13:05.379Z 29 580-599 73 73 73 122 122 122 F F F
81822024-02-03T17:13:05.379ZMax gen: 75, Max flush: 127
81832024-02-03T17:13:05.382Z
81842024-02-03T17:13:05.383ZBegin loop 38
81852024-02-03T17:13:05.383ZDownstairs to restart: 0
81862024-02-03T17:13:05.411Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.965208207Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436}
81872024-02-03T17:13:05.411Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.965682937Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436}
81882024-02-03T17:13:05.411Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.965711329Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436}
81892024-02-03T17:13:05.415Z{"msg":"Crucible db19996b-8078-43eb-baa0-04a01a802687 has session id: 3c69ee3c-adb3-4eca-a256-d30f3df9537c","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.965737515Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
81902024-02-03T17:13:05.415Z{"msg":"Upstairs opts: Upstairs UUID: db19996b-8078-43eb-baa0-04a01a802687, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
81912024-02-03T17:13:05.416Z,"time":"2024-02-03T17:13:04.965829539Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
81922024-02-03T17:13:05.416Z{"msg":"Crucible stats registered with UUID: db19996b-8078-43eb-baa0-04a01a802687","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.965878623Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
81932024-02-03T17:13:05.416Z{"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.965971389Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436}
81942024-02-03T17:13:05.416Z{"msg":"db19996b-8078-43eb-baa0-04a01a802687 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.96629591Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
81952024-02-03T17:13:05.416Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.96635713Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"client":"0","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
81962024-02-03T17:13:05.416Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.966383661Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"client":"0","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
81972024-02-03T17:13:05.416Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.966462116Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"client":"1","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
81982024-02-03T17:13:05.416Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.966508681Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"client":"1","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
81992024-02-03T17:13:05.416Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.966615805Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"client":"2","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82002024-02-03T17:13:05.416Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.966653934Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"client":"2","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82012024-02-03T17:13:05.416Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.966677533Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"":"io task","client":"0","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82022024-02-03T17:13:05.416Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.966716252Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"":"io task","client":"2","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82032024-02-03T17:13:05.416Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.966762148Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"":"io task","client":"1","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82042024-02-03T17:13:05.416Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.967165799Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"":"io task","client":"2","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82052024-02-03T17:13:05.416Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.967261219Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"":"io task","client":"0","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82062024-02-03T17:13:05.416Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.967305136Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"":"io task","client":"1","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82072024-02-03T17:13:05.416Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.968520866Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"client":"2","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82082024-02-03T17:13:05.416Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.968623391Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"client":"2","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82092024-02-03T17:13:05.416Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.96883797Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"client":"1","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82102024-02-03T17:13:05.416Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.968991484Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"client":"1","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82112024-02-03T17:13:05.416Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.969052212Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"client":"0","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82122024-02-03T17:13:05.416Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.96914347Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"client":"0","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82132024-02-03T17:13:05.416Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.96961095Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"client":"1","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82142024-02-03T17:13:05.421Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.970028728Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"client":"2","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82152024-02-03T17:13:05.421Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.970194441Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"client":"0","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82162024-02-03T17:13:05.422Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.97047684Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"client":"1","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82172024-02-03T17:13:05.422Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.970546973Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"client":"1","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82182024-02-03T17:13:05.422Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.970591388Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82192024-02-03T17:13:05.422Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.970874412Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"client":"2","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82202024-02-03T17:13:05.422Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.970965886Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"client":"2","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82212024-02-03T17:13:05.422Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.970992448Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82222024-02-03T17:13:05.422Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.971244701Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"client":"0","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82232024-02-03T17:13:05.422Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.971305863Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"client":"0","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82242024-02-03T17:13:05.422Z{"msg":"[0]R flush_numbers[0..12]: [115, 127, 127, 126, 126, 124, 120, 120, 125, 125, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.971333193Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"":"downstairs","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82252024-02-03T17:13:05.422Z{"msg":"[0]R generation[0..12]: [66, 75, 75, 74, 74, 74, 70, 70, 74, 74, 70, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.97135653Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"":"downstairs","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82262024-02-03T17:13:05.422Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.97140586Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"":"downstairs","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82272024-02-03T17:13:05.422Z{"msg":"[1]R flush_numbers[0..12]: [115, 127, 127, 126, 126, 124, 120, 120, 125, 125, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.971431698Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"":"downstairs","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82282024-02-03T17:13:05.422Z{"msg":"[1]R generation[0..12]: [66, 75, 75, 74, 74, 74, 70, 70, 74, 74, 70, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.971479806Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"":"downstairs","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82292024-02-03T17:13:05.422Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.971505908Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"":"downstairs","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82302024-02-03T17:13:05.422Z{"msg":Wait for a query_work_queue command to finish before sending IO
82312024-02-03T17:13:05.422Z"[2]R flush_numbers[0..12]: [115, 127, 127, 126, 126, 124, 120, 120, 125, 125, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.971580547Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"":"downstairs","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82322024-02-03T17:13:05.422Z{"msg":"[2]R generation[0..12]: [66, 75, 75, 74, 74, 74, 70, 70, 74, 74, 70, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.97166184Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"":"downstairs","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82332024-02-03T17:13:05.422Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.971856863Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"":"downstairs","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82342024-02-03T17:13:05.422Z{"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.971939572Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"":"downstairs","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82352024-02-03T17:13:05.422Z{"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.972043803Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"":"downstairs","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82362024-02-03T17:13:05.422Z{"msg":"Next flush: 128","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.97212658Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"":"downstairs","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82372024-02-03T17:13:05.422Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.972205508Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"":"downstairs","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82382024-02-03T17:13:05.422Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.972285583Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82392024-02-03T17:13:05.422Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.972361435Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"client":"0","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82402024-02-03T17:13:05.423Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.972439898Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"client":"1","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82412024-02-03T17:13:05.423Z{"msgRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
82422024-02-03T17:13:05.423Z":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.972524394Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"client":"2","":"downstairs","session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82432024-02-03T17:13:05.423Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.972590538Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82442024-02-03T17:13:05.423Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.97261444Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82452024-02-03T17:13:05.423Z{"msg":"db19996b-8078-43eb-baa0-04a01a802687 is now active with session: 3c69ee3c-adb3-4eca-a256-d30f3df9537c","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.972661227Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82462024-02-03T17:13:05.423Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.972682826Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436,"session_id":"3c69ee3c-adb3-4eca-a256-d30f3df9537c"}
82472024-02-03T17:13:05.423Z{"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:04.972727732Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436}
82482024-02-03T17:13:05.423ZLoading write count information from file "/tmp/test_repair_verify.data"
82492024-02-03T17:13:05.423ZRun Repair workload
82502024-02-03T17:13:05.423Z01/30 Write block 162 len 24576 data: 10 10 11 10 12 12
82512024-02-03T17:13:05.426Z02/30 Write block 278 len 24576 data: 10 12 11 11 13 13
82522024-02-03T17:13:05.426Z03/30 Read block 519 len 24576
82532024-02-03T17:13:05.427Z04/30 Write block 207 len 20480 data: 7 6 6 8 7
82542024-02-03T17:13:05.427Z05/30 Write block 160 len 40960 data: 9 8 11 11 12 11 13 13 8 8
82552024-02-03T17:13:05.430Z06/30 Read block 422 len 28672
82562024-02-03T17:13:05.430Z07/30 Read block 432 len 4096
82572024-02-03T17:13:05.433Z08/30 Read block 317 len 16384
82582024-02-03T17:13:05.433Z09/30 Read block 65 len 8192
82592024-02-03T17:13:05.433Z10/30 Flush
82602024-02-03T17:13:05.436Z11/30 Write block 550 len 12288 data: 8 7 7
82612024-02-03T17:13:05.439Z12/30 Write block 132 len 8192 data: 6 6
82622024-02-03T17:13:05.439Z13/30 Write block 185 len 20480 data: 13 10 12 14 14
82632024-02-03T17:13:05.442Z14/30 Write block 446 len 16384 data: 10 11 11 8
82642024-02-03T17:13:05.442Z15/30 Read block 372 len 8192
82652024-02-03T17:13:05.442Z16/30 Read block 183 len 20480
82662024-02-03T17:13:05.445Z17/30 Write block 181 len 4096 data: 6
82672024-02-03T17:13:05.445Z18/30 Write block 300 len 16384 data: 5 7 7 6
82682024-02-03T17:13:05.445Z19/30 Read block 503 len 36864
82692024-02-03T17:13:05.451Z20/30 Read block 214 len 40960
82702024-02-03T17:13:05.454Z21/30 Read block 62 len 40960
82712024-02-03T17:13:05.454Z22/30 Write block 342 len 32768 data: 5 4 5 5 4 5 8 8
82722024-02-03T17:13:05.454Z23/30 Flush
82732024-02-03T17:13:05.457Z24/30 Write block 268 len 40960 data: 13 11 11 10 10 9 8 9 9 9
82742024-02-03T17:13:05.460Z25/30 Write block 141 len 12288 data: 6 6 6
82752024-02-03T17:13:05.460Z26/30 Write block 53 len 40960 data: 5 6 6 6 5 6 6 5 3 4
82762024-02-03T17:13:05.460Z27/30 Read block 550 len 20480
82772024-02-03T17:13:05.463Z28/30 Write block 410 len 4096 data: 5
82782024-02-03T17:13:05.463Z29/30 Read block 427 len 4096
82792024-02-03T17:13:05.467Z30/30 Read block 595 len 16384
82802024-02-03T17:13:05.467Z----------------------------------------------------------------
82812024-02-03T17:13:05.467Z Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:30
82822024-02-03T17:13:05.467ZStates: Active Active Active
82832024-02-03T17:13:05.467ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
82842024-02-03T17:13:05.467Z 1 Acked 1000 Write 6 Sent Done Done false
82852024-02-03T17:13:05.467Z 2 Acked 1001 Write 6 Sent Done Done false
82862024-02-03T17:13:05.467Z 3 Acked 1002 Read 6 Sent Done Done false
82872024-02-03T17:13:05.467Z 4 Acked 1003 Write 5 Sent Done Done false
82882024-02-03T17:13:05.467Z 5 Acked 1004 Write 10 Sent Done Done false
82892024-02-03T17:13:05.467Z 6 Acked 1005 Read 7 Sent Done Done false
82902024-02-03T17:13:05.467Z 7 Acked 1006 Read 1 Sent Done Done false
82912024-02-03T17:13:05.467Z 8 Acked 1007 Read 4 Sent Done Done false
82922024-02-03T17:13:05.467Z 9 Acked 1008 Read 2 Sent Done Done false
82932024-02-03T17:13:05.467Z 10 Acked 1009 Flush 0 Sent Done Done false
82942024-02-03T17:13:05.468Z 11 Acked 1010 Write 3 Sent Done Done false
82952024-02-03T17:13:05.468Z 12 Acked 1011 Write 2 Sent Done Done false
82962024-02-03T17:13:05.468Z 13 Acked 1012 Write 5 Sent Done Done false
82972024-02-03T17:13:05.468Z 14 Acked 1013 Write 4 Sent Done Done false
82982024-02-03T17:13:05.468Z 15 Acked 1014 Read 2 Sent Done Done false
82992024-02-03T17:13:05.468Z 16 Acked 1015 Read 5 Sent Done Done false
83002024-02-03T17:13:05.468Z 17 Acked 1016 Write 1 Sent Done Done false
83012024-02-03T17:13:05.468Z 18 Acked 1017 Write 4 Sent Done Done false
83022024-02-03T17:13:05.468Z 19 Acked 1018 Read 9 Sent Done Done false
83032024-02-03T17:13:05.468Z 20 Acked 1019 Read 10 Sent Done Done false
83042024-02-03T17:13:05.468Z 21 Acked 1020 Read 10 Sent Done Done false
83052024-02-03T17:13:05.468Z 22 Acked 1021 Write 8 Sent Done Done false
83062024-02-03T17:13:05.468Z 23 Acked 1022 Flush 0 Sent Done Done false
83072024-02-03T17:13:05.468Z 24 Acked 1023 Write 10 Sent Done Done false
83082024-02-03T17:13:05.468Z 25 Acked 1024 Write 3 Sent Done Done false
83092024-02-03T17:13:05.468Z 26 Acked 1025 Write 10 Sent Done Done false
83102024-02-03T17:13:05.468Z 27 Acked 1026 Read 5 Sent Done Done false
83112024-02-03T17:13:05.468Z 28 Acked 1027 Write 1 Sent Done Done false
83122024-02-03T17:13:05.468Z 29 Acked 1028 Read 1 Sent Done Done false
83132024-02-03T17:13:05.468Z 30 Acked 1029 Read 4 Sent Sent Done false
83142024-02-03T17:13:05.468Z STATES DS:0 DS:1 DS:2 TOTAL
83152024-02-03T17:13:05.468Z New 0 0 0 0
83162024-02-03T17:13:05.468Z Sent 30 1 0 31
83172024-02-03T17:13:05.468Z Done 0 29 30 59
83182024-02-03T17:13:05.469Z Skipped 0 0 0 0
83192024-02-03T17:13:05.469Z Error 0 0 0 0
83202024-02-03T17:13:05.469ZLast Flush: 0 1022 1022
83212024-02-03T17:13:05.469ZDownstairs last five completed:
83222024-02-03T17:13:05.469ZUpstairs last five completed: 30 29 28 27 26
83232024-02-03T17:13:05.469Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:05.021137119Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1436}
83242024-02-03T17:13:05.469ZWrote out file "/tmp/test_repair_verify.data"
83252024-02-03T17:13:05.473Z
83262024-02-03T17:13:05.480ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
83272024-02-03T17:13:05.506ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83282024-02-03T17:13:05.506Z 0 000-019 66 66 66 115 115 115 F F F
83292024-02-03T17:13:05.506Z 1 020-039 75 75 75 127 127 127 F F F
83302024-02-03T17:13:05.506Z 2 040-059 75 75 75 127 127 127 F T T <---
83312024-02-03T17:13:05.506Z 3 060-079 74 74 74 126 126 126 F T T <---
83322024-02-03T17:13:05.509Z 4 080-099 74 74 74 126 126 126 F F F
83332024-02-03T17:13:05.509Z 5 100-119 74 74 74 124 124 124 F F F
83342024-02-03T17:13:05.509Z 6 120-139 70 76 76 120 129 129 F F F <---
83352024-02-03T17:13:05.509Z 7 140-159 70 70 70 120 120 120 F T T <---
83362024-02-03T17:13:05.509Z 8 160-179 74 76 76 125 128 128 F F F <---
83372024-02-03T17:13:05.509Z 9 180-199 74 76 76 125 129 129 F F F <---
83382024-02-03T17:13:05.509Z 10 200-219 70 76 76 120 128 128 F F F <---
83392024-02-03T17:13:05.509Z 11 220-239 70 70 70 120 120 120 F F F
83402024-02-03T17:13:05.510Z 12 240-259 75 75 75 127 127 127 F F F
83412024-02-03T17:13:05.510Z 13 260-279 73 76 76 122 128 128 F T T <---
83422024-02-03T17:13:05.510Z 14 280-299 74 76 76 125 128 128 F F F <---
83432024-02-03T17:13:05.510Z 15 300-319 74 76 76 126 129 129 F F F <---
83442024-02-03T17:13:05.510Z 16 320-339 68 68 68 117 117 117 F F F
83452024-02-03T17:13:05.510Z 17 340-359 66 76 76 115 129 129 F F F <---
83462024-02-03T17:13:05.510Z 18 360-379 62 62 62 106 106 106 F F F
83472024-02-03T17:13:05.510Z 19 380-399 74 74 74 126 126 126 F F F
83482024-02-03T17:13:05.510Z 20 400-419 73 73 73 122 122 122 F T T <---
83492024-02-03T17:13:05.510Z 21 420-439 70 70 70 120 120 120 F F F
83502024-02-03T17:13:05.510Z 22 440-459 74 76 76 125 129 129 F F F <---
83512024-02-03T17:13:05.510Z 23 460-479 73 73 73 122 122 122 F F F
83522024-02-03T17:13:05.510Z 24 480-499 73 73 73 122 122 122 F F F
83532024-02-03T17:13:05.510Z 25 500-519 74 74 74 125 125 125 F F F
83542024-02-03T17:13:05.510Z 26 520-539 74 74 74 125 125 125 F F F
83552024-02-03T17:13:05.510Z 27 540-559 73 76 76 122 129 129 F F F <---
83562024-02-03T17:13:05.510Z 28 560-579 70 70 70 120 120 120 F F F
83572024-02-03T17:13:05.510Z 29 580-599 73 73 73 122 122 122 F F F
83582024-02-03T17:13:05.510ZMax gen: 76, Max flush: 129
83592024-02-03T17:13:05.510ZError: Difference in extent metadata found!
83602024-02-03T17:13:05.513ZA Difference in extent metadata is expected here
83612024-02-03T17:13:05.513ZOn loop 38
83622024-02-03T17:13:05.513Z
83632024-02-03T17:13:05.514ZVerifying data now
83642024-02-03T17:13:10.725ZNo change after verify
83652024-02-03T17:13:10.725ZLoop: 38 Downstairs dump after verify (and repair):
83662024-02-03T17:13:11.154ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83672024-02-03T17:13:11.154Z 0 000-019 66 66 66 115 115 115 F F F
83682024-02-03T17:13:11.154Z 1 020-039 75 75 75 127 127 127 F F F
83692024-02-03T17:13:11.154Z 2 040-059 77 77 77 130 130 130 F F F
83702024-02-03T17:13:11.154Z 3 060-079 77 77 77 130 130 130 F F F
83712024-02-03T17:13:11.161Z 4 080-099 74 74 74 126 126 126 F F F
83722024-02-03T17:13:11.161Z 5 100-119 74 74 74 124 124 124 F F F
83732024-02-03T17:13:11.161Z 6 120-139 76 76 76 129 129 129 F F F
83742024-02-03T17:13:11.161Z 7 140-159 77 77 77 130 130 130 F F F
83752024-02-03T17:13:11.161Z 8 160-179 76 76 76 128 128 128 F F F
83762024-02-03T17:13:11.162Z 9 180-199 76 76 76 129 129 129 F F F
83772024-02-03T17:13:11.162Z 10 200-219 76 76 76 128 128 128 F F F
83782024-02-03T17:13:11.162Z 11 220-239 70 70 70 120 120 120 F F F
83792024-02-03T17:13:11.162Z 12 240-259 75 75 75 127 127 127 F F F
83802024-02-03T17:13:11.162Z 13 260-279 77 77 77 130 130 130 F F F
83812024-02-03T17:13:11.162Z 14 280-299 76 76 76 128 128 128 F F F
83822024-02-03T17:13:11.162Z 15 300-319 76 76 76 129 129 129 F F F
83832024-02-03T17:13:11.162Z 16 320-339 68 68 68 117 117 117 F F F
83842024-02-03T17:13:11.162Z 17 340-359 76 76 76 129 129 129 F F F
83852024-02-03T17:13:11.162Z 18 360-379 62 62 62 106 106 106 F F F
83862024-02-03T17:13:11.162Z 19 380-399 74 74 74 126 126 126 F F F
83872024-02-03T17:13:11.162Z 20 400-419 77 77 77 130 130 130 F F F
83882024-02-03T17:13:11.162Z 21 420-439 70 70 70 120 120 120 F F F
83892024-02-03T17:13:11.162Z 22 440-459 76 76 76 129 129 129 F F F
83902024-02-03T17:13:11.162Z 23 460-479 73 73 73 122 122 122 F F F
83912024-02-03T17:13:11.162Z 24 480-499 73 73 73 122 122 122 F F F
83922024-02-03T17:13:11.162Z 25 500-519 74 74 74 125 125 125 F F F
83932024-02-03T17:13:11.162Z 26 520-539 74 74 74 125 125 125 F F F
83942024-02-03T17:13:11.162Z 27 540-559 76 76 76 129 129 129 F F F
83952024-02-03T17:13:11.162Z 28 560-579 70 70 70 120 120 120 F F F
83962024-02-03T17:13:11.162Z 29 580-599 73 73 73 122 122 122 F F F
83972024-02-03T17:13:11.163ZMax gen: 77, Max flush: 130
83982024-02-03T17:13:11.163Z
83992024-02-03T17:13:11.163ZBegin loop 39
84002024-02-03T17:13:11.163ZDownstairs to restart: 2
84012024-02-03T17:13:11.163Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.331894088Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444}
84022024-02-03T17:13:11.163Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.332429385Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444}
84032024-02-03T17:13:11.163Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.332476887Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444}
84042024-02-03T17:13:11.163Z{"msg":"Crucible eadce014-0fa9-4f94-98f6-46a62f38f120 has session id: 49d683eb-0699-45da-ac82-63941f2334f8","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.332515211Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84052024-02-03T17:13:11.163Z{"Crucible runtime is spawned
84062024-02-03T17:13:11.163Zmsg":"Upstairs opts: Upstairs UUID: eadce014-0fa9-4f94-98f6-46a62f38f120, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.332620396Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84072024-02-03T17:13:11.163Z{"msg":"Crucible stats registered with UUID: eadce014-0fa9-4f94-98f6-46a62f38f120","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.33268041Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84082024-02-03T17:13:11.163Z{"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.332704923Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444}
84092024-02-03T17:13:11.163Z{"msg":"eadce014-0fa9-4f94-98f6-46a62f38f120 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.332890672Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84102024-02-03T17:13:11.163Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.332946514Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"client":"0","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84112024-02-03T17:13:11.163Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.33297246Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"client":"0","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84122024-02-03T17:13:11.163Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.333016484Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"client":"1","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84132024-02-03T17:13:11.163Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.333106024Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"client":"1","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84142024-02-03T17:13:11.163Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.333184448Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"client":"2","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84152024-02-03T17:13:11.163Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.333319969Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"client":"2","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84162024-02-03T17:13:11.163Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.333389915Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"":"io task","client":"2","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84172024-02-03T17:13:11.163Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.333415992Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"":"io task","client":"0","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84182024-02-03T17:13:11.163Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.333456323Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"":"io task","client":"1","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84192024-02-03T17:13:11.163Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.333748097Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"":"io task","client":"0","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84202024-02-03T17:13:11.164Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.334035095Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"":"io task","client":"2","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84212024-02-03T17:13:11.164Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.334086655Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"":"io task","client":"1","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84222024-02-03T17:13:11.164Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.335307209Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"client":"1","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84232024-02-03T17:13:11.164Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.335381493Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"client":"1","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84242024-02-03T17:13:11.164Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.33540847Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"client":"0","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84252024-02-03T17:13:11.164Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.335430976Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"client":"0","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84262024-02-03T17:13:11.164Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.335508529Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"client":"2","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84272024-02-03T17:13:11.164Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.335556001Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"client":"2","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84282024-02-03T17:13:11.164Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.336287843Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"client":"0","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84292024-02-03T17:13:11.164Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.336582587Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"client":"1","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84302024-02-03T17:13:11.164Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.336860307Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"client":"2","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84312024-02-03T17:13:11.164Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.337488231Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"client":"1","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84322024-02-03T17:13:11.164Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.337559861Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"client":"1","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84332024-02-03T17:13:11.164Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.33758656Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84342024-02-03T17:13:11.164Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.337686153Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"client":"2","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84352024-02-03T17:13:11.164Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.33779154Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"client":"2","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84362024-02-03T17:13:11.164Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.337820225Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84372024-02-03T17:13:11.164Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.337966706Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"client":"0","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84382024-02-03T17:13:11.164Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.338028534Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"client":"0","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84392024-02-03T17:13:11.164Z{"msg":"[0]R flush_numbers[0..12]: [115, 127, 130, 130, 126, 124, 129, 130, 128, 129, 128, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.33805524Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"":"downstairs","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84402024-02-03T17:13:11.164Z{"msg":"[0]R generation[0..12]: [66, 75, 77, 77, 74, 74, 76, 77, 76, 76, 76, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.338105977Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"":"downstairs","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84412024-02-03T17:13:11.164Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.338132396Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"":"downstairs","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84422024-02-03T17:13:11.164Z{"msg":"[1]R flush_numbers[0..12]: [115, 127, 130, 130, 126, 124, 129, 130, 128, 129, 128, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.338189162Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"":"downstairs","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84432024-02-03T17:13:11.165Z{"msg":"[1]R generation[0..12]: [66, 75, 77, 77, 74, 74, 76, 77, 76, 76, 76, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.338214004Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"":"downstairs","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84442024-02-03T17:13:11.165Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.338264588Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"":"downstairs","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84452024-02-03T17:13:11.165Z{"msg":"[2]R flush_numbers[0..12]: [115, 127, 130, 130, 126, 124, 129, 130, 128, 129, 128, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.338289457Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"":"downstairs","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84462024-02-03T17:13:11.165ZWait for a query_work_queue command to finish before sending IO
84472024-02-03T17:13:11.165Z{"msg":"[2]R generation[0..12]: [66, 75, 77, 77, 74, 74, 76, 77, 76, 76, 76, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.338344507Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"":"downstairs","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84482024-02-03T17:13:11.165Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.338370055Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"":"downstairs","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84492024-02-03T17:13:11.165Z{"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.338418789Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"":"downstairs","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84502024-02-03T17:13:11.165Z{"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.338442836Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"":"downstairs","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84512024-02-03T17:13:11.165Z{"msg":"Next flush: 131","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.338492638Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"":"downstairs","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84522024-02-03T17:13:11.165Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.3385159Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"":"downstairs","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84532024-02-03T17:13:11.165Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.338538246Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84542024-02-03T17:13:11.165Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.338595714Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"client":"0","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84552024-02-03T17:13:11.165Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.338633191Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"client":"1","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84562024-02-03T17:13:11.165Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.338657923Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"client":"2","":"downstairs","session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84572024-02-03T17:13:11.165Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.338680004Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84582024-02-03T17:13:11.165Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.338730624Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84592024-02-03T17:13:11.165Z{"msg":"eadce014-0fa9-4f94-98f6-46a62f38f120 is now active with session: 49d683eb-0699-45da-ac82-63941f2334f8","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.33875453Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84602024-02-03T17:13:11.165Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.338774607Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
84612024-02-03T17:13:11.165Z{"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:10.338823692Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444}
84622024-02-03T17:13:11.165ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
84632024-02-03T17:13:11.165ZLoading write count information from file "/tmp/test_repair_verify.data"
84642024-02-03T17:13:11.165ZRun Repair workload
84652024-02-03T17:13:11.165Z01/30 Write block 21 len 4096 data: 8
84662024-02-03T17:13:11.165Z02/30 Read block 484 len 36864
84672024-02-03T17:13:11.165Z03/30 Write block 62 len 4096 data: 5
84682024-02-03T17:13:11.166Z04/30 Write block 160 len 16384 data: 10 9 12 12
84692024-02-03T17:13:11.166Z05/30 Read block 505 len 36864
84702024-02-03T17:13:11.166Z06/30 Read block 509 len 20480
84712024-02-03T17:13:11.173Z07/30 Read block 402 len 36864
84722024-02-03T17:13:11.173Z08/30 Write block 184 len 28672 data: 10 14 11 13 15 15 14
84732024-02-03T17:13:11.173Z09/30 Read block 125 len 28672
84742024-02-03T17:13:11.173Z10/30 Read block 309 len 20480
84752024-02-03T17:13:11.173Z11/30 Flush
84762024-02-03T17:13:11.173Z12/30 Flush
84772024-02-03T17:13:11.173Z13/30 Write block 90 len 28672 data: 8 7 7 9 9 7 9
84782024-02-03T17:13:11.173Z14/30 Read block 487 len 28672
84792024-02-03T17:13:11.173Z15/30 Write block 261 len 16384 data: 7 7 9 10
84802024-02-03T17:13:11.173Z16/30 Write block 415 len 12288 data: 9 8 10
84812024-02-03T17:13:11.174Z17/30 Read block 399 len 40960
84822024-02-03T17:13:11.174Z18/30 Read block 414 len 20480
84832024-02-03T17:13:11.174Z19/30 Flush
84842024-02-03T17:13:11.174Z20/30 Write block 117 len 32768 data: 8 8 7 6 6 7 6 5
84852024-02-03T17:13:11.174Z21/30 Read block 548 len 12288
84862024-02-03T17:13:11.174Z22/30 Read block 206 len 36864
84872024-02-03T17:13:11.174Z23/30 Read block 462 len 20480
84882024-02-03T17:13:11.174Z24/30 Read block 465 len 12288
84892024-02-03T17:13:11.174Z25/30 Read block 192 len 24576
84902024-02-03T17:13:11.174Z26/30 Flush
84912024-02-03T17:13:11.174Z27/30 Write block 439 len 32768 data: 5 6 8 11 9 8 8 11
84922024-02-03T17:13:11.174Z28/30 Read block 554 len 8192
84932024-02-03T17:13:11.175Z29/30 Write block 386 len 32768 data: 5 6 5 4 8 9 11 12
84942024-02-03T17:13:11.175Z30/30 Write block 555 len 24576 data: 5 4 4 3 4 4
84952024-02-03T17:13:11.175Z----------------------------------------------------------------
84962024-02-03T17:13:11.175Z Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30
84972024-02-03T17:13:11.175ZStates: Active Active Active
84982024-02-03T17:13:11.175ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
84992024-02-03T17:13:11.175Z 1 Acked 1000 Write 1 Done Done Done false
85002024-02-03T17:13:11.175Z 2 Acked 1001 Read 9 Done Done Done false
85012024-02-03T17:13:11.175Z 3 Acked 1002 Write 1 Done Done Done false
85022024-02-03T17:13:11.175Z 4 Acked 1003 Write 4 Done Done Done false
85032024-02-03T17:13:11.175Z 5 Acked 1004 Read 9 Done Done Done false
85042024-02-03T17:13:11.175Z 6 Acked 1005 Read 5 Done Done Sent false
85052024-02-03T17:13:11.175Z 7 Acked 1006 Read 9 Done Done Sent false
85062024-02-03T17:13:11.176Z 8 Acked 1007 Write 7 Done Done Sent false
85072024-02-03T17:13:11.176Z 9 Acked 1008 Read 7 Done Done Sent false
85082024-02-03T17:13:11.176Z 10 Acked 1009 Read 5 Done Done Sent false
85092024-02-03T17:13:11.176Z 11 Acked 1010 Flush 0 Done Done Sent false
85102024-02-03T17:13:11.176Z 12 Acked 1011 Flush 0 Done Done Sent false
85112024-02-03T17:13:11.176Z 13 Acked 1012 Write 7 Done Done Sent false
85122024-02-03T17:13:11.176Z 14 Acked 1013 Read 7 Done Done Sent false
85132024-02-03T17:13:11.176Z 15 Acked 1014 Write 4 Done Done Sent false
85142024-02-03T17:13:11.176Z 16 Acked 1015 Write 3 Done Done Sent false
85152024-02-03T17:13:11.176Z 17 Acked 1016 Read 10 Done Done Sent false
85162024-02-03T17:13:11.176Z 18 Acked 1017 Read 5 Done Done Sent false
85172024-02-03T17:13:11.176Z 19 Acked 1018 Flush 0 Done Done Sent false
85182024-02-03T17:13:11.176Z 20 Acked 1019 Write 8 Done Done Sent false
85192024-02-03T17:13:11.177Z 21 Acked 1020 Read 3 Done Done Sent false
85202024-02-03T17:13:11.177Z 22 Acked 1021 Read 9 Done Done Sent false
85212024-02-03T17:13:11.177Z 23 Acked 1022 Read 5 Done Done Sent false
85222024-02-03T17:13:11.177Z 24 Acked 1023 Read 3 Done Done Sent false
85232024-02-03T17:13:11.177Z 25 Acked 1024 Read 6 Done Done Sent false
85242024-02-03T17:13:11.177Z 26 Acked 1025 Flush 0 Done Done Sent false
85252024-02-03T17:13:11.177Z 27 Acked 1026 Write 8 Done Done Sent false
85262024-02-03T17:13:11.177Z 28 Acked 1027 Read 2 Done Done Sent false
85272024-02-03T17:13:11.177Z 29 Acked 1028 Write 8 Done Done Sent false
85282024-02-03T17:13:11.177Z 30 Acked 1029 Write 6 Sent Sent Sent false
85292024-02-03T17:13:11.177Z STATES DS:0 DS:1 DS:2 TOTAL
85302024-02-03T17:13:11.177Z New 0 0 0 0
85312024-02-03T17:13:11.177Z Sent 1 1 25 27
85322024-02-03T17:13:11.178Z Done 29 29 5 63
85332024-02-03T17:13:11.178Z Skipped 0 0 0 0
85342024-02-03T17:13:11.178Z Error 0 0 0 0
85352024-02-03T17:13:11.178ZLast Flush: 1025 1025 0
85362024-02-03T17:13:11.178ZDownstairs last five completed:
85372024-02-03T17:13:11.178ZUpstairs last five completed: 30 29 28 27 26
85382024-02-03T17:13:11.178Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.392786746Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444}
85392024-02-03T17:13:11.178ZWrote out file "/tmp/test_repair_verify.data"
85402024-02-03T17:13:11.178Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.393458567Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85412024-02-03T17:13:11.178Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.393670441Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85422024-02-03T17:13:11.178Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.393713208Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85432024-02-03T17:13:11.178Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.393751179Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85442024-02-03T17:13:11.179Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.393783878Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85452024-02-03T17:13:11.179Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.393805709Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85462024-02-03T17:13:11.179Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.393827338Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85472024-02-03T17:13:11.179Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.393909877Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85482024-02-03T17:13:11.179Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.393951792Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85492024-02-03T17:13:11.179Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.393990877Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85502024-02-03T17:13:11.179Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.394095663Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85512024-02-03T17:13:11.179Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.394129542Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85522024-02-03T17:13:11.179Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.39419271Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85532024-02-03T17:13:11.179Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.394235518Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85542024-02-03T17:13:11.179Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.394280564Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85552024-02-03T17:13:11.179Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.394310222Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85562024-02-03T17:13:11.179Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.394331133Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85572024-02-03T17:13:11.180Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.394379449Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85582024-02-03T17:13:11.180Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.394402623Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85592024-02-03T17:13:11.180Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.394424389Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85602024-02-03T17:13:11.180Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.394444558Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85612024-02-03T17:13:11.180Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.394464381Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85622024-02-03T17:13:11.180Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.394509665Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85632024-02-03T17:13:11.180Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.39453193Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85642024-02-03T17:13:11.180Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.394553713Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85652024-02-03T17:13:11.180Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.39457406Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85662024-02-03T17:13:11.180Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.394619024Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85672024-02-03T17:13:11.180Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.394641451Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85682024-02-03T17:13:11.180Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.394686457Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85692024-02-03T17:13:11.180Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.394708871Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85702024-02-03T17:13:11.181Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.394730564Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85712024-02-03T17:13:11.188Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.39475086Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85722024-02-03T17:13:11.188Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.39477095Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85732024-02-03T17:13:11.188Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.394790951Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85742024-02-03T17:13:11.188Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.394835564Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85752024-02-03T17:13:11.188Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.394857959Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85762024-02-03T17:13:11.188Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.394879637Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85772024-02-03T17:13:11.188Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.394924769Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85782024-02-03T17:13:11.189Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.39494711Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85792024-02-03T17:13:11.189Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.394991787Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85802024-02-03T17:13:11.189Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395014068Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85812024-02-03T17:13:11.189Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.39505829Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85822024-02-03T17:13:11.189Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395080333Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85832024-02-03T17:13:11.189Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395101474Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85842024-02-03T17:13:11.189Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395121493Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85852024-02-03T17:13:11.189Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395141279Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85862024-02-03T17:13:11.189Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395186211Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85872024-02-03T17:13:11.189Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395208534Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85882024-02-03T17:13:11.189Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395230262Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85892024-02-03T17:13:11.189Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395250208Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85902024-02-03T17:13:11.189Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395295049Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85912024-02-03T17:13:11.190Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395317212Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85922024-02-03T17:13:11.190Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395338506Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85932024-02-03T17:13:11.190Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395383334Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85942024-02-03T17:13:11.190Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395405663Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85952024-02-03T17:13:11.190Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395450346Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85962024-02-03T17:13:11.190Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395472578Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85972024-02-03T17:13:11.190Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395492466Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85982024-02-03T17:13:11.190Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395536801Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
85992024-02-03T17:13:11.190Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395559072Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86002024-02-03T17:13:11.190Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.3955807Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86012024-02-03T17:13:11.190Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395601067Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86022024-02-03T17:13:11.190Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.3956458Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86032024-02-03T17:13:11.190Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395668132Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86042024-02-03T17:13:11.191Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395712724Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86052024-02-03T17:13:11.191Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395734818Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86062024-02-03T17:13:11.191Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395779443Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86072024-02-03T17:13:11.191Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395801802Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86082024-02-03T17:13:11.191Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395821852Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86092024-02-03T17:13:11.191Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395841682Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86102024-02-03T17:13:11.191Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395885942Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86112024-02-03T17:13:11.191Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395908153Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86122024-02-03T17:13:11.191Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395929944Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86132024-02-03T17:13:11.191Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.39597456Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86142024-02-03T17:13:11.191Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.395996837Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86152024-02-03T17:13:11.191Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.39604086Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86162024-02-03T17:13:11.191Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.396062686Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86172024-02-03T17:13:11.192Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.396084459Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86182024-02-03T17:13:11.192Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.396129155Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86192024-02-03T17:13:11.192Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.396151212Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86202024-02-03T17:13:11.192Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.396171377Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86212024-02-03T17:13:11.192Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.396215656Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86222024-02-03T17:13:11.192Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.396237971Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86232024-02-03T17:13:11.192Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.396259873Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86242024-02-03T17:13:11.192Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.396304389Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86252024-02-03T17:13:11.192Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.396326279Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86262024-02-03T17:13:11.192Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.396371104Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86272024-02-03T17:13:11.192Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.396393305Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86282024-02-03T17:13:11.192Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.396414859Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86292024-02-03T17:13:11.193Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.396459542Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86302024-02-03T17:13:11.193Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.396505643Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86312024-02-03T17:13:11.193Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.396551632Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86322024-02-03T17:13:11.193Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.396575508Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86332024-02-03T17:13:11.193Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.396619925Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86342024-02-03T17:13:11.193Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.396666049Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86352024-02-03T17:13:11.193Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.396712177Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86362024-02-03T17:13:11.193Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.396758336Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86372024-02-03T17:13:11.193Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.396804416Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86382024-02-03T17:13:11.193Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.396850454Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86392024-02-03T17:13:11.193Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.396896708Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86402024-02-03T17:13:11.193Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.396942597Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86412024-02-03T17:13:11.193Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.396988917Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86422024-02-03T17:13:11.193Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.39703522Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86432024-02-03T17:13:11.194Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.397058067Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86442024-02-03T17:13:11.194Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.397102422Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86452024-02-03T17:13:11.194Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.397148513Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86462024-02-03T17:13:11.194Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.397194505Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86472024-02-03T17:13:11.194Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.397240794Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86482024-02-03T17:13:11.194Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.397286943Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86492024-02-03T17:13:11.194Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.3973328Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86502024-02-03T17:13:11.194Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.397379032Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86512024-02-03T17:13:11.194Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.397425254Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86522024-02-03T17:13:11.194Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.397471314Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86532024-02-03T17:13:11.194Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.397517576Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86542024-02-03T17:13:11.194Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.397563772Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86552024-02-03T17:13:11.194Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.397609932Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86562024-02-03T17:13:11.195Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.397632231Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86572024-02-03T17:13:11.195Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.397676593Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86582024-02-03T17:13:11.195Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.397722545Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86592024-02-03T17:13:11.195Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.39776866Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86602024-02-03T17:13:11.195Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.397815151Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86612024-02-03T17:13:11.195Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.397861346Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86622024-02-03T17:13:11.195Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.397907579Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86632024-02-03T17:13:11.195Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.397953979Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86642024-02-03T17:13:11.195Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:10.398000243Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1444,"session_id":"49d683eb-0699-45da-ac82-63941f2334f8"}
86652024-02-03T17:13:11.195Z
86662024-02-03T17:13:11.195ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
86672024-02-03T17:13:11.195ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
86682024-02-03T17:13:11.195Z 0 000-019 66 66 66 115 115 115 F F F
86692024-02-03T17:13:11.196Z 1 020-039 78 78 75 131 131 127 F F T <---
86702024-02-03T17:13:11.196Z 2 040-059 77 77 77 130 130 130 F F F
86712024-02-03T17:13:11.211Z 3 060-079 78 78 77 131 131 130 F F T <---
86722024-02-03T17:13:11.212Z 4 080-099 78 78 74 133 133 126 F F F <---
86732024-02-03T17:13:11.212Z 5 100-119 78 78 74 134 134 124 F F F <---
86742024-02-03T17:13:11.212Z 6 120-139 78 78 76 134 134 129 F F F <---
86752024-02-03T17:13:11.212Z 7 140-159 77 77 77 130 130 130 F F F
86762024-02-03T17:13:11.212Z 8 160-179 78 78 76 131 131 128 F F T <---
86772024-02-03T17:13:11.212Z 9 180-199 78 78 76 131 131 129 F F F <---
86782024-02-03T17:13:11.212Z 10 200-219 76 76 76 128 128 128 F F F
86792024-02-03T17:13:11.212Z 11 220-239 70 70 70 120 120 120 F F F
86802024-02-03T17:13:11.212Z 12 240-259 75 75 75 127 127 127 F F F
86812024-02-03T17:13:11.212Z 13 260-279 78 78 77 133 133 130 F F F <---
86822024-02-03T17:13:11.212Z 14 280-299 76 76 76 128 128 128 F F F
86832024-02-03T17:13:11.212Z 15 300-319 76 76 76 129 129 129 F F F
86842024-02-03T17:13:11.212Z 16 320-339 68 68 68 117 117 117 F F F
86852024-02-03T17:13:11.213Z 17 340-359 76 76 76 129 129 129 F F F
86862024-02-03T17:13:11.213Z 18 360-379 62 62 62 106 106 106 F F F
86872024-02-03T17:13:11.213Z 19 380-399 74 74 74 126 126 126 T T F <---
86882024-02-03T17:13:11.213Z 20 400-419 78 78 77 133 133 130 F F F <---
86892024-02-03T17:13:11.213Z 21 420-439 70 70 70 120 120 120 T T F <---
86902024-02-03T17:13:11.213Z 22 440-459 76 76 76 129 129 129 T T F <---
86912024-02-03T17:13:11.213Z 23 460-479 73 73 73 122 122 122 F F F
86922024-02-03T17:13:11.213Z 24 480-499 73 73 73 122 122 122 F F F
86932024-02-03T17:13:11.213Z 25 500-519 74 74 74 125 125 125 F F F
86942024-02-03T17:13:11.213Z 26 520-539 74 74 74 125 125 125 F F F
86952024-02-03T17:13:11.213Z 27 540-559 76 76 76 129 129 129 T T F <---
86962024-02-03T17:13:11.213Z 28 560-579 70 70 70 120 120 120 T T F <---
86972024-02-03T17:13:11.213Z 29 580-599 73 73 73 122 122 122 F F F
86982024-02-03T17:13:11.214ZMax gen: 78, Max flush: 134
86992024-02-03T17:13:11.214ZError: Difference in extent metadata found!
87002024-02-03T17:13:11.214ZA Difference in extent metadata is expected here
87012024-02-03T17:13:11.214ZOn loop 39
87022024-02-03T17:13:11.214Z
87032024-02-03T17:13:11.214ZVerifying data now
87042024-02-03T17:13:17.106ZNo change after verify
87052024-02-03T17:13:17.107ZLoop: 39 Downstairs dump after verify (and repair):
87062024-02-03T17:13:17.130ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
87072024-02-03T17:13:17.130Z 0 000-019 66 66 66 115 115 115 F F F
87082024-02-03T17:13:17.135Z 1 020-039 78 78 78 131 131 131 F F F
87092024-02-03T17:13:17.135Z 2 040-059 77 77 77 130 130 130 F F F
87102024-02-03T17:13:17.135Z 3 060-079 78 78 78 131 131 131 F F F
87112024-02-03T17:13:17.135Z 4 080-099 78 78 78 133 133 133 F F F
87122024-02-03T17:13:17.135Z 5 100-119 78 78 78 134 134 134 F F F
87132024-02-03T17:13:17.135Z 6 120-139 78 78 78 134 134 134 F F F
87142024-02-03T17:13:17.136Z 7 140-159 77 77 77 130 130 130 F F F
87152024-02-03T17:13:17.136Z 8 160-179 78 78 78 131 131 131 F F F
87162024-02-03T17:13:17.136Z 9 180-199 78 78 78 131 131 131 F F F
87172024-02-03T17:13:17.136Z 10 200-219 76 76 76 128 128 128 F F F
87182024-02-03T17:13:17.136Z 11 220-239 70 70 70 120 120 120 F F F
87192024-02-03T17:13:17.136Z 12 240-259 75 75 75 127 127 127 F F F
87202024-02-03T17:13:17.136Z 13 260-279 78 78 78 133 133 133 F F F
87212024-02-03T17:13:17.136Z 14 280-299 76 76 76 128 128 128 F F F
87222024-02-03T17:13:17.136Z 15 300-319 76 76 76 129 129 129 F F F
87232024-02-03T17:13:17.136Z 16 320-339 68 68 68 117 117 117 F F F
87242024-02-03T17:13:17.136Z 17 340-359 76 76 76 129 129 129 F F F
87252024-02-03T17:13:17.136Z 18 360-379 62 62 62 106 106 106 F F F
87262024-02-03T17:13:17.136Z 19 380-399 79 79 79 135 135 135 F F F
87272024-02-03T17:13:17.136Z 20 400-419 78 78 78 133 133 133 F F F
87282024-02-03T17:13:17.136Z 21 420-439 79 79 79 135 135 135 F F F
87292024-02-03T17:13:17.136Z 22 440-459 79 79 79 135 135 135 F F F
87302024-02-03T17:13:17.136Z 23 460-479 73 73 73 122 122 122 F F F
87312024-02-03T17:13:17.136Z 24 480-499 73 73 73 122 122 122 F F F
87322024-02-03T17:13:17.136Z 25 500-519 74 74 74 125 125 125 F F F
87332024-02-03T17:13:17.136Z 26 520-539 74 74 74 125 125 125 F F F
87342024-02-03T17:13:17.137Z 27 540-559 79 79 79 135 135 135 F F F
87352024-02-03T17:13:17.137Z 28 560-579 79 79 79 135 135 135 F F F
87362024-02-03T17:13:17.137Z 29 580-599 73 73 73 122 122 122 F F F
87372024-02-03T17:13:17.137ZMax gen: 79, Max flush: 135
87382024-02-03T17:13:17.137Z
87392024-02-03T17:13:17.137ZBegin loop 40
87402024-02-03T17:13:17.137ZDownstairs to restart: 1
87412024-02-03T17:13:17.166Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30Crucible runtime is spawned
87422024-02-03T17:13:17.166Z,"time":"2024-02-03T17:13:16.721738539Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452}
87432024-02-03T17:13:17.174Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.722335297Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452}
87442024-02-03T17:13:17.174Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.722421814Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452}
87452024-02-03T17:13:17.174Z{"msg":"Crucible c2f74be3-ba28-492a-b145-32cde76075a6 has session id: 7cca78bb-8ebb-441b-81a6-5f186b6086e7","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.722462261Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87462024-02-03T17:13:17.174Z{"msg":"Upstairs opts: Upstairs UUID: c2f74be3-ba28-492a-b145-32cde76075a6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.722556364Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87472024-02-03T17:13:17.174Z{"msg":"Crucible stats registered with UUID: c2f74be3-ba28-492a-b145-32cde76075a6","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.722713773Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87482024-02-03T17:13:17.174Z{"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.722754582Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452}
87492024-02-03T17:13:17.174Z{"msg":"c2f74be3-ba28-492a-b145-32cde76075a6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.722786449Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87502024-02-03T17:13:17.174Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.722822574Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"client":"0","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87512024-02-03T17:13:17.174Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.722926963Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"client":"0","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87522024-02-03T17:13:17.174Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.723107697Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"client":"1","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87532024-02-03T17:13:17.174Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.723191642Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"client":"1","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87542024-02-03T17:13:17.174Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.723220646Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"client":"2","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87552024-02-03T17:13:17.174Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.723244092Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"client":"2","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87562024-02-03T17:13:17.174Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.723292558Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"":"io task","client":"2","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87572024-02-03T17:13:17.174Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.72331854Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"":"io task","client":"0","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87582024-02-03T17:13:17.174Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.723341986Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"":"io task","client":"1","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87592024-02-03T17:13:17.175Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.723408183Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"":"io task","client":"2","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87602024-02-03T17:13:17.175Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.723458014Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"":"io task","client":"0","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87612024-02-03T17:13:17.175Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.723501517Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"":"io task","client":"1","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87622024-02-03T17:13:17.175Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.72491181Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"client":"2","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87632024-02-03T17:13:17.175Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.724991414Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"client":"2","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87642024-02-03T17:13:17.175Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.725106533Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"client":"0","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87652024-02-03T17:13:17.175Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.725151123Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"client":"0","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87662024-02-03T17:13:17.175Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.725775084Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"client":"1","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87672024-02-03T17:13:17.175Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.725866878Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"client":"1","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87682024-02-03T17:13:17.175Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.726069177Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"client":"2","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87692024-02-03T17:13:17.175Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.726099289Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"client":"0","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87702024-02-03T17:13:17.175Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.726914705Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"client":"1","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87712024-02-03T17:13:17.175Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.727033419Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"client":"0","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87722024-02-03T17:13:17.175Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.727061128Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"client":"0","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87732024-02-03T17:13:17.175Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.727084341Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87742024-02-03T17:13:17.175Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.727104903Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"client":"2","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87752024-02-03T17:13:17.175Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.727127609Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"client":"2","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87762024-02-03T17:13:17.175Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.72717821Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87772024-02-03T17:13:17.175Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.727680997Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"client":"1","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87782024-02-03T17:13:17.175Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.727748095Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"client":"1","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87792024-02-03T17:13:17.180Z{"msg":"[0]R flush_numbers[0..12]: [115, 131, 130, 131, 133, 134, 134, 130, 131, 131, 128, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.727856854Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"":"downstairs","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87802024-02-03T17:13:17.180Z{"msg":"[0]R generation[0..12]: [66, 78, 77, 78, 78, 78, 78, 77, 78, 78, 76, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.727957115Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"":"downstairs","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87812024-02-03T17:13:17.180Z{"msg":"Wait for a query_work_queue command to finish before sending IO
87822024-02-03T17:13:17.180Z[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.728088028Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"":"downstairs","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87832024-02-03T17:13:17.180Z{"msg":"[1]R flush_numbers[0..12]: [115, 131, 130, 131, 133, 134, 134, 130, 131, 131, 128, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.728139695Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"":"downstairs","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87842024-02-03T17:13:17.180Z{"msg":"[1]R generation[0..12]: [66, 78, 77, 78, 78, 78, 78, 77, 78, 78, 76, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.72820665Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"":"downstairs","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87852024-02-03T17:13:17.180Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.7282356Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"":"downstairs","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87862024-02-03T17:13:17.181Z{"msg":"[2]R flush_numbers[0..12]: [115, 131, 130, 131, 133, 134, 134, 130, 131, 131, 128, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.728304189Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"":"downstairs","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87872024-02-03T17:13:17.181Z{"msg":"[2]R generation[0..12]: [66, 78, 77, 78, 78, 78, 78, 77, 78, 78, 76, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.728332752Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"":"downstairs","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87882024-02-03T17:13:17.181Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.728398554Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"":"downstairs","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87892024-02-03T17:13:17.181Z{"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.728429687Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"":"downstairs","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87902024-02-03T17:13:17.181Z{"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.728493422Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"":"downstairs","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87912024-02-03T17:13:17.181Z{"msg":"Next flush: 136","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.728519747Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"":"downstairs","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87922024-02-03T17:13:17.181Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.728583665Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"":"downstairs","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87932024-02-03T17:13:17.181Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.728613767Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87942024-02-03T17:13:17.181Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.728675317Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"client":"0","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87952024-02-03T17:13:17.181Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.728702412Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"client":"1","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87962024-02-03T17:13:17.181Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.728732283Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"client":"2","":"downstairs","session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87972024-02-03T17:13:17.181Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.728793925Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87982024-02-03T17:13:17.181Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.728819104Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
87992024-02-03T17:13:17.181Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
88002024-02-03T17:13:17.181Zc2f74be3-ba28-492a-b145-32cde76075a6 is now active with session: 7cca78bb-8ebb-441b-81a6-5f186b6086e7","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.728857122Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
88012024-02-03T17:13:17.181Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.728881247Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452,"session_id":"7cca78bb-8ebb-441b-81a6-5f186b6086e7"}
88022024-02-03T17:13:17.181Z{"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:16.728946262Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452}
88032024-02-03T17:13:17.181ZLoading write count information from file "/tmp/test_repair_verify.data"
88042024-02-03T17:13:17.181ZRun Repair workload
88052024-02-03T17:13:17.181Z01/30 Write block 352 len 36864 data: 6 6 6 8 6 7 7 8 8
88062024-02-03T17:13:17.182Z02/30 Read block 140 len 28672
88072024-02-03T17:13:17.187Z03/30 Read block 220 len 4096
88082024-02-03T17:13:17.188Z04/30 Flush
88092024-02-03T17:13:17.188Z05/30 Write block 196 len 12288 data: 7 6 5
88102024-02-03T17:13:17.188Z06/30 Write block 181 len 16384 data: 7 6 9 11
88112024-02-03T17:13:17.188Z07/30 Read block 149 len 12288
88122024-02-03T17:13:17.188Z08/30 Write block 535 len 16384 data: 7 5 4 4
88132024-02-03T17:13:17.191Z09/30 Write block 4 len 28672 data: 6 6 7 6 7 8 8
88142024-02-03T17:13:17.191Z10/30 Write block 19 len 24576 data: 8 6 9 11 11 11
88152024-02-03T17:13:17.191Z11/30 Write block 6 len 4096 data: 8
88162024-02-03T17:13:17.191Z12/30 Read block 315 len 20480
88172024-02-03T17:13:17.194Z13/30 Write block 168 len 20480 data: 9 9 9 8 6
88182024-02-03T17:13:17.194Z14/30 Read block 570 len 28672
88192024-02-03T17:13:17.197Z15/30 Read block 11 len 32768
88202024-02-03T17:13:17.197Z16/30 Write block 163 len 28672 data: 13 13 12 14 14 10 10
88212024-02-03T17:13:17.200Z17/30 Write block 13 len 36864 data: 6 7 5 5 6 7 9 7 10
88222024-02-03T17:13:17.200Z18/30 Read block 36 len 40960
88232024-02-03T17:13:17.207Z19/30 Read block 567 len 20480
88242024-02-03T17:13:17.210Z20/30 Write block 455 len 32768 data: 3 3 4 4 5 7 6 6
88252024-02-03T17:13:17.210Z21/30 Read block 529 len 12288
88262024-02-03T17:13:17.210Z22/30 Read block 456 len 20480
88272024-02-03T17:13:17.210Z23/30 Flush
88282024-02-03T17:13:17.215Z24/30 Write block 368 len 8192 data: 8 7
88292024-02-03T17:13:17.220Z25/30 Read block 500 len 16384
88302024-02-03T17:13:17.220Z26/30 Flush
88312024-02-03T17:13:17.220Z27/30 Write block 256 len 12288 data: 5 5 6
88322024-02-03T17:13:17.224Z28/30 Write block 440 len 12288 data: 7 9 12
88332024-02-03T17:13:17.224Z29/30 Write block 494 len 40960 data: 6 6 6 5 4 4 4 6 6 7
88342024-02-03T17:13:17.224Z30/30 Write block 550 len 32768 data: 9 8 8 9 8 6 5 5
88352024-02-03T17:13:17.228Z----------------------------------------------------------------
88362024-02-03T17:13:17.228Z Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:30
88372024-02-03T17:13:17.228ZStates: Active Active Active
88382024-02-03T17:13:17.228ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
88392024-02-03T17:13:17.229Z 1 Acked 1000 Write 9 Done Done Done false
88402024-02-03T17:13:17.229Z 2 Acked 1001 Read 7 Done Done Done false
88412024-02-03T17:13:17.229Z 3 Acked 1002 Read 1 Done Done Done false
88422024-02-03T17:13:17.229Z 4 Acked 1003 Flush 0 Done Sent Done false
88432024-02-03T17:13:17.229Z 5 Acked 1004 Write 3 Done Sent Done false
88442024-02-03T17:13:17.229Z 6 Acked 1005 Write 4 Done Sent Done false
88452024-02-03T17:13:17.229Z 7 Acked 1006 Read 3 Done Sent Done false
88462024-02-03T17:13:17.229Z 8 Acked 1007 Write 4 Done Sent Done false
88472024-02-03T17:13:17.229Z 9 Acked 1008 Write 7 Done Sent Done false
88482024-02-03T17:13:17.229Z 10 Acked 1009 Write 6 Done Sent Done false
88492024-02-03T17:13:17.229Z 11 Acked 1010 Write 1 Done Sent Done false
88502024-02-03T17:13:17.229Z 12 Acked 1011 Read 5 Done Sent Done false
88512024-02-03T17:13:17.229Z 13 Acked 1012 Write 5 Done Sent Done false
88522024-02-03T17:13:17.229Z 14 Acked 1013 Read 7 Done Sent Done false
88532024-02-03T17:13:17.229Z 15 Acked 1014 Read 8 Done Sent Done false
88542024-02-03T17:13:17.229Z 16 Acked 1015 Write 7 Done Sent Done false
88552024-02-03T17:13:17.229Z 17 Acked 1016 Write 9 Done Sent Done false
88562024-02-03T17:13:17.229Z 18 Acked 1017 Read 10 Done Sent Done false
88572024-02-03T17:13:17.229Z 19 Acked 1018 Read 5 Done Sent Done false
88582024-02-03T17:13:17.229Z 20 Acked 1019 Write 8 Done Sent Done false
88592024-02-03T17:13:17.229Z 21 Acked 1020 Read 3 Done Sent Done false
88602024-02-03T17:13:17.229Z 22 Acked 1021 Read 5 Done Sent Done false
88612024-02-03T17:13:17.230Z 23 Acked 1022 Flush 0 Done Sent Done false
88622024-02-03T17:13:17.230Z 24 Acked 1023 Write 2 Done Sent Done false
88632024-02-03T17:13:17.230Z 25 Acked 1024 Read 4 Done Sent Done false
88642024-02-03T17:13:17.230Z 26 Acked 1025 Flush 0 Done Sent Done false
88652024-02-03T17:13:17.230Z 27 Acked 1026 Write 3 Done Sent Done false
88662024-02-03T17:13:17.230Z 28 Acked 1027 Write 3 Done Sent Done false
88672024-02-03T17:13:17.230Z 29 Acked 1028 Write 10 Done Sent Done false
88682024-02-03T17:13:17.230Z 30 Acked 1029 Write 8 Sent Sent Sent false
88692024-02-03T17:13:17.230Z STATES DS:0 DS:1 DS:2 TOTAL
88702024-02-03T17:13:17.230Z New 0 0 0 0
88712024-02-03T17:13:17.230Z Sent 1 27 1 29
88722024-02-03T17:13:17.230Z Done 29 3 29 61
88732024-02-03T17:13:17.230Z Skipped 0 0 0 0
88742024-02-03T17:13:17.230Z Error 0 0 0 0
88752024-02-03T17:13:17.231ZLast Flush: 1025 0 1025
88762024-02-03T17:13:17.231ZDownstairs last five completed:
88772024-02-03T17:13:17.231ZUpstairs last five completed: 30 29 28 27 26
88782024-02-03T17:13:17.231Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:16.780916841Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1452}
88792024-02-03T17:13:17.231ZWrote out file "/tmp/test_repair_verify.data"
88802024-02-03T17:13:17.234Z
88812024-02-03T17:13:17.239ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
88822024-02-03T17:13:17.265ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
88832024-02-03T17:13:17.265Z 0 000-019 80 66 80 137 115 137 F F F <---
88842024-02-03T17:13:17.265Z 1 020-039 80 78 80 137 131 137 F F F <---
88852024-02-03T17:13:17.265Z 2 040-059 77 77 77 130 130 130 F F F
88862024-02-03T17:13:17.265Z 3 060-079 78 78 78 131 131 131 F F F
88872024-02-03T17:13:17.265Z 4 080-099 78 78 78 133 133 133 F F F
88882024-02-03T17:13:17.269Z 5 100-119 78 78 78 134 134 134 F F F
88892024-02-03T17:13:17.269Z 6 120-139 78 78 78 134 134 134 F F F
88902024-02-03T17:13:17.269Z 7 140-159 77 77 77 130 130 130 F F F
88912024-02-03T17:13:17.269Z 8 160-179 80 78 80 137 131 137 F F F <---
88922024-02-03T17:13:17.269Z 9 180-199 80 78 80 137 131 137 F F F <---
88932024-02-03T17:13:17.269Z 10 200-219 76 76 76 128 128 128 F F F
88942024-02-03T17:13:17.269Z 11 220-239 70 70 70 120 120 120 F F F
88952024-02-03T17:13:17.269Z 12 240-259 75 75 75 127 127 127 T F T <---
88962024-02-03T17:13:17.269Z 13 260-279 78 78 78 133 133 133 F F F
88972024-02-03T17:13:17.269Z 14 280-299 76 76 76 128 128 128 F F F
88982024-02-03T17:13:17.269Z 15 300-319 76 76 76 129 129 129 F F F
88992024-02-03T17:13:17.269Z 16 320-339 68 68 68 117 117 117 F F F
89002024-02-03T17:13:17.269Z 17 340-359 80 76 80 136 129 136 F T F <---
89012024-02-03T17:13:17.270Z 18 360-379 80 62 80 138 106 138 F T F <---
89022024-02-03T17:13:17.270Z 19 380-399 79 79 79 135 135 135 F F F
89032024-02-03T17:13:17.270Z 20 400-419 78 78 78 133 133 133 F F F
89042024-02-03T17:13:17.270Z 21 420-439 79 79 79 135 135 135 F F F
89052024-02-03T17:13:17.270Z 22 440-459 80 79 80 137 135 137 T F T <---
89062024-02-03T17:13:17.270Z 23 460-479 80 73 80 137 122 137 F F F <---
89072024-02-03T17:13:17.270Z 24 480-499 73 73 73 122 122 122 T F T <---
89082024-02-03T17:13:17.270Z 25 500-519 74 74 74 125 125 125 T F T <---
89092024-02-03T17:13:17.270Z 26 520-539 80 74 80 137 125 137 F F F <---
89102024-02-03T17:13:17.270Z 27 540-559 79 79 79 135 135 135 T F T <---
89112024-02-03T17:13:17.270Z 28 560-579 79 79 79 135 135 135 F F F
89122024-02-03T17:13:17.270Z 29 580-599 73 73 73 122 122 122 F F F
89132024-02-03T17:13:17.270ZMax gen: 80, Max flush: 138
89142024-02-03T17:13:17.270ZError: Difference in extent metadata found!
89152024-02-03T17:13:17.274ZA Difference in extent metadata is expected here
89162024-02-03T17:13:17.274ZOn loop 40
89172024-02-03T17:13:17.274Z
89182024-02-03T17:13:17.274ZVerifying data now
89192024-02-03T17:13:23.416ZNo change after verify
89202024-02-03T17:13:23.416ZLoop: 40 Downstairs dump after verify (and repair):
89212024-02-03T17:13:23.439ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
89222024-02-03T17:13:23.439Z 0 000-019 80 80 80 137 137 137 F F F
89232024-02-03T17:13:23.439Z 1 020-039 80 80 80 137 137 137 F F F
89242024-02-03T17:13:23.439Z 2 040-059 77 77 77 130 130 130 F F F
89252024-02-03T17:13:23.443Z 3 060-079 78 78 78 131 131 131 F F F
89262024-02-03T17:13:23.443Z 4 080-099 78 78 78 133 133 133 F F F
89272024-02-03T17:13:23.443Z 5 100-119 78 78 78 134 134 134 F F F
89282024-02-03T17:13:23.443Z 6 120-139 78 78 78 134 134 134 F F F
89292024-02-03T17:13:23.443Z 7 140-159 77 77 77 130 130 130 F F F
89302024-02-03T17:13:23.443Z 8 160-179 80 80 80 137 137 137 F F F
89312024-02-03T17:13:23.443Z 9 180-199 80 80 80 137 137 137 F F F
89322024-02-03T17:13:23.443Z 10 200-219 76 76 76 128 128 128 F F F
89332024-02-03T17:13:23.443Z 11 220-239 70 70 70 120 120 120 F F F
89342024-02-03T17:13:23.443Z 12 240-259 81 81 81 139 139 139 F F F
89352024-02-03T17:13:23.443Z 13 260-279 78 78 78 133 133 133 F F F
89362024-02-03T17:13:23.443Z 14 280-299 76 76 76 128 128 128 F F F
89372024-02-03T17:13:23.444Z 15 300-319 76 76 76 129 129 129 F F F
89382024-02-03T17:13:23.444Z 16 320-339 68 68 68 117 117 117 F F F
89392024-02-03T17:13:23.444Z 17 340-359 80 80 80 136 136 136 F F F
89402024-02-03T17:13:23.444Z 18 360-379 80 80 80 138 138 138 F F F
89412024-02-03T17:13:23.444Z 19 380-399 79 79 79 135 135 135 F F F
89422024-02-03T17:13:23.444Z 20 400-419 78 78 78 133 133 133 F F F
89432024-02-03T17:13:23.444Z 21 420-439 79 79 79 135 135 135 F F F
89442024-02-03T17:13:23.444Z 22 440-459 81 81 81 139 139 139 F F F
89452024-02-03T17:13:23.444Z 23 460-479 80 80 80 137 137 137 F F F
89462024-02-03T17:13:23.444Z 24 480-499 81 81 81 139 139 139 F F F
89472024-02-03T17:13:23.444Z 25 500-519 81 81 81 139 139 139 F F F
89482024-02-03T17:13:23.444Z 26 520-539 80 80 80 137 137 137 F F F
89492024-02-03T17:13:23.444Z 27 540-559 81 81 81 139 139 139 F F F
89502024-02-03T17:13:23.444Z 28 560-579 79 79 79 135 135 135 F F F
89512024-02-03T17:13:23.444Z 29 580-599 73 73 73 122 122 122 F F F
89522024-02-03T17:13:23.444ZMax gen: 81, Max flush: 139
89532024-02-03T17:13:23.458Z
89542024-02-03T17:13:23.458ZBegin loop 41
89552024-02-03T17:13:23.458ZDownstairs to restart: 1
89562024-02-03T17:13:23.480Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.035766323Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460}
89572024-02-03T17:13:23.480Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.036309845Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460}
89582024-02-03T17:13:23.484ZCrucible runtime is spawned
89592024-02-03T17:13:23.484Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.03644924Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460}
89602024-02-03T17:13:23.485Z{"msg":"Crucible c9317e7f-5c42-4d10-8aab-d039e4efc4be has session id: 735ce847-6629-4f0d-b46f-7f1235701ce7","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.036524556Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89612024-02-03T17:13:23.485Z{"msg":"Upstairs opts: Upstairs UUID: c9317e7f-5c42-4d10-8aab-d039e4efc4be, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.036589823Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89622024-02-03T17:13:23.485Z{"msg":"Crucible stats registered with UUID: c9317e7f-5c42-4d10-8aab-d039e4efc4be","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.036654141Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89632024-02-03T17:13:23.485Z{"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.036695591Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460}
89642024-02-03T17:13:23.485Z{"msg":"c9317e7f-5c42-4d10-8aab-d039e4efc4be active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.036864784Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89652024-02-03T17:13:23.485Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.036949062Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"client":"0","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89662024-02-03T17:13:23.485Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.036981402Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"client":"0","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89672024-02-03T17:13:23.485Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.037052803Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"client":"1","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89682024-02-03T17:13:23.485Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.037083149Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"client":"1","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89692024-02-03T17:13:23.485Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.03714348Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"client":"2","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89702024-02-03T17:13:23.485Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.037188228Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"client":"2","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89712024-02-03T17:13:23.485Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.037226867Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"":"io task","client":"2","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89722024-02-03T17:13:23.485Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.037310024Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"":"io task","client":"0","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89732024-02-03T17:13:23.485Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.037358058Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"":"io task","client":"1","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89742024-02-03T17:13:23.485Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.037399561Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"":"io task","client":"0","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89752024-02-03T17:13:23.485Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.037512434Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"":"io task","client":"1","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89762024-02-03T17:13:23.485Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.037615887Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"":"io task","client":"2","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89772024-02-03T17:13:23.485Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.038663107Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"client":"2","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89782024-02-03T17:13:23.485Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.038751826Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"client":"2","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89792024-02-03T17:13:23.485Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.038783633Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"client":"0","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89802024-02-03T17:13:23.486Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.038861134Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"client":"0","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89812024-02-03T17:13:23.492Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.04014978Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"client":"2","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89822024-02-03T17:13:23.492Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.040257088Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"client":"0","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89832024-02-03T17:13:23.492Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.040412537Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"client":"1","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89842024-02-03T17:13:23.492Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.040479105Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"client":"1","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89852024-02-03T17:13:23.492Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.041279846Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"client":"0","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89862024-02-03T17:13:23.492Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.041350105Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"client":"0","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89872024-02-03T17:13:23.492Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.041376895Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89882024-02-03T17:13:23.492Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.041398001Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"client":"2","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89892024-02-03T17:13:23.492Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.041425699Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"client":"2","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89902024-02-03T17:13:23.492Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.041570394Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89912024-02-03T17:13:23.493Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.041840405Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"client":"1","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89922024-02-03T17:13:23.493Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.042621761Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"client":"1","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89932024-02-03T17:13:23.493Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.042676709Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"client":"1","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89942024-02-03T17:13:23.493Z{"msg":"[0]R flush_numbers[0..12]: [137, 137, 130, 131, 133, 134, 134, 130, 137, 137, 128, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.042724034Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"":"downstairs","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89952024-02-03T17:13:23.493Z{"msg":"[0]R generation[0..12]: [80, 80, 77, 78, 78, 78, 78, 77, 80, 80, 76, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.042787854Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"":"downstairs","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89962024-02-03T17:13:23.493Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.042815187Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"":"downstairs","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89972024-02-03T17:13:23.493Z{"msg":"[1]R flush_numbers[0..12]: [137, 137, 130, 131, 133, 134, 134, 130, 137, 137, 128, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.042866402Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"":"downstairs","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89982024-02-03T17:13:23.493Z{"msg":"[1]R generation[0..12]: [80, 80, 77, 78, 78, 78, 78, 77, 80, 80, 76, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.042891255Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"":"downstairs","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
89992024-02-03T17:13:23.493Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.042915245Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"":"downstairs","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
90002024-02-03T17:13:23.493Z{"msg":"[2]R flush_numbers[0..12]: [137, 137, 130, 131, 133, 134, 134, 130, 137, 137, 128, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.042938551Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"":"downstairs","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
90012024-02-03T17:13:23.493Z{"msg":"[2]R generation[0..12]: [80, 80, 77, 78, 78, 78, 78, 77, 80, 80, 76, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.042961188Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"":"downstairs","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
90022024-02-03T17:13:23.494Z{"msg":"Wait for a query_work_queue command to finish before sending IO
90032024-02-03T17:13:23.494Z[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.042988191Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"":"downstairs","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
90042024-02-03T17:13:23.494Z{"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.043009785Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"":"downstairs","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
90052024-02-03T17:13:23.494Z{"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.043033436Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"":"downstairs","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
90062024-02-03T17:13:23.494Z{"msg":"Next flush: 140","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.043105202Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"":"downstairs","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
90072024-02-03T17:13:23.494Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.043129114Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"":"downstairs","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
90082024-02-03T17:13:23.494Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.043151111Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
90092024-02-03T17:13:23.494Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.043180442Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"client":"0","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
90102024-02-03T17:13:23.494Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.043203118Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"client":"1","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
90112024-02-03T17:13:23.494Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.043224718Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"client":"2","":"downstairs","session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
90122024-02-03T17:13:23.494Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.043275689Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
90132024-02-03T17:13:23.495Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.043298312Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
90142024-02-03T17:13:23.495Z{"msg":"c9317e7f-5c42-4d10-8aab-d039e4efc4be is now active with session: 735ce847-6629-4f0d-b46f-7f1235701ce7","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.043319672Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
90152024-02-03T17:13:23.495Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.043339153Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460,"session_id":"735ce847-6629-4f0d-b46f-7f1235701ce7"}
90162024-02-03T17:13:23.495Z{"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:23.043359261Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460}
90172024-02-03T17:13:23.495ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
90182024-02-03T17:13:23.495ZLoading write count information from file "/tmp/test_repair_verify.data"
90192024-02-03T17:13:23.495ZRun Repair workload
90202024-02-03T17:13:23.495Z01/30 Write block 126 len 28672 data: 5 6 6 7 7 7 7
90212024-02-03T17:13:23.499Z02/30 Write block 359 len 36864 data: 9 9 9 6 5 6 6 6 7
90222024-02-03T17:13:23.499Z03/30 Read block 159 len 8192
90232024-02-03T17:13:23.499Z04/30 Write block 334 len 36864 data: 7 5 5 6 8 7 8 6 6
90242024-02-03T17:13:23.499Z05/30 Write block 502 len 32768 data: 7 8 9 7 8 8 8 9
90252024-02-03T17:13:23.499Z06/30 Flush
90262024-02-03T17:13:23.503Z07/30 Write block 515 len 8192 data: 9 8
90272024-02-03T17:13:23.503Z08/30 Read block 311 len 16384
90282024-02-03T17:13:23.506Z09/30 Read block 67 len 36864
90292024-02-03T17:13:23.509Z10/30 Write block 226 len 20480 data: 3 4 6 4 4
90302024-02-03T17:13:23.509Z11/30 Flush
90312024-02-03T17:13:23.512Z12/30 Write block 128 len 20480 data: 7 8 8 8 8
90322024-02-03T17:13:23.512Z13/30 Read block 460 len 36864
90332024-02-03T17:13:23.529Z14/30 Read block 539 len 12288
90342024-02-03T17:13:23.529Z15/30 Read block 64 len 16384
90352024-02-03T17:13:23.536Z16/30 Read block 256 len 36864
90362024-02-03T17:13:23.536Z17/30 Read block 117 len 28672
90372024-02-03T17:13:23.536Z18/30 Write block 577 len 40960 data: 4 4 5 5 5 4 4 3 4 8
90382024-02-03T17:13:23.537Z19/30 Flush
90392024-02-03T17:13:23.537Z20/30 Write block 545 len 24576 data: 5 8 11 8 9 10
90402024-02-03T17:13:23.537Z21/30 Write block 112 len 36864 data: 6 4 6 5 6 9 9 8 7
90412024-02-03T17:13:23.537Z22/30 Flush
90422024-02-03T17:13:23.537Z23/30 Write block 212 len 4096 data: 7
90432024-02-03T17:13:23.542Z24/30 Read block 120 len 36864
90442024-02-03T17:13:23.543Z25/30 Read block 426 len 20480
90452024-02-03T17:13:23.543Z26/30 Read block 77 len 36864
90462024-02-03T17:13:23.543Z27/30 Read block 230 len 24576
90472024-02-03T17:13:23.543Z28/30 Write block 453 len 32768 data: 5 5 4 4 5 5 6 8
90482024-02-03T17:13:23.543Z29/30 Write block 93 len 20480 data: 10 10 8 10 9
90492024-02-03T17:13:23.543Z30/30 Read block 108 len 8192
90502024-02-03T17:13:23.547Z----------------------------------------------------------------
90512024-02-03T17:13:23.547Z Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:30
90522024-02-03T17:13:23.547ZStates: Active Active Active
90532024-02-03T17:13:23.547ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
90542024-02-03T17:13:23.547Z 1 Acked 1000 Write 7 Done Done Done false
90552024-02-03T17:13:23.547Z 2 Acked 1001 Write 9 Done Done Done false
90562024-02-03T17:13:23.547Z 3 Acked 1002 Read 2 Done Done Done false
90572024-02-03T17:13:23.547Z 4 Acked 1003 Write 9 Done Sent Done false
90582024-02-03T17:13:23.548Z 5 Acked 1004 Write 8 Done Sent Done false
90592024-02-03T17:13:23.548Z 6 Acked 1005 Flush 0 Done Sent Done false
90602024-02-03T17:13:23.548Z 7 Acked 1006 Write 2 Done Sent Done false
90612024-02-03T17:13:23.548Z 8 Acked 1007 Read 4 Done Sent Done false
90622024-02-03T17:13:23.548Z 9 Acked 1008 Read 9 Done Sent Done false
90632024-02-03T17:13:23.548Z 10 Acked 1009 Write 5 Done Sent Done false
90642024-02-03T17:13:23.548Z 11 Acked 1010 Flush 0 Done Sent Done false
90652024-02-03T17:13:23.548Z 12 Acked 1011 Write 5 Done Sent Done false
90662024-02-03T17:13:23.548Z 13 Acked 1012 Read 9 Done Sent Done false
90672024-02-03T17:13:23.548Z 14 Acked 1013 Read 3 Done Sent Done false
90682024-02-03T17:13:23.548Z 15 Acked 1014 Read 4 Done Sent Done false
90692024-02-03T17:13:23.548Z 16 Acked 1015 Read 9 Done Sent Done false
90702024-02-03T17:13:23.548Z 17 Acked 1016 Read 7 Done Sent Done false
90712024-02-03T17:13:23.549Z 18 Acked 1017 Write 10 Done Sent Done false
90722024-02-03T17:13:23.549Z 19 Acked 1018 Flush 0 Done Sent Done false
90732024-02-03T17:13:23.549Z 20 Acked 1019 Write 6 Done Sent Done false
90742024-02-03T17:13:23.549Z 21 Acked 1020 Write 9 Done Sent Done false
90752024-02-03T17:13:23.549Z 22 Acked 1021 Flush 0 Done Sent Done false
90762024-02-03T17:13:23.549Z 23 Acked 1022 Write 1 Done Sent Done false
90772024-02-03T17:13:23.549Z 24 Acked 1023 Read 9 Done Sent Done false
90782024-02-03T17:13:23.549Z 25 Acked 1024 Read 5 Done Sent Done false
90792024-02-03T17:13:23.549Z 26 Acked 1025 Read 9 Done Sent Done false
90802024-02-03T17:13:23.549Z 27 Acked 1026 Read 6 Done Sent Done false
90812024-02-03T17:13:23.549Z 28 Acked 1027 Write 8 Done Sent Done false
90822024-02-03T17:13:23.549Z 29 Acked 1028 Write 5 Done Sent Done false
90832024-02-03T17:13:23.549Z 30 Acked 1029 Read 2 Done Sent Sent false
90842024-02-03T17:13:23.550Z STATES DS:0 DS:1 DS:2 TOTAL
90852024-02-03T17:13:23.550Z New 0 0 0 0
90862024-02-03T17:13:23.550Z Sent 0 27 1 28
90872024-02-03T17:13:23.550Z Done 30 3 29 62
90882024-02-03T17:13:23.550Z Skipped 0 0 0 0
90892024-02-03T17:13:23.550Z Error 0 0 0 0
90902024-02-03T17:13:23.550ZLast Flush: 1021 0 1021
90912024-02-03T17:13:23.550ZDownstairs last five completed:
90922024-02-03T17:13:23.550ZUpstairs last five completed: 30 29 28 27 26
90932024-02-03T17:13:23.550Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:23.100343528Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1460}
90942024-02-03T17:13:23.550ZWrote out file "/tmp/test_repair_verify.data"
90952024-02-03T17:13:23.554Z
90962024-02-03T17:13:23.559ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
90972024-02-03T17:13:23.605ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
90982024-02-03T17:13:23.605Z 0 000-019 80 80 80 137 137 137 F F F
90992024-02-03T17:13:23.606Z 1 020-039 80 80 80 137 137 137 F F F
91002024-02-03T17:13:23.606Z 2 040-059 77 77 77 130 130 130 F F F
91012024-02-03T17:13:23.606Z 3 060-079 78 78 78 131 131 131 F F F
91022024-02-03T17:13:23.606Z 4 080-099 78 78 78 133 133 133 T F T <---
91032024-02-03T17:13:23.616Z 5 100-119 82 78 82 143 134 143 F F F <---
91042024-02-03T17:13:23.616Z 6 120-139 82 78 82 143 134 143 F T F <---
91052024-02-03T17:13:23.616Z 7 140-159 77 77 77 130 130 130 F F F
91062024-02-03T17:13:23.616Z 8 160-179 80 80 80 137 137 137 F F F
91072024-02-03T17:13:23.616Z 9 180-199 80 80 80 137 137 137 F F F
91082024-02-03T17:13:23.617Z 10 200-219 76 76 76 128 128 128 T F T <---
91092024-02-03T17:13:23.617Z 11 220-239 82 70 82 141 120 141 F F F <---
91102024-02-03T17:13:23.617Z 12 240-259 81 81 81 139 139 139 F F F
91112024-02-03T17:13:23.617Z 13 260-279 78 78 78 133 133 133 F F F
91122024-02-03T17:13:23.617Z 14 280-299 76 76 76 128 128 128 F F F
91132024-02-03T17:13:23.617Z 15 300-319 76 76 76 129 129 129 F F F
91142024-02-03T17:13:23.617Z 16 320-339 82 68 82 140 117 140 F F F <---
91152024-02-03T17:13:23.617Z 17 340-359 82 80 82 140 136 140 F T F <---
91162024-02-03T17:13:23.617Z 18 360-379 82 80 82 140 138 140 F T F <---
91172024-02-03T17:13:23.617Z 19 380-399 79 79 79 135 135 135 F F F
91182024-02-03T17:13:23.617Z 20 400-419 78 78 78 133 133 133 F F F
91192024-02-03T17:13:23.617Z 21 420-439 79 79 79 135 135 135 F F F
91202024-02-03T17:13:23.618Z 22 440-459 81 81 81 139 139 139 T F T <---
91212024-02-03T17:13:23.618Z 23 460-479 80 80 80 137 137 137 T F T <---
91222024-02-03T17:13:23.618Z 24 480-499 81 81 81 139 139 139 F F F
91232024-02-03T17:13:23.618Z 25 500-519 82 81 82 141 139 141 F F F <---
91242024-02-03T17:13:23.618Z 26 520-539 80 80 80 137 137 137 F F F
91252024-02-03T17:13:23.618Z 27 540-559 82 81 82 143 139 143 F F F <---
91262024-02-03T17:13:23.618Z 28 560-579 82 79 82 142 135 142 F F F <---
91272024-02-03T17:13:23.618Z 29 580-599 82 73 82 142 122 142 F F F <---
91282024-02-03T17:13:23.618ZMax gen: 82, Max flush: 143
91292024-02-03T17:13:23.618ZError: Difference in extent metadata found!
91302024-02-03T17:13:23.618ZA Difference in extent metadata is expected here
91312024-02-03T17:13:23.618ZOn loop 41
91322024-02-03T17:13:23.618Z
91332024-02-03T17:13:23.619ZVerifying data now
91342024-02-03T17:13:29.819ZNo change after verify
91352024-02-03T17:13:29.819ZLoop: 41 Downstairs dump after verify (and repair):
91362024-02-03T17:13:29.841ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
91372024-02-03T17:13:29.842Z 0 000-019 80 80 80 137 137 137 F F F
91382024-02-03T17:13:29.842Z 1 020-039 80 80 80 137 137 137 F F F
91392024-02-03T17:13:29.842Z 2 040-059 77 77 77 130 130 130 F F F
91402024-02-03T17:13:29.842Z 3 060-079 78 78 78 131 131 131 F F F
91412024-02-03T17:13:29.842Z 4 080-099 83 83 83 144 144 144 F F F
91422024-02-03T17:13:29.845Z 5 100-119 82 82 82 143 143 143 F F F
91432024-02-03T17:13:29.845Z 6 120-139 82 82 82 143 143 143 F F F
91442024-02-03T17:13:29.845Z 7 140-159 77 77 77 130 130 130 F F F
91452024-02-03T17:13:29.845Z 8 160-179 80 80 80 137 137 137 F F F
91462024-02-03T17:13:29.845Z 9 180-199 80 80 80 137 137 137 F F F
91472024-02-03T17:13:29.845Z 10 200-219 83 83 83 144 144 144 F F F
91482024-02-03T17:13:29.845Z 11 220-239 82 82 82 141 141 141 F F F
91492024-02-03T17:13:29.845Z 12 240-259 81 81 81 139 139 139 F F F
91502024-02-03T17:13:29.845Z 13 260-279 78 78 78 133 133 133 F F F
91512024-02-03T17:13:29.845Z 14 280-299 76 76 76 128 128 128 F F F
91522024-02-03T17:13:29.845Z 15 300-319 76 76 76 129 129 129 F F F
91532024-02-03T17:13:29.845Z 16 320-339 82 82 82 140 140 140 F F F
91542024-02-03T17:13:29.845Z 17 340-359 82 82 82 140 140 140 F F F
91552024-02-03T17:13:29.845Z 18 360-379 82 82 82 140 140 140 F F F
91562024-02-03T17:13:29.846Z 19 380-399 79 79 79 135 135 135 F F F
91572024-02-03T17:13:29.846Z 20 400-419 78 78 78 133 133 133 F F F
91582024-02-03T17:13:29.846Z 21 420-439 79 79 79 135 135 135 F F F
91592024-02-03T17:13:29.846Z 22 440-459 83 83 83 144 144 144 F F F
91602024-02-03T17:13:29.846Z 23 460-479 83 83 83 144 144 144 F F F
91612024-02-03T17:13:29.846Z 24 480-499 81 81 81 139 139 139 F F F
91622024-02-03T17:13:29.846Z 25 500-519 82 82 82 141 141 141 F F F
91632024-02-03T17:13:29.846Z 26 520-539 80 80 80 137 137 137 F F F
91642024-02-03T17:13:29.846Z 27 540-559 82 82 82 143 143 143 F F F
91652024-02-03T17:13:29.846Z 28 560-579 82 82 82 142 142 142 F F F
91662024-02-03T17:13:29.846Z 29 580-599 82 82 82 142 142 142 F F F
91672024-02-03T17:13:29.846ZMax gen: 83, Max flush: 144
91682024-02-03T17:13:29.851Z
91692024-02-03T17:13:29.851ZBegin loop 42
91702024-02-03T17:13:29.851ZDownstairs to restart: 1
91712024-02-03T17:13:29.885Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.433748558Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468}
91722024-02-03T17:13:29.905Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.43428887Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468}
91732024-02-03T17:13:29.905Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
91742024-02-03T17:13:29.905Z,"time":"2024-02-03T17:13:29.434382859Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468}
91752024-02-03T17:13:29.905Z{"msg":"Crucible 2bd5ba65-bbcd-4370-816e-4be7eb80740d has session id: 005751df-c297-4c1f-abfc-340ba90c1c89","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.434564283Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
91762024-02-03T17:13:29.905Z{"msg":"Upstairs opts: Upstairs UUID: 2bd5ba65-bbcd-4370-816e-4be7eb80740d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.434634267Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
91772024-02-03T17:13:29.905Z{"msg":"Crucible stats registered with UUID: 2bd5ba65-bbcd-4370-816e-4be7eb80740d","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.434696351Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
91782024-02-03T17:13:29.905Z{"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.434721122Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468}
91792024-02-03T17:13:29.905Z{"msg":"2bd5ba65-bbcd-4370-816e-4be7eb80740d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.434897684Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
91802024-02-03T17:13:29.905Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.434967627Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"client":"0","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
91812024-02-03T17:13:29.905Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.434997523Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"client":"0","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
91822024-02-03T17:13:29.905Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.435043181Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"client":"1","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
91832024-02-03T17:13:29.905Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.435080425Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"client":"1","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
91842024-02-03T17:13:29.905Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.435143508Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"client":"2","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
91852024-02-03T17:13:29.905Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.435170612Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"client":"2","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
91862024-02-03T17:13:29.905Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.435192746Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"":"io task","client":"2","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
91872024-02-03T17:13:29.905Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.435230861Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"":"io task","client":"0","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
91882024-02-03T17:13:29.905Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.435268828Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"":"io task","client":"1","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
91892024-02-03T17:13:29.905Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.435535717Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"":"io task","client":"0","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
91902024-02-03T17:13:29.905Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.435743583Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"":"io task","client":"1","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
91912024-02-03T17:13:29.905Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.435793866Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"":"io task","client":"2","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
91922024-02-03T17:13:29.905Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.436760021Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"client":"0","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
91932024-02-03T17:13:29.906Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.436829116Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"client":"0","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
91942024-02-03T17:13:29.906Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.43690632Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"client":"2","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
91952024-02-03T17:13:29.906Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.436938343Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"client":"2","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
91962024-02-03T17:13:29.906Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.437407184Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"client":"1","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
91972024-02-03T17:13:29.906Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.437495089Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"client":"1","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
91982024-02-03T17:13:29.906Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.437796756Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"client":"2","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
91992024-02-03T17:13:29.906Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.438190263Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"client":"0","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92002024-02-03T17:13:29.906Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.438501008Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"client":"1","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92012024-02-03T17:13:29.906Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.438596532Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"client":"2","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92022024-02-03T17:13:29.906Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.438632551Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"client":"2","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92032024-02-03T17:13:29.906Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.438656382Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92042024-02-03T17:13:29.906Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.438949245Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"client":"0","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92052024-02-03T17:13:29.906Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.439043715Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"client":"0","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92062024-02-03T17:13:29.906Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.439091755Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92072024-02-03T17:13:29.906Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.439486189Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"client":"1","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92082024-02-03T17:13:29.906Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.439644175Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"client":"1","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92092024-02-03T17:13:29.906Z{"msg":"[0]R flush_numbers[0..12]: [137, 137, 130, 131, 144, 143, 143, 130, 137, 137, 144, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.439716844Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"":"downstairs","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92102024-02-03T17:13:29.906Z{"msg":"[0]R generation[0..12]: [80, 80, 77, 78, 83, 82, 82, 77, 80, 80, 83, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.439746889Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"":"downstairs","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92112024-02-03T17:13:29.906Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.439797445Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"":"downstairs","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92122024-02-03T17:13:29.906Z{"msg":Wait for a query_work_queue command to finish before sending IO
92132024-02-03T17:13:29.906Z"[1]R flush_numbers[0..12]: [137, 137, 130, 131, 144, 143, 143, 130, 137, 137, 144, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.439827409Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"":"downstairs","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92142024-02-03T17:13:29.906Z{"msg":"[1]R generation[0..12]: [80, 80, 77, 78, 83, 82, 82, 77, 80, 80, 83, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.439850387Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"":"downstairs","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92152024-02-03T17:13:29.906Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.439911736Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"":"downstairs","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92162024-02-03T17:13:29.906Z{"msg":"[2]R flush_numbers[0..12]: [137, 137, 130, 131, 144, 143, 143, 130, 137, 137, 144, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.439980817Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"":"downstairs","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92172024-02-03T17:13:29.907Z{"msg":"[2]R generation[0..12]: [80, 80, 77, 78, 83, 82, 82, 77, 80, 80, 83, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.440007838Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"":"downstairs","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92182024-02-03T17:13:29.907Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.440036814Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"":"downstairs","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92192024-02-03T17:13:29.907Z{"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.440106605Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"":"downstairs","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92202024-02-03T17:13:29.907Z{"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.440136043Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"":"downstairs","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92212024-02-03T17:13:29.907Z{"msg":"Next flush: 145","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.440201637Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"":"downstairs","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92222024-02-03T17:13:29.907Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.440230935Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"":"downstairs","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92232024-02-03T17:13:29.907Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.440253507Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92242024-02-03T17:13:29.907Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.440276814Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"client":"0","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92252024-02-03T17:13:29.907Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.440347647Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"client":"1","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92262024-02-03T17:13:29.907Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.440376157Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"client":"2","":"downstairs","session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92272024-02-03T17:13:29.907Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.440442602Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92282024-02-03T17:13:29.907Z{"msg":"Set Downstairs and Upstairs active after reconciliation","Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
92292024-02-03T17:13:29.907Zv":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.440550452Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92302024-02-03T17:13:29.907Z{"msg":"2bd5ba65-bbcd-4370-816e-4be7eb80740d is now active with session: 005751df-c297-4c1f-abfc-340ba90c1c89","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.440628708Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92312024-02-03T17:13:29.907Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.440656099Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468,"session_id":"005751df-c297-4c1f-abfc-340ba90c1c89"}
92322024-02-03T17:13:29.907Z{"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:29.440703388Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468}
92332024-02-03T17:13:29.907ZLoading write count information from file "/tmp/test_repair_verify.data"
92342024-02-03T17:13:29.907ZRun Repair workload
92352024-02-03T17:13:29.907Z01/30 Write block 405 len 12288 data: 6 7 7
92362024-02-03T17:13:29.907Z02/30 Write block 571 len 4096 data: 7
92372024-02-03T17:13:29.907Z03/30 Read block 380 len 40960
92382024-02-03T17:13:29.907Z04/30 Read block 561 len 32768
92392024-02-03T17:13:29.907Z05/30 Write block 264 len 8192 data: 11 12
92402024-02-03T17:13:29.911Z06/30 Write block 392 len 20480 data: 12 13 11 10 9
92412024-02-03T17:13:29.911Z07/30 Read block 580 len 12288
92422024-02-03T17:13:29.911Z08/30 Read block 304 len 40960
92432024-02-03T17:13:29.911Z09/30 Flush
92442024-02-03T17:13:29.911Z10/30 Write block 558 len 24576 data: 4 5 5 6 7 7
92452024-02-03T17:13:29.911Z11/30 Read block 212 len 4096
92462024-02-03T17:13:29.911Z12/30 Read block 531 len 24576
92472024-02-03T17:13:29.911Z13/30 Write block 363 len 4096 data: 6
92482024-02-03T17:13:29.912Z14/30 Read block 451 len 40960
92492024-02-03T17:13:29.912Z15/30 Read block 371 len 24576
92502024-02-03T17:13:29.912Z16/30 Write block 357 len 4096 data: 8
92512024-02-03T17:13:29.912Z17/30 Read block 248 len 28672
92522024-02-03T17:13:29.915Z18/30 Write block 315 len 28672 data: 8 8 9 6 9 8 9
92532024-02-03T17:13:29.915Z19/30 Read block 285 len 36864
92542024-02-03T17:13:29.918Z20/30 Write block 497 len 12288 data: 6 5 5
92552024-02-03T17:13:29.918Z21/30 Read block 264 len 28672
92562024-02-03T17:13:29.920Z22/30 Read block 104 len 28672
92572024-02-03T17:13:29.921Z23/30 Flush
92582024-02-03T17:13:29.924Z24/30 Flush
92592024-02-03T17:13:29.928Z25/30 Write block 408 len 20480 data: 5 5 6 6 8
92602024-02-03T17:13:29.928Z26/30 Write block 540 len 12288 data: 3 4 4
92612024-02-03T17:13:29.931Z27/30 Read block 519 len 20480
92622024-02-03T17:13:29.931Z28/30 Read block 387 len 16384
92632024-02-03T17:13:29.931Z29/30 Read block 383 len 24576
92642024-02-03T17:13:29.935Z30/30 Write block 286 len 20480 data: 8 7 5 6 7
92652024-02-03T17:13:29.935Z----------------------------------------------------------------
92662024-02-03T17:13:29.935Z Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:30
92672024-02-03T17:13:29.935ZStates: Active Active Active
92682024-02-03T17:13:29.935ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
92692024-02-03T17:13:29.935Z 1 Acked 1000 Write 3 Done Done Done false
92702024-02-03T17:13:29.935Z 2 Acked 1001 Write 1 Done Done Done false
92712024-02-03T17:13:29.935Z 3 Acked 1002 Read 10 Done Sent Done false
92722024-02-03T17:13:29.935Z 4 Acked 1003 Read 8 Done Sent Done false
92732024-02-03T17:13:29.935Z 5 Acked 1004 Write 2 Done Sent Done false
92742024-02-03T17:13:29.935Z 6 Acked 1005 Write 5 Done Sent Done false
92752024-02-03T17:13:29.935Z 7 Acked 1006 Read 3 Done Sent Done false
92762024-02-03T17:13:29.935Z 8 Acked 1007 Read 10 Done Sent Done false
92772024-02-03T17:13:29.935Z 9 Acked 1008 Flush 0 Done Sent Done false
92782024-02-03T17:13:29.935Z 10 Acked 1009 Write 6 Done Sent Done false
92792024-02-03T17:13:29.935Z 11 Acked 1010 Read 1 Done Sent Done false
92802024-02-03T17:13:29.936Z 12 Acked 1011 Read 6 Done Sent Done false
92812024-02-03T17:13:29.936Z 13 Acked 1012 Write 1 Done Sent Done false
92822024-02-03T17:13:29.936Z 14 Acked 1013 Read 10 Done Sent Done false
92832024-02-03T17:13:29.936Z 15 Acked 1014 Read 6 Done Sent Done false
92842024-02-03T17:13:29.936Z 16 Acked 1015 Write 1 Done Sent Done false
92852024-02-03T17:13:29.936Z 17 Acked 1016 Read 7 Done Sent Done false
92862024-02-03T17:13:29.936Z 18 Acked 1017 Write 7 Done Sent Done false
92872024-02-03T17:13:29.936Z 19 Acked 1018 Read 9 Done Sent Done false
92882024-02-03T17:13:29.936Z 20 Acked 1019 Write 3 Done Sent Done false
92892024-02-03T17:13:29.936Z 21 Acked 1020 Read 7 Done Sent Done false
92902024-02-03T17:13:29.936Z 22 Acked 1021 Read 7 Done Sent Done false
92912024-02-03T17:13:29.936Z 23 Acked 1022 Flush 0 Done Sent Done false
92922024-02-03T17:13:29.936Z 24 Acked 1023 Flush 0 Done Sent Done false
92932024-02-03T17:13:29.936Z 25 Acked 1024 Write 5 Done Sent Done false
92942024-02-03T17:13:29.936Z 26 Acked 1025 Write 3 Done Sent Done false
92952024-02-03T17:13:29.936Z 27 Acked 1026 Read 5 Done Sent Done false
92962024-02-03T17:13:29.936Z 28 Acked 1027 Read 4 Done Sent Done false
92972024-02-03T17:13:29.936Z 29 Acked 1028 Read 6 Done Sent Done false
92982024-02-03T17:13:29.936Z 30 Acked 1029 Write 5 Sent Sent Sent false
92992024-02-03T17:13:29.936Z STATES DS:0 DS:1 DS:2 TOTAL
93002024-02-03T17:13:29.936Z New 0 0 0 0
93012024-02-03T17:13:29.936Z Sent 1 28 1 30
93022024-02-03T17:13:29.936Z Done 29 2 29 60
93032024-02-03T17:13:29.936Z Skipped 0 0 0 0
93042024-02-03T17:13:29.937Z Error 0 0 0 0
93052024-02-03T17:13:29.937ZLast Flush: 1023 0 1023
93062024-02-03T17:13:29.937ZDownstairs last five completed:
93072024-02-03T17:13:29.937ZUpstairs last five completed: 30 29 28 27 26
93082024-02-03T17:13:29.937Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:29.490427188Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1468}
93092024-02-03T17:13:29.937ZWrote out file "/tmp/test_repair_verify.data"
93102024-02-03T17:13:29.940Z
93112024-02-03T17:13:29.946ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
93122024-02-03T17:13:29.971ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
93132024-02-03T17:13:29.971Z 0 000-019 80 80 80 137 137 137 F F F
93142024-02-03T17:13:29.971Z 1 020-039 80 80 80 137 137 137 F F F
93152024-02-03T17:13:29.971Z 2 040-059 77 77 77 130 130 130 F F F
93162024-02-03T17:13:29.972Z 3 060-079 78 78 78 131 131 131 F F F
93172024-02-03T17:13:29.972Z 4 080-099 83 83 83 144 144 144 F F F
93182024-02-03T17:13:29.975Z 5 100-119 82 82 82 143 143 143 F F F
93192024-02-03T17:13:29.975Z 6 120-139 82 82 82 143 143 143 F F F
93202024-02-03T17:13:29.975Z 7 140-159 77 77 77 130 130 130 F F F
93212024-02-03T17:13:29.975Z 8 160-179 80 80 80 137 137 137 F F F
93222024-02-03T17:13:29.976Z 9 180-199 80 80 80 137 137 137 F F F
93232024-02-03T17:13:29.976Z 10 200-219 83 83 83 144 144 144 F F F
93242024-02-03T17:13:29.976Z 11 220-239 82 82 82 141 141 141 F F F
93252024-02-03T17:13:29.976Z 12 240-259 81 81 81 139 139 139 F F F
93262024-02-03T17:13:29.976Z 13 260-279 84 78 84 145 133 145 F F F <---
93272024-02-03T17:13:29.976Z 14 280-299 76 76 76 128 128 128 T F T <---
93282024-02-03T17:13:29.976Z 15 300-319 84 76 84 146 129 146 F F F <---
93292024-02-03T17:13:29.976Z 16 320-339 84 82 84 146 140 146 F F F <---
93302024-02-03T17:13:29.976Z 17 340-359 84 82 84 146 140 146 F F F <---
93312024-02-03T17:13:29.976Z 18 360-379 84 82 84 146 140 146 F F F <---
93322024-02-03T17:13:29.976Z 19 380-399 84 79 84 145 135 145 F F F <---
93332024-02-03T17:13:29.976Z 20 400-419 84 78 84 145 133 145 T T T <---
93342024-02-03T17:13:29.976Z 21 420-439 79 79 79 135 135 135 F F F
93352024-02-03T17:13:29.976Z 22 440-459 83 83 83 144 144 144 F F F
93362024-02-03T17:13:29.976Z 23 460-479 83 83 83 144 144 144 F F F
93372024-02-03T17:13:29.976Z 24 480-499 84 81 84 146 139 146 F F F <---
93382024-02-03T17:13:29.976Z 25 500-519 82 82 82 141 141 141 F F F
93392024-02-03T17:13:29.976Z 26 520-539 80 80 80 137 137 137 F F F
93402024-02-03T17:13:29.976Z 27 540-559 84 82 84 146 143 146 T F T <---
93412024-02-03T17:13:29.976Z 28 560-579 84 82 84 146 142 146 F T F <---
93422024-02-03T17:13:29.976Z 29 580-599 82 82 82 142 142 142 F F F
93432024-02-03T17:13:29.976ZMax gen: 84, Max flush: 146
93442024-02-03T17:13:29.976ZError: Difference in extent metadata found!
93452024-02-03T17:13:29.980ZA Difference in extent metadata is expected here
93462024-02-03T17:13:29.980ZOn loop 42
93472024-02-03T17:13:29.980Z
93482024-02-03T17:13:30.035ZVerifying data now
93492024-02-03T17:13:35.974ZNo change after verify
93502024-02-03T17:13:35.974ZLoop: 42 Downstairs dump after verify (and repair):
93512024-02-03T17:13:36.406ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
93522024-02-03T17:13:36.406Z 0 000-019 80 80 80 137 137 137 F F F
93532024-02-03T17:13:36.406Z 1 020-039 80 80 80 137 137 137 F F F
93542024-02-03T17:13:36.407Z 2 040-059 77 77 77 130 130 130 F F F
93552024-02-03T17:13:36.407Z 3 060-079 78 78 78 131 131 131 F F F
93562024-02-03T17:13:36.407Z 4 080-099 83 83 83 144 144 144 F F F
93572024-02-03T17:13:36.407Z 5 100-119 82 82 82 143 143 143 F F F
93582024-02-03T17:13:36.839Z 6 120-139 82 82 82 143 143 143 F F F
93592024-02-03T17:13:36.839Z 7 140-159 77 77 77 130 130 130 F F F
93602024-02-03T17:13:36.839Z 8 160-179 80 80 80 137 137 137 F F F
93612024-02-03T17:13:36.839Z 9 180-199 80 80 80 137 137 137 F F F
93622024-02-03T17:13:36.840Z 10 200-219 83 83 83 144 144 144 F F F
93632024-02-03T17:13:36.840Z 11 220-239 82 82 82 141 141 141 F F F
93642024-02-03T17:13:36.840Z 12 240-259 81 81 81 139 139 139 F F F
93652024-02-03T17:13:36.840Z 13 260-279 84 84 84 145 145 145 F F F
93662024-02-03T17:13:36.840Z 14 280-299 85 85 85 147 147 147 F F F
93672024-02-03T17:13:36.840Z 15 300-319 84 84 84 146 146 146 F F F
93682024-02-03T17:13:36.840Z 16 320-339 84 84 84 146 146 146 F F F
93692024-02-03T17:13:36.840Z 17 340-359 84 84 84 146 146 146 F F F
93702024-02-03T17:13:36.840Z 18 360-379 84 84 84 146 146 146 F F F
93712024-02-03T17:13:36.840Z 19 380-399 84 84 84 145 145 145 F F F
93722024-02-03T17:13:36.840Z 20 400-419 85 85 85 147 147 147 F F F
93732024-02-03T17:13:36.840Z 21 420-439 79 79 79 135 135 135 F F F
93742024-02-03T17:13:36.840Z 22 440-459 83 83 83 144 144 144 F F F
93752024-02-03T17:13:36.841Z 23 460-479 83 83 83 144 144 144 F F F
93762024-02-03T17:13:36.841Z 24 480-499 84 84 84 146 146 146 F F F
93772024-02-03T17:13:36.841Z 25 500-519 82 82 82 141 141 141 F F F
93782024-02-03T17:13:36.841Z 26 520-539 80 80 80 137 137 137 F F F
93792024-02-03T17:13:36.841Z 27 540-559 85 85 85 147 147 147 F F F
93802024-02-03T17:13:36.841Z 28 560-579 84 84 84 146 146 146 F F F
93812024-02-03T17:13:36.841Z 29 580-599 82 82 82 142 142 142 F F F
93822024-02-03T17:13:36.841ZMax gen: 85, Max flush: 147
93832024-02-03T17:13:36.841Z
93842024-02-03T17:13:36.841ZBegin loop 43
93852024-02-03T17:13:36.841ZDownstairs to restart: 1
93862024-02-03T17:13:36.841Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.588901103Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476}
93872024-02-03T17:13:36.841Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.589525783Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476}
93882024-02-03T17:13:36.842Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.589603379Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476}
93892024-02-03T17:13:36.842Z{"msg":"Crucible 5b2491f4-bc13-49d5-995f-71fd58180889 has session id: 9c377ac9-8777-4896-aa0e-146d44cd834f","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
93902024-02-03T17:13:36.842Z2024-02-03T17:13:35.589681742Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
93912024-02-03T17:13:36.842Z{"msg":"Upstairs opts: Upstairs UUID: 5b2491f4-bc13-49d5-995f-71fd58180889, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.589783916Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
93922024-02-03T17:13:36.842Z{"msg":"Crucible stats registered with UUID: 5b2491f4-bc13-49d5-995f-71fd58180889","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.589824899Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
93932024-02-03T17:13:36.842Z{"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.589878126Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476}
93942024-02-03T17:13:36.842Z{"msg":"5b2491f4-bc13-49d5-995f-71fd58180889 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.589984226Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
93952024-02-03T17:13:36.842Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.590043929Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"client":"0","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
93962024-02-03T17:13:36.842Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.590070892Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"client":"0","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
93972024-02-03T17:13:36.842Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.590093214Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"client":"1","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
93982024-02-03T17:13:36.842Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.59015958Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"client":"1","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
93992024-02-03T17:13:36.842Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.590184805Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"client":"2","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94002024-02-03T17:13:36.842Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.590207429Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"client":"2","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94012024-02-03T17:13:36.842Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.590234291Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"":"io task","client":"0","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94022024-02-03T17:13:36.843Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.590276484Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"":"io task","client":"1","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94032024-02-03T17:13:36.843Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.59057082Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"":"io task","client":"2","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94042024-02-03T17:13:36.843Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.590630421Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"":"io task","client":"0","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94052024-02-03T17:13:36.843Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.590722941Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"":"io task","client":"2","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94062024-02-03T17:13:36.843Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.59076931Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"":"io task","client":"1","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94072024-02-03T17:13:36.843Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.592026382Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"client":"2","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94082024-02-03T17:13:36.843Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.592100543Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"client":"2","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94092024-02-03T17:13:36.843Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.592127809Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"client":"0","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94102024-02-03T17:13:36.843Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.592150526Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"client":"0","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94112024-02-03T17:13:36.843Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.592407483Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"client":"1","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94122024-02-03T17:13:36.843Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.592479708Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"client":"1","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94132024-02-03T17:13:36.843Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.593033036Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"client":"0","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94142024-02-03T17:13:36.843Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.593275216Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"client":"2","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94152024-02-03T17:13:36.843Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.593786099Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"client":"1","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94162024-02-03T17:13:36.843Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.594249172Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"client":"2","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94172024-02-03T17:13:36.844Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.594308429Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"client":"2","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94182024-02-03T17:13:36.844Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.594353476Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94192024-02-03T17:13:36.844Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.594432526Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"client":"0","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94202024-02-03T17:13:36.844Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.594513482Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"client":"0","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94212024-02-03T17:13:36.844Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.594558224Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94222024-02-03T17:13:36.844Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.594867513Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"client":"1","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94232024-02-03T17:13:36.844Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.59503651Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"client":"1","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94242024-02-03T17:13:36.844Z{"msg":"[0]R flush_numbers[0..12]: [137, 137, 130, 131, 144, 143, 143, 130, 137, 137, 144, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.595072193Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"":"downstairs","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94252024-02-03T17:13:36.844Z{"msg":"[0]R generation[0..12]: [80, 80, 77, 78, 83, 82, 82, 77, 80, 80, 83, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.595124189Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"":"downstairs","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94262024-02-03T17:13:36.844Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.595151735Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"":"downstairs","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94272024-02-03T17:13:36.844Z{"msg":"[1]R flush_numbers[0..12]: [137, 137, 130, 131, 144, 143, 143, 130, 137, 137, 144, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.595202457Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"":"downstairs","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94282024-02-03T17:13:36.844Z{"msg":"[1]R generation[0..12]: [80, 80, 77, 78, 83, 82, 82, 77, 80, 80, 83, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.595228036Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"":"downstairs","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94292024-02-03T17:13:36.844Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.595252168Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"":"downstairs","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94302024-02-03T17:13:36.844ZWait for a query_work_queue command to finish before sending IO
94312024-02-03T17:13:36.844Z{"msg":"[2]R flush_numbers[0..12]: [137, 137, 130, 131, 144, 143, 143, 130, 137, 137, 144, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.595305701Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"":"downstairs","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94322024-02-03T17:13:36.844Z{"msg":"[2]R generation[0..12]: [80, 80, 77, 78, 83, 82, 82, 77, 80, 80, 83, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.595331338Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"":"downstairs","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94332024-02-03T17:13:36.844Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.595355143Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"":"downstairs","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94342024-02-03T17:13:36.844Z{"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.595377087Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"":"downstairs","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94352024-02-03T17:13:36.844Z{"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.595399196Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"":"downstairs","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94362024-02-03T17:13:36.844Z{"msg":"Next flush: 148","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.59542065Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"":"downstairs","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94372024-02-03T17:13:36.844Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.595442255Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"":"downstairs","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94382024-02-03T17:13:36.844Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.595464217Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94392024-02-03T17:13:36.845Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.595484076Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"client":"0","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94402024-02-03T17:13:36.845Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.595506703Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"client":"1","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94412024-02-03T17:13:36.845Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.595573495Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"client":"2","":"downstairs","session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94422024-02-03T17:13:36.845Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.595598623Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94432024-02-03T17:13:36.845Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.595619565Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94442024-02-03T17:13:36.845Z{"msg":"5b2491f4-bc13-49d5-995f-71fd58180889 is now active with session: 9c377ac9-8777-4896-aa0e-146d44cd834f","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.595640874Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94452024-02-03T17:13:36.845Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.595685786Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476,"session_id":"9c377ac9-8777-4896-aa0e-146d44cd834f"}
94462024-02-03T17:13:36.845Z{"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:35.595708943Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476}
94472024-02-03T17:13:36.845ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
94482024-02-03T17:13:36.845ZLoading write count information from file "/tmp/test_repair_verify.data"
94492024-02-03T17:13:36.845ZRun Repair workload
94502024-02-03T17:13:36.845Z01/30 Write block 385 len 8192 data: 5 6
94512024-02-03T17:13:36.845Z02/30 Read block 278 len 20480
94522024-02-03T17:13:36.845Z03/30 Read block 521 len 40960
94532024-02-03T17:13:36.845Z04/30 Read block 388 len 32768
94542024-02-03T17:13:36.845Z05/30 Read block 495 len 24576
94552024-02-03T17:13:36.845Z06/30 Read block 152 len 24576
94562024-02-03T17:13:36.845Z07/30 Write block 76 len 16384 data: 9 8 7 8
94572024-02-03T17:13:36.845Z08/30 Flush
94582024-02-03T17:13:36.860Z09/30 Write block 18 len 24576 data: 8 10 8 11 12 12
94592024-02-03T17:13:36.860Z10/30 Write block 396 len 20480 data: 10 7 9 10 8
94602024-02-03T17:13:36.860Z11/30 Read block 99 len 36864
94612024-02-03T17:13:36.860Z12/30 Read block 526 len 28672
94622024-02-03T17:13:36.860Z13/30 Write block 373 len 20480 data: 7 7 6 7 8
94632024-02-03T17:13:36.860Z14/30 Read block 71 len 24576
94642024-02-03T17:13:36.860Z15/30 Write block 42 len 24576 data: 6 4 5 6 5 5
94652024-02-03T17:13:36.860Z16/30 Read block 219 len 36864
94662024-02-03T17:13:36.860Z17/30 Read block 276 len 24576
94672024-02-03T17:13:36.860Z18/30 Read block 445 len 32768
94682024-02-03T17:13:36.860Z19/30 Flush
94692024-02-03T17:13:36.860Z20/30 Write block 428 len 20480 data: 7 10 10 8 9
94702024-02-03T17:13:36.860Z21/30 Write block 507 len 8192 data: 9 9
94712024-02-03T17:13:36.860Z22/30 Flush
94722024-02-03T17:13:36.861Z23/30 Write block 283 len 28672 data: 14 10 9 9 8 6 7
94732024-02-03T17:13:36.861Z24/30 Flush
94742024-02-03T17:13:36.861Z25/30 Write block 175 len 16384 data: 6 7 8 6
94752024-02-03T17:13:36.861Z26/30 Flush
94762024-02-03T17:13:36.861Z27/30 Write block 68 len 4096 data: 6
94772024-02-03T17:13:36.861Z28/30 Read block 271 len 8192
94782024-02-03T17:13:36.861Z29/30 Write block 402 len 40960 data: 6 6 6 7 8 8 6 6 7 7
94792024-02-03T17:13:36.861Z30/30 Read block 345 len 28672
94802024-02-03T17:13:36.861Z----------------------------------------------------------------
94812024-02-03T17:13:36.861Z Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:30
94822024-02-03T17:13:36.861ZStates: Active Active Active
94832024-02-03T17:13:36.861ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
94842024-02-03T17:13:36.861Z 1 Acked 1000 Write 2 Done Done Done false
94852024-02-03T17:13:36.861Z 2 Acked 1001 Read 5 Done Done Done false
94862024-02-03T17:13:36.861Z 3 Acked 1002 Read 10 Done Done Done false
94872024-02-03T17:13:36.861Z 4 Acked 1003 Read 8 Done Sent Done false
94882024-02-03T17:13:36.861Z 5 Acked 1004 Read 6 Done Sent Done false
94892024-02-03T17:13:36.861Z 6 Acked 1005 Read 6 Done Sent Done false
94902024-02-03T17:13:36.861Z 7 Acked 1006 Write 4 Done Sent Done false
94912024-02-03T17:13:36.862Z 8 Acked 1007 Flush 0 Done Sent Done false
94922024-02-03T17:13:36.862Z 9 Acked 1008 Write 6 Done Sent Done false
94932024-02-03T17:13:36.862Z 10 Acked 1009 Write 5 Done Sent Done false
94942024-02-03T17:13:36.862Z 11 Acked 1010 Read 9 Done Sent Done false
94952024-02-03T17:13:36.862Z 12 Acked 1011 Read 7 Done Sent Done false
94962024-02-03T17:13:36.862Z 13 Acked 1012 Write 5 Done Sent Done false
94972024-02-03T17:13:36.862Z 14 Acked 1013 Read 6 Done Sent Done false
94982024-02-03T17:13:36.862Z 15 Acked 1014 Write 6 Done Sent Done false
94992024-02-03T17:13:36.862Z 16 Acked 1015 Read 9 Done Sent Done false
95002024-02-03T17:13:36.862Z 17 Acked 1016 Read 6 Done Sent Done false
95012024-02-03T17:13:36.862Z 18 Acked 1017 Read 8 Done Sent Done false
95022024-02-03T17:13:36.862Z 19 Acked 1018 Flush 0 Done Sent Done false
95032024-02-03T17:13:36.862Z 20 Acked 1019 Write 5 Done Sent Done false
95042024-02-03T17:13:36.862Z 21 Acked 1020 Write 2 Done Sent Done false
95052024-02-03T17:13:36.862Z 22 Acked 1021 Flush 0 Done Sent Done false
95062024-02-03T17:13:36.862Z 23 Acked 1022 Write 7 Done Sent Done false
95072024-02-03T17:13:36.862Z 24 Acked 1023 Flush 0 Done Sent Done false
95082024-02-03T17:13:36.862Z 25 Acked 1024 Write 4 Done Sent Done false
95092024-02-03T17:13:36.863Z 26 Acked 1025 Flush 0 Done Sent Done false
95102024-02-03T17:13:36.863Z 27 Acked 1026 Write 1 Done Sent Done false
95112024-02-03T17:13:36.863Z 28 Acked 1027 Read 2 Done Sent Done false
95122024-02-03T17:13:36.863Z 29 Acked 1028 Write 10 Sent Sent Done false
95132024-02-03T17:13:36.863Z 30 Acked 1029 Read 7 Sent Sent Done false
95142024-02-03T17:13:36.863Z STATES DS:0 DS:1 DS:2 TOTAL
95152024-02-03T17:13:36.863Z New 0 0 0 0
95162024-02-03T17:13:36.863Z Sent 2 27 0 29
95172024-02-03T17:13:36.863Z Done 28 3 30 61
95182024-02-03T17:13:36.863Z Skipped 0 0 0 0
95192024-02-03T17:13:36.863Z Error 0 0 0 0
95202024-02-03T17:13:36.863ZLast Flush: 1025 0 1025
95212024-02-03T17:13:36.863ZDownstairs last five completed:
95222024-02-03T17:13:36.863ZUpstairs last five completed: 30 29 28 27 26
95232024-02-03T17:13:36.863Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:35.652026801Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1476}
95242024-02-03T17:13:36.863ZWrote out file "/tmp/test_repair_verify.data"
95252024-02-03T17:13:36.863Z
95262024-02-03T17:13:36.863ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
95272024-02-03T17:13:36.863ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
95282024-02-03T17:13:36.863Z 0 000-019 86 80 86 149 137 149 F F F <---
95292024-02-03T17:13:36.863Z 1 020-039 86 80 86 149 137 149 F F F <---
95302024-02-03T17:13:36.863Z 2 040-059 86 77 86 149 130 149 F F F <---
95312024-02-03T17:13:36.863Z 3 060-079 86 78 86 148 131 148 T F T <---
95322024-02-03T17:13:36.863Z 4 080-099 83 83 83 144 144 144 F F F
95332024-02-03T17:13:36.864Z 5 100-119 82 82 82 143 143 143 F F F
95342024-02-03T17:13:36.864Z 6 120-139 82 82 82 143 143 143 F F F
95352024-02-03T17:13:36.864Z 7 140-159 77 77 77 130 130 130 F F F
95362024-02-03T17:13:36.864Z 8 160-179 86 80 86 152 137 152 F F F <---
95372024-02-03T17:13:36.864Z 9 180-199 80 80 80 137 137 137 F F F
95382024-02-03T17:13:36.864Z 10 200-219 83 83 83 144 144 144 F F F
95392024-02-03T17:13:36.864Z 11 220-239 82 82 82 141 141 141 F F F
95402024-02-03T17:13:36.864Z 12 240-259 81 81 81 139 139 139 F F F
95412024-02-03T17:13:36.864Z 13 260-279 84 84 84 145 145 145 F F F
95422024-02-03T17:13:36.864Z 14 280-299 86 85 86 151 147 151 F F F <---
95432024-02-03T17:13:36.864Z 15 300-319 84 84 84 146 146 146 F F F
95442024-02-03T17:13:36.864Z 16 320-339 84 84 84 146 146 146 F F F
95452024-02-03T17:13:36.864Z 17 340-359 84 84 84 146 146 146 F F F
95462024-02-03T17:13:36.864Z 18 360-379 86 84 86 149 146 149 F F F <---
95472024-02-03T17:13:36.864Z 19 380-399 86 84 86 149 145 149 F T F <---
95482024-02-03T17:13:36.864Z 20 400-419 86 85 86 149 147 149 T F T <---
95492024-02-03T17:13:36.864Z 21 420-439 86 79 86 150 135 150 F F F <---
95502024-02-03T17:13:36.864Z 22 440-459 83 83 83 144 144 144 F F F
95512024-02-03T17:13:36.864Z 23 460-479 83 83 83 144 144 144 F F F
95522024-02-03T17:13:36.864Z 24 480-499 84 84 84 146 146 146 F F F
95532024-02-03T17:13:36.864Z 25 500-519 86 82 86 150 141 150 F F F <---
95542024-02-03T17:13:36.864Z 26 520-539 80 80 80 137 137 137 F F F
95552024-02-03T17:13:36.864Z 27 540-559 85 85 85 147 147 147 F F F
95562024-02-03T17:13:36.864Z 28 560-579 84 84 84 146 146 146 F F F
95572024-02-03T17:13:36.865Z 29 580-599 82 82 82 142 142 142 F F F
95582024-02-03T17:13:36.871ZMax gen: 86, Max flush: 152
95592024-02-03T17:13:36.871ZError: Difference in extent metadata found!
95602024-02-03T17:13:36.872ZA Difference in extent metadata is expected here
95612024-02-03T17:13:36.872ZOn loop 43
95622024-02-03T17:13:36.872Z
95632024-02-03T17:13:36.872ZVerifying data now
95642024-02-03T17:13:41.126ZNo change after verify
95652024-02-03T17:13:41.126ZLoop: 43 Downstairs dump after verify (and repair):
95662024-02-03T17:13:41.152ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
95672024-02-03T17:13:41.152Z 0 000-019 86 86 86 149 149 149 F F F
95682024-02-03T17:13:41.152Z 1 020-039 86 86 86 149 149 149 F F F
95692024-02-03T17:13:41.152Z 2 040-059 86 86 86 149 149 149 F F F
95702024-02-03T17:13:41.152Z 3 060-079 87 87 87 153 153 153 F F F
95712024-02-03T17:13:41.162Z 4 080-099 83 83 83 144 144 144 F F F
95722024-02-03T17:13:41.163Z 5 100-119 82 82 82 143 143 143 F F F
95732024-02-03T17:13:41.163Z 6 120-139 82 82 82 143 143 143 F F F
95742024-02-03T17:13:41.163Z 7 140-159 77 77 77 130 130 130 F F F
95752024-02-03T17:13:41.163Z 8 160-179 86 86 86 152 152 152 F F F
95762024-02-03T17:13:41.163Z 9 180-199 80 80 80 137 137 137 F F F
95772024-02-03T17:13:41.163Z 10 200-219 83 83 83 144 144 144 F F F
95782024-02-03T17:13:41.164Z 11 220-239 82 82 82 141 141 141 F F F
95792024-02-03T17:13:41.164Z 12 240-259 81 81 81 139 139 139 F F F
95802024-02-03T17:13:41.164Z 13 260-279 84 84 84 145 145 145 F F F
95812024-02-03T17:13:41.164Z 14 280-299 86 86 86 151 151 151 F F F
95822024-02-03T17:13:41.164Z 15 300-319 84 84 84 146 146 146 F F F
95832024-02-03T17:13:41.165Z 16 320-339 84 84 84 146 146 146 F F F
95842024-02-03T17:13:41.165Z 17 340-359 84 84 84 146 146 146 F F F
95852024-02-03T17:13:41.165Z 18 360-379 86 86 86 149 149 149 F F F
95862024-02-03T17:13:41.165Z 19 380-399 86 86 86 149 149 149 F F F
95872024-02-03T17:13:41.165Z 20 400-419 87 87 87 153 153 153 F F F
95882024-02-03T17:13:41.165Z 21 420-439 86 86 86 150 150 150 F F F
95892024-02-03T17:13:41.166Z 22 440-459 83 83 83 144 144 144 F F F
95902024-02-03T17:13:41.166Z 23 460-479 83 83 83 144 144 144 F F F
95912024-02-03T17:13:41.166Z 24 480-499 84 84 84 146 146 146 F F F
95922024-02-03T17:13:41.166Z 25 500-519 86 86 86 150 150 150 F F F
95932024-02-03T17:13:41.166Z 26 520-539 80 80 80 137 137 137 F F F
95942024-02-03T17:13:41.166Z 27 540-559 85 85 85 147 147 147 F F F
95952024-02-03T17:13:41.167Z 28 560-579 84 84 84 146 146 146 F F F
95962024-02-03T17:13:41.167Z 29 580-599 82 82 82 142 142 142 F F F
95972024-02-03T17:13:41.167ZMax gen: 87, Max flush: 153
95982024-02-03T17:13:41.167Z
95992024-02-03T17:13:41.167ZBegin loop 44
96002024-02-03T17:13:41.167ZDownstairs to restart: 1
96012024-02-03T17:13:41.180Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.736953616Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484}
96022024-02-03T17:13:41.180Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.73752888Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484}
96032024-02-03T17:13:41.186Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.737602669Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484}
96042024-02-03T17:13:41.186Z{"msg":"Crucible 91749de3-ad30-4538-8c5d-a25645a825b2 has session id: 9bc93ad9-92ea-40bb-8897-e1d6a96fcc26","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.737647311Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96052024-02-03T17:13:41.186Z{"msg":"Upstairs opts: Upstairs UUID: 91749de3-ad30-4538-8c5d-a25645a825b2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.737708567Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96062024-02-03T17:13:41.186ZCrucible runtime is spawned
96072024-02-03T17:13:41.187Z{"msg":"Crucible stats registered with UUID: 91749de3-ad30-4538-8c5d-a25645a825b2","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.737820756Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96082024-02-03T17:13:41.187Z{"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.737919551Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484}
96092024-02-03T17:13:41.187Z{"msg":"91749de3-ad30-4538-8c5d-a25645a825b2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.738168668Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96102024-02-03T17:13:41.187Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.738257888Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"client":"0","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96112024-02-03T17:13:41.187Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.738285873Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"client":"0","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96122024-02-03T17:13:41.187Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.738308611Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"client":"1","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96132024-02-03T17:13:41.188Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.738331523Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"client":"1","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96142024-02-03T17:13:41.188Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.738425763Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"client":"2","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96152024-02-03T17:13:41.188Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.738459417Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"client":"2","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96162024-02-03T17:13:41.188Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.738489848Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"":"io task","client":"0","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96172024-02-03T17:13:41.188Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.738517251Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"":"io task","client":"2","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96182024-02-03T17:13:41.188Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.738577336Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"":"io task","client":"1","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96192024-02-03T17:13:41.189Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.738924597Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"":"io task","client":"0","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96202024-02-03T17:13:41.189Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.739006788Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"":"io task","client":"2","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96212024-02-03T17:13:41.189Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.739125521Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"":"io task","client":"1","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96222024-02-03T17:13:41.189Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.740463022Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"client":"2","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96232024-02-03T17:13:41.189Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.740601947Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"client":"2","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96242024-02-03T17:13:41.189Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.740649949Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"client":"0","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96252024-02-03T17:13:41.190Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.740694639Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"client":"0","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96262024-02-03T17:13:41.190Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.741488531Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"client":"0","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96272024-02-03T17:13:41.190Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.741688504Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"client":"2","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96282024-02-03T17:13:41.190Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.741896504Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"client":"1","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96292024-02-03T17:13:41.196Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.741999771Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"client":"1","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96302024-02-03T17:13:41.196Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.742321469Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"client":"0","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96312024-02-03T17:13:41.196Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.742441996Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"client":"0","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96322024-02-03T17:13:41.196Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.742491913Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96332024-02-03T17:13:41.196Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.742529107Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"client":"2","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96342024-02-03T17:13:41.196Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.742590483Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"client":"2","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96352024-02-03T17:13:41.196Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.742633881Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96362024-02-03T17:13:41.196Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.74289563Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"client":"1","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96372024-02-03T17:13:41.196Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.743779915Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"client":"1","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96382024-02-03T17:13:41.196Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.743849115Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"client":"1","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96392024-02-03T17:13:41.196Z{"msg":"[0]R flush_numbers[0..12]: [149, 149, 149, 153, 144, 143, 143, 130, 152, 137, 144, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.743876777Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"":"downstairs","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96402024-02-03T17:13:41.196Z{"msg":"[0]R generation[0..12]: [86, 86, 86, 87, 83, 82, 82, 77, 86, 80, 83, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.743900487Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"":"downstairs","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96412024-02-03T17:13:41.196Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.743965713Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"":"downstairs","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96422024-02-03T17:13:41.196Z{"msg":"[1]R flush_numbers[0..12]: [149, 149, 149, 153, 144, 143, 143, 130, 152, 137, 144, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.743999169Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"":"Wait for a query_work_queue command to finish before sending IO
96432024-02-03T17:13:41.196Zdownstairs","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96442024-02-03T17:13:41.196Z{"msg":"[1]R generation[0..12]: [86, 86, 86, 87, 83, 82, 82, 77, 86, 80, 83, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.744130786Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"":"downstairs","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96452024-02-03T17:13:41.196Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.744162311Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"":"downstairs","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96462024-02-03T17:13:41.196Z{"msg":"[2]R flush_numbers[0..12]: [149, 149, 149, 153, 144, 143, 143, 130, 152, 137, 144, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.744219184Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"":"downstairs","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96472024-02-03T17:13:41.196Z{"msg":"[2]R generation[0..12]: [86, 86, 86, 87, 83, 82, 82, 77, 86, 80, 83, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.744246108Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"":"downstairs","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96482024-02-03T17:13:41.196Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.74427021Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"":"downstairs","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96492024-02-03T17:13:41.196Z{"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.744339848Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"":"downstairs","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96502024-02-03T17:13:41.196Z{"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.744370283Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"":"downstairs","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96512024-02-03T17:13:41.197Z{"msg":"Next flush: 154","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.744409576Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"":"downstairs","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96522024-02-03T17:13:41.197Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.744483868Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"":"downstairs","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96532024-02-03T17:13:41.197Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.744511589Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96542024-02-03T17:13:41.197Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.744531839Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"client":"0","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96552024-02-03T17:13:41.197Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.744598864Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"client":"1","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96562024-02-03T17:13:41.197Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.744643688Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"client":"2","":"downstairs","session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96572024-02-03T17:13:41.197Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.74471282Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96582024-02-03T17:13:41.197Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.744738842Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96592024-02-03T17:13:41.197Z{"msg":"91749de3-ad30-4538-8c5d-a25645a825b2 is now active with session: 9bc93ad9-92ea-40bb-8897-e1d6a96fcc26","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.744760901Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96602024-02-03T17:13:41.197Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.744828761Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484,"session_id":"9bc93ad9-92ea-40bb-8897-e1d6a96fcc26"}
96612024-02-03T17:13:41.197Z{"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:40.744862655Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484}
96622024-02-03T17:13:41.197ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
96632024-02-03T17:13:41.197ZLoading write count information from file "/tmp/test_repair_verify.data"
96642024-02-03T17:13:41.197ZRun Repair workload
96652024-02-03T17:13:41.197Z01/30 Write block 486 len 4096 data: 7
96662024-02-03T17:13:41.197Z02/30 Read block 328 len 16384
96672024-02-03T17:13:41.197Z03/30 Write block 542 len 16384 data: 5 4 4 6
96682024-02-03T17:13:41.197Z04/30 Write block 512 len 4096 data: 9
96692024-02-03T17:13:41.197Z05/30 Read block 292 len 24576
96702024-02-03T17:13:41.201Z06/30 Read block 543 len 32768
96712024-02-03T17:13:41.201Z07/30 Read block 371 len 40960
96722024-02-03T17:13:41.201Z08/30 Write block 56 len 8192 data: 7 6
96732024-02-03T17:13:41.201Z09/30 Read block 98 len 24576
96742024-02-03T17:13:41.204Z10/30 Read block 170 len 40960
96752024-02-03T17:13:41.204Z11/30 Read block 204 len 8192
96762024-02-03T17:13:41.207Z12/30 Write block 421 len 28672 data: 8 6 5 5 5 5 7
96772024-02-03T17:13:41.207Z13/30 Write block 60 len 16384 data: 6 4 6 4
96782024-02-03T17:13:41.210Z14/30 Write block 73 len 12288 data: 5 6 8
96792024-02-03T17:13:41.210Z15/30 Read block 117 len 20480
96802024-02-03T17:13:41.210Z16/30 Read block 448 len 28672
96812024-02-03T17:13:41.213Z17/30 Write block 258 len 16384 data: 7 6 7 8
96822024-02-03T17:13:41.216Z18/30 Write block 433 len 28672 data: 7 7 8 7 6 6 6
96832024-02-03T17:13:41.216Z19/30 Write block 573 len 8192 data: 7 7
96842024-02-03T17:13:41.216Z20/30 Read block 128 len 16384
96852024-02-03T17:13:41.219Z21/30 Write block 151 len 12288 data: 8 7 7
96862024-02-03T17:13:41.219Z22/30 Write block 295 len 16384 data: 9 9 8 7
96872024-02-03T17:13:41.219Z23/30 Read block 340 len 28672
96882024-02-03T17:13:41.222Z24/30 Read block 450 len 36864
96892024-02-03T17:13:41.225Z25/30 Read block 2 len 20480
96902024-02-03T17:13:41.225Z26/30 Write block 14 len 24576 data: 8 6 6 7 9 11
96912024-02-03T17:13:41.228Z27/30 Write block 479 len 28672 data: 10 10 9 9 8 7 6
96922024-02-03T17:13:41.228Z28/30 Read block 202 len 12288
96932024-02-03T17:13:41.231Z29/30 Read block 219 len 32768
96942024-02-03T17:13:41.231Z30/30 Write block 187 len 12288 data: 14 16 16
96952024-02-03T17:13:41.234Z----------------------------------------------------------------
96962024-02-03T17:13:41.234Z Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30
96972024-02-03T17:13:41.235ZStates: Active Active Active
96982024-02-03T17:13:41.235ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
96992024-02-03T17:13:41.235Z 1 Acked 1000 Write 1 Done Done Done false
97002024-02-03T17:13:41.235Z 2 Acked 1001 Read 4 Done Done Done false
97012024-02-03T17:13:41.235Z 3 Acked 1002 Write 4 Done Done Done false
97022024-02-03T17:13:41.235Z 4 Acked 1003 Write 1 Done Done Done false
97032024-02-03T17:13:41.235Z 5 Acked 1004 Read 6 Done Sent Done false
97042024-02-03T17:13:41.235Z 6 Acked 1005 Read 8 Done Sent Done false
97052024-02-03T17:13:41.235Z 7 Acked 1006 Read 10 Done Sent Done false
97062024-02-03T17:13:41.235Z 8 Acked 1007 Write 2 Done Sent Done false
97072024-02-03T17:13:41.235Z 9 Acked 1008 Read 6 Done Sent Done false
97082024-02-03T17:13:41.235Z 10 Acked 1009 Read 10 Done Sent Done false
97092024-02-03T17:13:41.235Z 11 Acked 1010 Read 2 Done Sent Done false
97102024-02-03T17:13:41.235Z 12 Acked 1011 Write 7 Done Sent Done false
97112024-02-03T17:13:41.235Z 13 Acked 1012 Write 4 Done Sent Done false
97122024-02-03T17:13:41.235Z 14 Acked 1013 Write 3 Done Sent Done false
97132024-02-03T17:13:41.235Z 15 Acked 1014 Read 5 Done Sent Done false
97142024-02-03T17:13:41.235Z 16 Acked 1015 Read 7 Done Sent Done false
97152024-02-03T17:13:41.235Z 17 Acked 1016 Write 4 Done Sent Done false
97162024-02-03T17:13:41.235Z 18 Acked 1017 Write 7 Done Sent Done false
97172024-02-03T17:13:41.235Z 19 Acked 1018 Write 2 Done Sent Done false
97182024-02-03T17:13:41.235Z 20 Acked 1019 Read 4 Done Sent Done false
97192024-02-03T17:13:41.235Z 21 Acked 1020 Write 3 Done Sent Done false
97202024-02-03T17:13:41.235Z 22 Acked 1021 Write 4 Done Sent Done false
97212024-02-03T17:13:41.236Z 23 Acked 1022 Read 7 Done Sent Done false
97222024-02-03T17:13:41.236Z 24 Acked 1023 Read 9 Done Sent Done false
97232024-02-03T17:13:41.236Z 25 Acked 1024 Read 5 Done Sent Done false
97242024-02-03T17:13:41.236Z 26 Acked 1025 Write 6 Done Sent Done false
97252024-02-03T17:13:41.236Z 27 Acked 1026 Write 7 Done Sent Done false
97262024-02-03T17:13:41.236Z 28 Acked 1027 Read 3 Done Sent Done false
97272024-02-03T17:13:41.236Z 29 Acked 1028 Read 8 Done Sent Done false
97282024-02-03T17:13:41.236Z 30 Acked 1029 Write 3 Sent Sent Sent false
97292024-02-03T17:13:41.236Z STATES DS:0 DS:1 DS:2 TOTAL
97302024-02-03T17:13:41.236Z New 0 0 0 0
97312024-02-03T17:13:41.236Z Sent 1 26 1 28
97322024-02-03T17:13:41.236Z Done 29 4 29 62
97332024-02-03T17:13:41.236Z Skipped 0 0 0 0
97342024-02-03T17:13:41.236Z Error 0 0 0 0
97352024-02-03T17:13:41.236ZLast Flush: 0 0 0
97362024-02-03T17:13:41.236ZDownstairs last five completed:
97372024-02-03T17:13:41.236ZUpstairs last five completed: 30 29 28 27 26
97382024-02-03T17:13:41.236Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:40.789678821Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1484}
97392024-02-03T17:13:41.236ZWrote out file "/tmp/test_repair_verify.data"
97402024-02-03T17:13:41.239Z
97412024-02-03T17:13:41.245ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
97422024-02-03T17:13:41.277ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
97432024-02-03T17:13:41.277Z 0 000-019 86 86 86 149 149 149 T F T <---
97442024-02-03T17:13:41.278Z 1 020-039 86 86 86 149 149 149 F F F
97452024-02-03T17:13:41.281Z 2 040-059 86 86 86 149 149 149 T F T <---
97462024-02-03T17:13:41.281Z 3 060-079 87 87 87 153 153 153 T F T <---
97472024-02-03T17:13:41.281Z 4 080-099 83 83 83 144 144 144 F F F
97482024-02-03T17:13:41.281Z 5 100-119 82 82 82 143 143 143 F F F
97492024-02-03T17:13:41.282Z 6 120-139 82 82 82 143 143 143 F F F
97502024-02-03T17:13:41.282Z 7 140-159 77 77 77 130 130 130 T F T <---
97512024-02-03T17:13:41.282Z 8 160-179 86 86 86 152 152 152 F F F
97522024-02-03T17:13:41.282Z 9 180-199 80 80 80 137 137 137 T F T <---
97532024-02-03T17:13:41.282Z 10 200-219 83 83 83 144 144 144 F F F
97542024-02-03T17:13:41.282Z 11 220-239 82 82 82 141 141 141 F F F
97552024-02-03T17:13:41.282Z 12 240-259 81 81 81 139 139 139 T F T <---
97562024-02-03T17:13:41.282Z 13 260-279 84 84 84 145 145 145 T F T <---
97572024-02-03T17:13:41.282Z 14 280-299 86 86 86 151 151 151 T F T <---
97582024-02-03T17:13:41.282Z 15 300-319 84 84 84 146 146 146 F F F
97592024-02-03T17:13:41.282Z 16 320-339 84 84 84 146 146 146 F F F
97602024-02-03T17:13:41.282Z 17 340-359 84 84 84 146 146 146 F F F
97612024-02-03T17:13:41.282Z 18 360-379 86 86 86 149 149 149 F F F
97622024-02-03T17:13:41.282Z 19 380-399 86 86 86 149 149 149 F F F
97632024-02-03T17:13:41.282Z 20 400-419 87 87 87 153 153 153 F F F
97642024-02-03T17:13:41.282Z 21 420-439 86 86 86 150 150 150 T F T <---
97652024-02-03T17:13:41.282Z 22 440-459 83 83 83 144 144 144 F F F
97662024-02-03T17:13:41.282Z 23 460-479 83 83 83 144 144 144 T F T <---
97672024-02-03T17:13:41.282Z 24 480-499 84 84 84 146 146 146 T T T
97682024-02-03T17:13:41.282Z 25 500-519 86 86 86 150 150 150 T T T
97692024-02-03T17:13:41.282Z 26 520-539 80 80 80 137 137 137 F F F
97702024-02-03T17:13:41.282Z 27 540-559 85 85 85 147 147 147 T T T
97712024-02-03T17:13:41.282Z 28 560-579 84 84 84 146 146 146 T F T <---
97722024-02-03T17:13:41.282Z 29 580-599 82 82 82 142 142 142 F F F
97732024-02-03T17:13:41.282ZMax gen: 87, Max flush: 153
97742024-02-03T17:13:41.283ZError: Difference in extent metadata found!
97752024-02-03T17:13:41.286ZA Difference in extent metadata is expected here
97762024-02-03T17:13:41.286ZOn loop 44
97772024-02-03T17:13:41.286Z
97782024-02-03T17:13:41.288ZVerifying data now
97792024-02-03T17:13:47.529ZNo change after verify
97802024-02-03T17:13:47.530ZLoop: 44 Downstairs dump after verify (and repair):
97812024-02-03T17:13:47.552ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
97822024-02-03T17:13:47.553Z 0 000-019 88 88 88 154 154 154 F F F
97832024-02-03T17:13:47.553Z 1 020-039 86 86 86 149 149 149 F F F
97842024-02-03T17:13:47.553Z 2 040-059 88 88 88 154 154 154 F F F
97852024-02-03T17:13:47.553Z 3 060-079 88 88 88 154 154 154 F F F
97862024-02-03T17:13:47.556Z 4 080-099 83 83 83 144 144 144 F F F
97872024-02-03T17:13:47.556Z 5 100-119 82 82 82 143 143 143 F F F
97882024-02-03T17:13:47.556Z 6 120-139 82 82 82 143 143 143 F F F
97892024-02-03T17:13:47.556Z 7 140-159 88 88 88 154 154 154 F F F
97902024-02-03T17:13:47.556Z 8 160-179 86 86 86 152 152 152 F F F
97912024-02-03T17:13:47.556Z 9 180-199 88 88 88 154 154 154 F F F
97922024-02-03T17:13:47.556Z 10 200-219 83 83 83 144 144 144 F F F
97932024-02-03T17:13:47.556Z 11 220-239 82 82 82 141 141 141 F F F
97942024-02-03T17:13:47.556Z 12 240-259 88 88 88 154 154 154 F F F
97952024-02-03T17:13:47.556Z 13 260-279 88 88 88 154 154 154 F F F
97962024-02-03T17:13:47.556Z 14 280-299 88 88 88 154 154 154 F F F
97972024-02-03T17:13:47.556Z 15 300-319 84 84 84 146 146 146 F F F
97982024-02-03T17:13:47.556Z 16 320-339 84 84 84 146 146 146 F F F
97992024-02-03T17:13:47.556Z 17 340-359 84 84 84 146 146 146 F F F
98002024-02-03T17:13:47.556Z 18 360-379 86 86 86 149 149 149 F F F
98012024-02-03T17:13:47.556Z 19 380-399 86 86 86 149 149 149 F F F
98022024-02-03T17:13:47.557Z 20 400-419 87 87 87 153 153 153 F F F
98032024-02-03T17:13:47.557Z 21 420-439 88 88 88 154 154 154 F F F
98042024-02-03T17:13:47.557Z 22 440-459 83 83 83 144 144 144 F F F
98052024-02-03T17:13:47.557Z 23 460-479 88 88 88 154 154 154 F F F
98062024-02-03T17:13:47.557Z 24 480-499 88 88 88 154 154 154 F F F
98072024-02-03T17:13:47.557Z 25 500-519 88 88 88 154 154 154 F F F
98082024-02-03T17:13:47.557Z 26 520-539 80 80 80 137 137 137 F F F
98092024-02-03T17:13:47.557Z 27 540-559 88 88 88 154 154 154 F F F
98102024-02-03T17:13:47.557Z 28 560-579 88 88 88 154 154 154 F F F
98112024-02-03T17:13:47.557Z 29 580-599 82 82 82 142 142 142 F F F
98122024-02-03T17:13:47.557ZMax gen: 88, Max flush: 154
98132024-02-03T17:13:47.560Z
98142024-02-03T17:13:47.560ZBegin loop 45
98152024-02-03T17:13:47.560ZDownstairs to restart: 1
98162024-02-03T17:13:47.588Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30Crucible runtime is spawned
98172024-02-03T17:13:47.592Z,"time":"2024-02-03T17:13:47.146069644Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492}
98182024-02-03T17:13:47.592Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.146668831Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492}
98192024-02-03T17:13:47.592Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.146753143Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492}
98202024-02-03T17:13:47.593Z{"msg":"Crucible 401d9d08-32cc-44a2-993b-0a2fa7ad32ed has session id: dd2739b3-0da8-4d5d-a74e-91fdcb0e5269","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.146795415Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98212024-02-03T17:13:47.593Z{"msg":"Upstairs opts: Upstairs UUID: 401d9d08-32cc-44a2-993b-0a2fa7ad32ed, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.146892451Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98222024-02-03T17:13:47.593Z{"msg":"Crucible stats registered with UUID: 401d9d08-32cc-44a2-993b-0a2fa7ad32ed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.14694089Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98232024-02-03T17:13:47.593Z{"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.14702323Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492}
98242024-02-03T17:13:47.593Z{"msg":"401d9d08-32cc-44a2-993b-0a2fa7ad32ed active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.147063114Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98252024-02-03T17:13:47.593Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.14714233Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"client":"0","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98262024-02-03T17:13:47.593Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.147188944Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"client":"0","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98272024-02-03T17:13:47.593Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.147265888Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"client":"1","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98282024-02-03T17:13:47.593Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.147313189Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"client":"1","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98292024-02-03T17:13:47.593Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.147708683Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"client":"2","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98302024-02-03T17:13:47.593Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.147760356Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"client":"2","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98312024-02-03T17:13:47.593Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.147805437Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"":"io task","client":"0","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98322024-02-03T17:13:47.593Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.147849979Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"":"io task","client":"1","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98332024-02-03T17:13:47.593Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.14793637Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"":"io task","client":"2","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98342024-02-03T17:13:47.593Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.148020519Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"":"io task","client":"0","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98352024-02-03T17:13:47.593Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.148102505Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"":"io task","client":"1","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98362024-02-03T17:13:47.593Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.148188526Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"":"io task","client":"2","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98372024-02-03T17:13:47.593Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.149201436Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"client":"2","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98382024-02-03T17:13:47.593Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.149284841Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"client":"2","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98392024-02-03T17:13:47.593Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.149331367Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"client":"0","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98402024-02-03T17:13:47.593Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.149375077Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"client":"0","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98412024-02-03T17:13:47.593Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.149413285Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"client":"1","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98422024-02-03T17:13:47.593Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.149458863Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"client":"1","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98432024-02-03T17:13:47.598Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.150284658Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"client":"2","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98442024-02-03T17:13:47.598Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.15055234Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"client":"1","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98452024-02-03T17:13:47.598Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.150885488Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"client":"0","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98462024-02-03T17:13:47.598Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.151316595Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"client":"2","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98472024-02-03T17:13:47.598Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.151394256Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"client":"2","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98482024-02-03T17:13:47.599Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.151508993Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98492024-02-03T17:13:47.599Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.15155327Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"client":"1","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98502024-02-03T17:13:47.599Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.151615066Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"client":"1","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98512024-02-03T17:13:47.599Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.151640799Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98522024-02-03T17:13:47.599Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.151661649Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"client":"0","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98532024-02-03T17:13:47.599Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.151713279Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"client":"0","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98542024-02-03T17:13:47.599Z{"msg":"[0]R flush_numbers[0..12]: [154, 149, 154, 154, 144, 143, 143, 154, 152, 154, 144, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.151740012Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"":"downstairs","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98552024-02-03T17:13:47.599Z{"msg":"[0]R generation[0..12]: [88, 86, 88, 88, 83, 82, 82, 88, 86, 88, 83, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.151789698Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"":"downstairs","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98562024-02-03T17:13:47.599Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.151816466Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"":"downstairs","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98572024-02-03T17:13:47.599Z{"msg":"[1]R flush_numbers[0..12]: [154, 149, 154, 154, 144, 143, 143, 154, 152, 154, 144, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.151839745Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"":"downstairs","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98582024-02-03T17:13:47.599Z{"msg":"[1]R generation[0..12]: [88, 86, 88, 88, 83, 82, 82, 88, 86, 88, 83, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.151887378Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"":"downstairs","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98592024-02-03T17:13:47.599Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.151919212Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"":"downstairs","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98602024-02-03T17:13:47.599Z{"msg":"[2]R flush_numbers[0..12]: [154, 149, 154, 154, 144, 143, 143, 154, 152, 154, 144, 141]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
98612024-02-03T17:13:47.599Z,"time":"2024-02-03T17:13:47.151967892Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"":"downstairs","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98622024-02-03T17:13:47.599Z{"msg":"[2]R generation[0..12]: [88, 86, 88, 88, 83, 82, 82, 88, 86, 88, 83, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.151993761Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"":"downstairs","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98632024-02-03T17:13:47.599Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.152045648Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"":"downstairs","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98642024-02-03T17:13:47.599Z{"msg":"Max found gen is 89","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.152069908Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"":"downstairs","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98652024-02-03T17:13:47.599Z{"msg":"Generation requested: 90 >= found:89","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.152091985Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"":"downstairs","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98662024-02-03T17:13:47.599Z{"msg":"Next flush: 155","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.152139118Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"":"downstairs","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98672024-02-03T17:13:47.599Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.152163462Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"":"downstairs","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98682024-02-03T17:13:47.599Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.152186271Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98692024-02-03T17:13:47.599Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.152250501Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"client":"0","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98702024-02-03T17:13:47.600Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.152274828Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"client":"1","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98712024-02-03T17:13:47.600Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.152344155Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"client":"2","":"downstairs","session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98722024-02-03T17:13:47.600Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.152368542Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98732024-02-03T17:13:47.600Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.152389247Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98742024-02-03T17:13:47.600Z{"msg":"401d9d08-32cc-44a2-993b-0a2fa7ad32ed is now active with session: dd2739b3-0da8-4d5d-a74e-91fdcb0e5269","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.152411412Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98752024-02-03T17:13:47.600Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.152446826Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492,"session_id":"dd2739b3-0da8-4d5d-a74e-91fdcb0e5269"}
98762024-02-03T17:13:47.600Z{"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:47.152514271Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492}
98772024-02-03T17:13:47.600ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
98782024-02-03T17:13:47.600ZLoading write count information from file "/tmp/test_repair_verify.data"
98792024-02-03T17:13:47.600ZRun Repair workload
98802024-02-03T17:13:47.600Z01/30 Write block 139 len 28672 data: 4 4 7 7 7 6 5
98812024-02-03T17:13:47.603Z02/30 Flush
98822024-02-03T17:13:47.603Z03/30 Write block 296 len 40960 data: 10 9 8 8 6 8 8 7 8 7
98832024-02-03T17:13:47.603Z04/30 Write block 75 len 12288 data: 9 10 9
98842024-02-03T17:13:47.606Z05/30 Flush
98852024-02-03T17:13:47.609Z06/30 Write block 202 len 4096 data: 6
98862024-02-03T17:13:47.609Z07/30 Write block 382 len 28672 data: 4 4 4 6 7 7 6
98872024-02-03T17:13:47.612Z08/30 Write block 97 len 28672 data: 10 8 8 8 7 4 4
98882024-02-03T17:13:47.612Z09/30 Read block 133 len 12288
98892024-02-03T17:13:47.615Z10/30 Read block 326 len 28672
98902024-02-03T17:13:47.615Z11/30 Read block 44 len 28672
98912024-02-03T17:13:47.618Z12/30 Read block 586 len 28672
98922024-02-03T17:13:47.618Z13/30 Read block 458 len 36864
98932024-02-03T17:13:47.621Z14/30 Flush
98942024-02-03T17:13:47.624Z15/30 Write block 560 len 24576 data: 6 7 8 8 8 7
98952024-02-03T17:13:47.626Z16/30 Read block 375 len 28672
98962024-02-03T17:13:47.627Z17/30 Read block 437 len 16384
98972024-02-03T17:13:47.629Z18/30 Write block 503 len 40960 data: 9 10 8 9 10 10 10 8 10 10
98982024-02-03T17:13:47.629Z19/30 Flush
98992024-02-03T17:13:47.634Z20/30 Write block 281 len 36864 data: 12 14 15 11 10 10 9 7 8
99002024-02-03T17:13:47.637Z21/30 Write block 275 len 4096 data: 10
99012024-02-03T17:13:47.637Z22/30 Write block 314 len 16384 data: 9 9 9 10
99022024-02-03T17:13:47.637Z23/30 Read block 302 len 36864
99032024-02-03T17:13:47.640Z24/30 Write block 318 len 8192 data: 7 10
99042024-02-03T17:13:47.640Z25/30 Read block 359 len 24576
99052024-02-03T17:13:47.643Z26/30 Write block 317 len 8192 data: 11 8
99062024-02-03T17:13:47.646Z27/30 Write block 23 len 40960 data: 13 12 10 10 10 10 8 7 8 8
99072024-02-03T17:13:47.646Z28/30 Read block 467 len 16384
99082024-02-03T17:13:47.648Z29/30 Write block 505 len 32768 data: 9 10 11 11 11 9 11 11
99092024-02-03T17:13:47.652Z30/30 Write block 340 len 32768 data: 9 7 7 5 6 6 5 6
99102024-02-03T17:13:47.652Z----------------------------------------------------------------
99112024-02-03T17:13:47.652Z Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:30
99122024-02-03T17:13:47.652ZStates: Active Active Active
99132024-02-03T17:13:47.652ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
99142024-02-03T17:13:47.652Z 1 Acked 1000 Write 7 Done Sent Done false
99152024-02-03T17:13:47.652Z 2 Acked 1001 Flush 0 Done Sent Done false
99162024-02-03T17:13:47.652Z 3 Acked 1002 Write 10 Done Sent Done false
99172024-02-03T17:13:47.652Z 4 Acked 1003 Write 3 Done Sent Done false
99182024-02-03T17:13:47.652Z 5 Acked 1004 Flush 0 Done Sent Done false
99192024-02-03T17:13:47.652Z 6 Acked 1005 Write 1 Done Sent Done false
99202024-02-03T17:13:47.652Z 7 Acked 1006 Write 7 Done Sent Done false
99212024-02-03T17:13:47.652Z 8 Acked 1007 Write 7 Done Sent Done false
99222024-02-03T17:13:47.652Z 9 Acked 1008 Read 3 Done Sent Done false
99232024-02-03T17:13:47.652Z 10 Acked 1009 Read 7 Done Sent Done false
99242024-02-03T17:13:47.653Z 11 Acked 1010 Read 7 Done Sent Done false
99252024-02-03T17:13:47.653Z 12 Acked 1011 Read 7 Done Sent Done false
99262024-02-03T17:13:47.653Z 13 Acked 1012 Read 9 Done Sent Done false
99272024-02-03T17:13:47.653Z 14 Acked 1013 Flush 0 Done Sent Done false
99282024-02-03T17:13:47.653Z 15 Acked 1014 Write 6 Done Sent Done false
99292024-02-03T17:13:47.653Z 16 Acked 1015 Read 7 Done Sent Done false
99302024-02-03T17:13:47.653Z 17 Acked 1016 Read 4 Done Sent Done false
99312024-02-03T17:13:47.653Z 18 Acked 1017 Write 10 Done Sent Done false
99322024-02-03T17:13:47.653Z 19 Acked 1018 Flush 0 Done Sent Done false
99332024-02-03T17:13:47.653Z 20 Acked 1019 Write 9 Done Sent Done false
99342024-02-03T17:13:47.653Z 21 Acked 1020 Write 1 Done Sent Done false
99352024-02-03T17:13:47.653Z 22 Acked 1021 Write 4 Done Sent Done false
99362024-02-03T17:13:47.653Z 23 Acked 1022 Read 9 Done Sent Done false
99372024-02-03T17:13:47.653Z 24 Acked 1023 Write 2 Done Sent Done false
99382024-02-03T17:13:47.653Z 25 Acked 1024 Read 6 Done Sent Done false
99392024-02-03T17:13:47.653Z 26 Acked 1025 Write 2 Done Sent Done false
99402024-02-03T17:13:47.653Z 27 Acked 1026 Write 10 Done Sent Done false
99412024-02-03T17:13:47.653Z 28 Acked 1027 Read 4 Done Sent Done false
99422024-02-03T17:13:47.653Z 29 Acked 1028 Write 8 Done Sent Done false
99432024-02-03T17:13:47.653Z 30 Acked 1029 Write 8 Sent Sent Sent false
99442024-02-03T17:13:47.653Z STATES DS:0 DS:1 DS:2 TOTAL
99452024-02-03T17:13:47.653Z New 0 0 0 0
99462024-02-03T17:13:47.653Z Sent 1 30 1 32
99472024-02-03T17:13:47.653Z Done 29 0 29 58
99482024-02-03T17:13:47.653Z Skipped 0 0 0 0
99492024-02-03T17:13:47.654Z Error 0 0 0 0
99502024-02-03T17:13:47.654ZLast Flush: 1018 0 1018
99512024-02-03T17:13:47.654ZDownstairs last five completed:
99522024-02-03T17:13:47.654ZUpstairs last five completed: 30 29 28 27 26
99532024-02-03T17:13:47.654Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:47.208818643Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1492}
99542024-02-03T17:13:47.654ZWrote out file "/tmp/test_repair_verify.data"
99552024-02-03T17:13:47.657Z
99562024-02-03T17:13:47.664ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
99572024-02-03T17:13:47.690ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
99582024-02-03T17:13:47.690Z 0 000-019 88 88 88 154 154 154 F F F
99592024-02-03T17:13:47.690Z 1 020-039 86 86 86 149 149 149 T F T <---
99602024-02-03T17:13:47.690Z 2 040-059 88 88 88 154 154 154 F F F
99612024-02-03T17:13:47.694Z 3 060-079 90 88 90 156 154 156 F F F <---
99622024-02-03T17:13:47.694Z 4 080-099 90 83 90 157 144 157 F F F <---
99632024-02-03T17:13:47.694Z 5 100-119 90 82 90 157 143 157 F F F <---
99642024-02-03T17:13:47.694Z 6 120-139 90 82 90 155 143 155 F F F <---
99652024-02-03T17:13:47.695Z 7 140-159 90 88 90 155 154 155 F F F <---
99662024-02-03T17:13:47.695Z 8 160-179 86 86 86 152 152 152 F F F
99672024-02-03T17:13:47.695Z 9 180-199 88 88 88 154 154 154 F F F
99682024-02-03T17:13:47.695Z 10 200-219 90 83 90 157 144 157 F F F <---
99692024-02-03T17:13:47.695Z 11 220-239 82 82 82 141 141 141 F F F
99702024-02-03T17:13:47.695Z 12 240-259 88 88 88 154 154 154 F F F
99712024-02-03T17:13:47.695Z 13 260-279 88 88 88 154 154 154 T F T <---
99722024-02-03T17:13:47.695Z 14 280-299 90 88 90 156 154 156 T F T <---
99732024-02-03T17:13:47.695Z 15 300-319 90 84 90 156 146 156 T F T <---
99742024-02-03T17:13:47.695Z 16 320-339 84 84 84 146 146 146 F F F
99752024-02-03T17:13:47.695Z 17 340-359 84 84 84 146 146 146 T F T <---
99762024-02-03T17:13:47.695Z 18 360-379 86 86 86 149 149 149 F F F
99772024-02-03T17:13:47.696Z 19 380-399 90 86 90 157 149 157 F F F <---
99782024-02-03T17:13:47.696Z 20 400-419 87 87 87 153 153 153 F F F
99792024-02-03T17:13:47.696Z 21 420-439 88 88 88 154 154 154 F F F
99802024-02-03T17:13:47.696Z 22 440-459 83 83 83 144 144 144 F F F
99812024-02-03T17:13:47.696Z 23 460-479 88 88 88 154 154 154 F F F
99822024-02-03T17:13:47.696Z 24 480-499 88 88 88 154 154 154 F F F
99832024-02-03T17:13:47.696Z 25 500-519 90 88 90 158 154 158 T F T <---
99842024-02-03T17:13:47.696Z 26 520-539 80 80 80 137 137 137 F F F
99852024-02-03T17:13:47.696Z 27 540-559 88 88 88 154 154 154 F F F
99862024-02-03T17:13:47.696Z 28 560-579 90 88 90 158 154 158 F F F <---
99872024-02-03T17:13:47.696Z 29 580-599 82 82 82 142 142 142 F F F
99882024-02-03T17:13:47.696ZMax gen: 90, Max flush: 158
99892024-02-03T17:13:47.696ZError: Difference in extent metadata found!
99902024-02-03T17:13:47.700ZA Difference in extent metadata is expected here
99912024-02-03T17:13:47.700ZOn loop 45
99922024-02-03T17:13:47.700Z
99932024-02-03T17:13:47.700ZVerifying data now
99942024-02-03T17:13:52.926ZNo change after verify
99952024-02-03T17:13:52.926ZLoop: 45 Downstairs dump after verify (and repair):
99962024-02-03T17:13:52.946ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
99972024-02-03T17:13:52.946Z 0 000-019 88 88 88 154 154 154 F F F
99982024-02-03T17:13:52.946Z 1 020-039 91 91 91 159 159 159 F F F
99992024-02-03T17:13:52.946Z 2 040-059 88 88 88 154 154 154 F F F
100002024-02-03T17:13:52.947Z 3 060-079 90 90 90 156 156 156 F F F
100012024-02-03T17:13:52.947Z 4 080-099 90 90 90 157 157 157 F F F
100022024-02-03T17:13:52.959Z 5 100-119 90 90 90 157 157 157 F F F
100032024-02-03T17:13:52.960Z 6 120-139 90 90 90 155 155 155 F F F
100042024-02-03T17:13:52.960Z 7 140-159 90 90 90 155 155 155 F F F
100052024-02-03T17:13:52.960Z 8 160-179 86 86 86 152 152 152 F F F
100062024-02-03T17:13:52.960Z 9 180-199 88 88 88 154 154 154 F F F
100072024-02-03T17:13:52.960Z 10 200-219 90 90 90 157 157 157 F F F
100082024-02-03T17:13:52.961Z 11 220-239 82 82 82 141 141 141 F F F
100092024-02-03T17:13:52.961Z 12 240-259 88 88 88 154 154 154 F F F
100102024-02-03T17:13:52.961Z 13 260-279 91 91 91 159 159 159 F F F
100112024-02-03T17:13:52.961Z 14 280-299 91 91 91 159 159 159 F F F
100122024-02-03T17:13:52.961Z 15 300-319 91 91 91 159 159 159 F F F
100132024-02-03T17:13:52.961Z 16 320-339 84 84 84 146 146 146 F F F
100142024-02-03T17:13:52.961Z 17 340-359 91 91 91 159 159 159 F F F
100152024-02-03T17:13:52.961Z 18 360-379 86 86 86 149 149 149 F F F
100162024-02-03T17:13:52.961Z 19 380-399 90 90 90 157 157 157 F F F
100172024-02-03T17:13:52.961Z 20 400-419 87 87 87 153 153 153 F F F
100182024-02-03T17:13:52.962Z 21 420-439 88 88 88 154 154 154 F F F
100192024-02-03T17:13:52.962Z 22 440-459 83 83 83 144 144 144 F F F
100202024-02-03T17:13:52.962Z 23 460-479 88 88 88 154 154 154 F F F
100212024-02-03T17:13:52.962Z 24 480-499 88 88 88 154 154 154 F F F
100222024-02-03T17:13:52.962Z 25 500-519 91 91 91 159 159 159 F F F
100232024-02-03T17:13:52.962Z 26 520-539 80 80 80 137 137 137 F F F
100242024-02-03T17:13:52.962Z 27 540-559 88 88 88 154 154 154 F F F
100252024-02-03T17:13:52.962Z 28 560-579 90 90 90 158 158 158 F F F
100262024-02-03T17:13:52.963Z 29 580-599 82 82 82 142 142 142 F F F
100272024-02-03T17:13:52.963ZMax gen: 91, Max flush: 159
100282024-02-03T17:13:52.963Z
100292024-02-03T17:13:52.963ZBegin loop 46
100302024-02-03T17:13:52.963ZDownstairs to restart: 0
100312024-02-03T17:13:52.991Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.542861478Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500}
100322024-02-03T17:13:52.991Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.543347791Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500}
100332024-02-03T17:13:52.991Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.543378412Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500}
100342024-02-03T17:13:53.003Z{"msg":"Crucible b7d536ab-d1cc-481d-a841-bf6c0da9f5b9 has session id: 00020421-8835-4fa1-8d79-66e5afc2b8d2","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.543411633Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100352024-02-03T17:13:53.003Z{"msg":"Upstairs opts: Upstairs UUID: b7d536ab-d1cc-481d-a841-bf6c0da9f5b9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"Crucible runtime is spawned
100362024-02-03T17:13:53.003Zcrucible","level":30,"time":"2024-02-03T17:13:52.543694747Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100372024-02-03T17:13:53.003Z{"msg":"Crucible stats registered with UUID: b7d536ab-d1cc-481d-a841-bf6c0da9f5b9","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.543823574Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100382024-02-03T17:13:53.003Z{"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.543901409Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500}
100392024-02-03T17:13:53.003Z{"msg":"b7d536ab-d1cc-481d-a841-bf6c0da9f5b9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.544039157Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100402024-02-03T17:13:53.003Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.544063152Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"client":"0","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100412024-02-03T17:13:53.003Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.544094292Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"client":"0","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100422024-02-03T17:13:53.004Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.544119484Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"client":"1","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100432024-02-03T17:13:53.004Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.544142503Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"client":"1","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100442024-02-03T17:13:53.004Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.544164463Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"client":"2","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100452024-02-03T17:13:53.004Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.544360423Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"client":"2","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100462024-02-03T17:13:53.004Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.544453445Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"":"io task","client":"2","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100472024-02-03T17:13:53.004Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.544497846Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"":"io task","client":"1","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100482024-02-03T17:13:53.004Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.544539396Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"":"io task","client":"0","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100492024-02-03T17:13:53.004Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.544615167Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"":"io task","client":"2","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100502024-02-03T17:13:53.004Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.544658622Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"":"io task","client":"1","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100512024-02-03T17:13:53.004Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.544735757Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"":"io task","client":"0","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100522024-02-03T17:13:53.004Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.54594296Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"client":"1","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100532024-02-03T17:13:53.004Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.54615167Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"client":"1","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100542024-02-03T17:13:53.004Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.546242782Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"client":"2","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100552024-02-03T17:13:53.004Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.546284944Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"client":"2","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100562024-02-03T17:13:53.004Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.546952414Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"client":"1","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100572024-02-03T17:13:53.004Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.547156157Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"client":"2","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100582024-02-03T17:13:53.004Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.547527287Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"client":"0","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100592024-02-03T17:13:53.004Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.547621407Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"client":"0","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100602024-02-03T17:13:53.004Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.547783307Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"client":"1","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100612024-02-03T17:13:53.004Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.54793797Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"client":"1","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100622024-02-03T17:13:53.004Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.548019886Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100632024-02-03T17:13:53.004Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.548062864Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"client":"2","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100642024-02-03T17:13:53.004Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.548096457Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"client":"2","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100652024-02-03T17:13:53.004Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.548122367Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100662024-02-03T17:13:53.005Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.548338785Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"client":"0","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100672024-02-03T17:13:53.005Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.549060452Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"client":"0","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100682024-02-03T17:13:53.005Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.54912395Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"client":"0","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100692024-02-03T17:13:53.005Z{"msg":"[0]R flush_numbers[0..12]: [154, 159, 154, 156, 157, 157, 155, 155, 152, 154, 157, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.549151548Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"":"downstairs","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100702024-02-03T17:13:53.005Z{"msg":"[0]R generation[0..12]: [88, 91, 88, 90, 90, 90, 90, 90, 86, 88, 90, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.549175766Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"":"downstairs","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100712024-02-03T17:13:53.005Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.549243932Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"":"downstairs","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100722024-02-03T17:13:53.005Z{"msg":"[1]R flush_numbers[0..12]: [154, 159, 154, 156, 157, 157, 155, 155, 152, 154, 157, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.549275828Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"":"downstairs","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100732024-02-03T17:13:53.005Z{"msg":"[1]R generation[0..12]: [88, 91, 88, 90, 90, 90, 90, 90, 86, 88, 90, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.549341035Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"":"downstairs","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100742024-02-03T17:13:53.005Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.549374001Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"":"downstairs"Wait for a query_work_queue command to finish before sending IO
100752024-02-03T17:13:53.005Z,"":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100762024-02-03T17:13:53.005Z{"msg":"[2]R flush_numbers[0..12]: [154, 159, 154, 156, 157, 157, 155, 155, 152, 154, 157, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.549516644Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"":"downstairs","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100772024-02-03T17:13:53.005Z{"msg":"[2]R generation[0..12]: [88, 91, 88, 90, 90, 90, 90, 90, 86, 88, 90, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.549552909Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"":"downstairs","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100782024-02-03T17:13:53.005Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.549618314Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"":"downstairs","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100792024-02-03T17:13:53.005Z{"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.549647786Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"":"downstairs","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100802024-02-03T17:13:53.005Z{"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.549712698Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"":"downstairs","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100812024-02-03T17:13:53.005Z{"msg":"Next flush: 160","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.549739582Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"":"downstairs","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100822024-02-03T17:13:53.005Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.549805878Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"":"downstairs","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100832024-02-03T17:13:53.005Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.549834394Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100842024-02-03T17:13:53.005Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.549898341Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"client":"0","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100852024-02-03T17:13:53.005Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.549925887Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"client":"1","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100862024-02-03T17:13:53.005Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.549992619Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"client":"2","":"downstairs","session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100872024-02-03T17:13:53.005Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.550020646Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"session_id":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
100882024-02-03T17:13:53.005Z00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100892024-02-03T17:13:53.005Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.55017711Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100902024-02-03T17:13:53.005Z{"msg":"b7d536ab-d1cc-481d-a841-bf6c0da9f5b9 is now active with session: 00020421-8835-4fa1-8d79-66e5afc2b8d2","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.550212152Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100912024-02-03T17:13:53.006Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.550273148Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500,"session_id":"00020421-8835-4fa1-8d79-66e5afc2b8d2"}
100922024-02-03T17:13:53.006Z{"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:52.550298278Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500}
100932024-02-03T17:13:53.006ZLoading write count information from file "/tmp/test_repair_verify.data"
100942024-02-03T17:13:53.006ZRun Repair workload
100952024-02-03T17:13:53.006Z01/30 Write block 165 len 40960 data: 13 15 15 11 11 10 9 7 8 8
100962024-02-03T17:13:53.010Z02/30 Write block 376 len 32768 data: 8 9 7 8 6 5 5 5
100972024-02-03T17:13:53.010Z03/30 Write block 19 len 16384 data: 12 9 12 13
100982024-02-03T17:13:53.010Z04/30 Write block 260 len 16384 data: 8 9 8 10
100992024-02-03T17:13:53.010Z05/30 Read block 430 len 12288
101002024-02-03T17:13:53.010Z06/30 Read block 17 len 16384
101012024-02-03T17:13:53.011Z07/30 Read block 505 len 32768
101022024-02-03T17:13:53.011Z08/30 Read block 238 len 20480
101032024-02-03T17:13:53.011Z09/30 Read block 433 len 28672
101042024-02-03T17:13:53.011Z10/30 Write block 18 len 4096 data: 10
101052024-02-03T17:13:53.011Z11/30 Write block 537 len 24576 data: 5 5 3 4 5 6
101062024-02-03T17:13:53.014Z12/30 Read block 486 len 40960
101072024-02-03T17:13:53.014Z13/30 Flush
101082024-02-03T17:13:53.017Z14/30 Write block 575 len 8192 data: 6 5
101092024-02-03T17:13:53.017Z15/30 Read block 286 len 32768
101102024-02-03T17:13:53.020Z16/30 Read block 554 len 36864
101112024-02-03T17:13:53.023Z17/30 Read block 340 len 4096
101122024-02-03T17:13:53.023Z18/30 Read block 453 len 16384
101132024-02-03T17:13:53.023Z19/30 Write block 390 len 16384 data: 9 10 13 14
101142024-02-03T17:13:53.026Z20/30 Read block 67 len 36864
101152024-02-03T17:13:53.026Z21/30 Flush
101162024-02-03T17:13:53.029Z22/30 Write block 392 len 4096 data: 14
101172024-02-03T17:13:53.032Z23/30 Read block 240 len 24576
101182024-02-03T17:13:53.032Z24/30 Read block 189 len 20480
101192024-02-03T17:13:53.035Z25/30 Read block 138 len 20480
101202024-02-03T17:13:53.035Z26/30 Write block 459 len 12288 data: 7 9 7
101212024-02-03T17:13:53.038Z27/30 Write block 485 len 4096 data: 7
101222024-02-03T17:13:53.038Z28/30 Read block 463 len 28672
101232024-02-03T17:13:53.042Z29/30 Write block 539 len 20480 data: 4 5 6 7 5
101242024-02-03T17:13:53.042Z30/30 Read block 519 len 4096
101252024-02-03T17:13:53.042Z----------------------------------------------------------------
101262024-02-03T17:13:53.042Z Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:30
101272024-02-03T17:13:53.042ZStates: Active Active Active
101282024-02-03T17:13:53.042ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
101292024-02-03T17:13:53.042Z 1 Acked 1000 Write 10 Done Done Done false
101302024-02-03T17:13:53.042Z 2 Acked 1001 Write 8 Done Done Done false
101312024-02-03T17:13:53.042Z 3 Acked 1002 Write 4 Done Done Done false
101322024-02-03T17:13:53.042Z 4 Acked 1003 Write 4 Done Done Done false
101332024-02-03T17:13:53.042Z 5 Acked 1004 Read 3 Done Done Done false
101342024-02-03T17:13:53.042Z 6 Acked 1005 Read 4 Done Done Done false
101352024-02-03T17:13:53.042Z 7 Acked 1006 Read 8 Done Done Done false
101362024-02-03T17:13:53.042Z 8 Acked 1007 Read 5 Done Done Done false
101372024-02-03T17:13:53.042Z 9 Acked 1008 Read 7 Sent Done Done false
101382024-02-03T17:13:53.042Z 10 Acked 1009 Write 1 Sent Done Done false
101392024-02-03T17:13:53.042Z 11 Acked 1010 Write 6 Sent Done Done false
101402024-02-03T17:13:53.042Z 12 Acked 1011 Read 10 Sent Done Done false
101412024-02-03T17:13:53.042Z 13 Acked 1012 Flush 0 Sent Done Done false
101422024-02-03T17:13:53.043Z 14 Acked 1013 Write 2 Sent Done Done false
101432024-02-03T17:13:53.043Z 15 Acked 1014 Read 8 Sent Done Done false
101442024-02-03T17:13:53.043Z 16 Acked 1015 Read 9 Sent Done Done false
101452024-02-03T17:13:53.043Z 17 Acked 1016 Read 1 Sent Done Done false
101462024-02-03T17:13:53.043Z 18 Acked 1017 Read 4 Sent Done Done false
101472024-02-03T17:13:53.043Z 19 Acked 1018 Write 4 Sent Done Done false
101482024-02-03T17:13:53.043Z 20 Acked 1019 Read 9 Sent Done Done false
101492024-02-03T17:13:53.043Z 21 Acked 1020 Flush 0 Sent Done Done false
101502024-02-03T17:13:53.043Z 22 Acked 1021 Write 1 Sent Done Done false
101512024-02-03T17:13:53.043Z 23 Acked 1022 Read 6 Sent Done Done false
101522024-02-03T17:13:53.043Z 24 Acked 1023 Read 5 Sent Done Done false
101532024-02-03T17:13:53.043Z 25 Acked 1024 Read 5 Sent Done Done false
101542024-02-03T17:13:53.043Z 26 Acked 1025 Write 3 Sent Done Done false
101552024-02-03T17:13:53.043Z 27 Acked 1026 Write 1 Sent Done Done false
101562024-02-03T17:13:53.043Z 28 Acked 1027 Read 7 Sent Done Done false
101572024-02-03T17:13:53.043Z 29 Acked 1028 Write 5 Sent Done Done false
101582024-02-03T17:13:53.043Z 30 Acked 1029 Read 1 Sent Done Done false
101592024-02-03T17:13:53.043Z STATES DS:0 DS:1 DS:2 TOTAL
101602024-02-03T17:13:53.043Z New 0 0 0 0
101612024-02-03T17:13:53.043Z Sent 22 0 0 22
101622024-02-03T17:13:53.043Z Done 8 30 30 68
101632024-02-03T17:13:53.047Z Skipped 0 0 0 0
101642024-02-03T17:13:53.047Z Error 0 0 0 0
101652024-02-03T17:13:53.047ZLast Flush: 0 1020 1020
101662024-02-03T17:13:53.047ZDownstairs last five completed:
101672024-02-03T17:13:53.047ZUpstairs last five completed: 30 29 28 27 26
101682024-02-03T17:13:53.047Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:52.60103095Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1500}
101692024-02-03T17:13:53.047ZWrote out file "/tmp/test_repair_verify.data"
101702024-02-03T17:13:53.050Z
101712024-02-03T17:13:53.056ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
101722024-02-03T17:13:53.095ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
101732024-02-03T17:13:53.095Z 0 000-019 88 92 92 154 160 160 T F F <---
101742024-02-03T17:13:53.095Z 1 020-039 91 92 92 159 160 160 T F F <---
101752024-02-03T17:13:53.107Z 2 040-059 88 88 88 154 154 154 F F F
101762024-02-03T17:13:53.107Z 3 060-079 90 90 90 156 156 156 F F F
101772024-02-03T17:13:53.107Z 4 080-099 90 90 90 157 157 157 F F F
101782024-02-03T17:13:53.107Z 5 100-119 90 90 90 157 157 157 F F F
101792024-02-03T17:13:53.107Z 6 120-139 90 90 90 155 155 155 F F F
101802024-02-03T17:13:53.107Z 7 140-159 90 90 90 155 155 155 F F F
101812024-02-03T17:13:53.107Z 8 160-179 86 92 92 152 160 160 T F F <---
101822024-02-03T17:13:53.107Z 9 180-199 88 88 88 154 154 154 F F F
101832024-02-03T17:13:53.107Z 10 200-219 90 90 90 157 157 157 F F F
101842024-02-03T17:13:53.107Z 11 220-239 82 82 82 141 141 141 F F F
101852024-02-03T17:13:53.107Z 12 240-259 88 88 88 154 154 154 F F F
101862024-02-03T17:13:53.107Z 13 260-279 91 92 92 159 160 160 T F F <---
101872024-02-03T17:13:53.107Z 14 280-299 91 91 91 159 159 159 F F F
101882024-02-03T17:13:53.107Z 15 300-319 91 91 91 159 159 159 F F F
101892024-02-03T17:13:53.107Z 16 320-339 84 84 84 146 146 146 F F F
101902024-02-03T17:13:53.107Z 17 340-359 91 91 91 159 159 159 F F F
101912024-02-03T17:13:53.107Z 18 360-379 86 92 92 149 160 160 T F F <---
101922024-02-03T17:13:53.108Z 19 380-399 90 92 92 157 161 161 T T T <---
101932024-02-03T17:13:53.108Z 20 400-419 87 87 87 153 153 153 F F F
101942024-02-03T17:13:53.108Z 21 420-439 88 88 88 154 154 154 F F F
101952024-02-03T17:13:53.108Z 22 440-459 83 83 83 144 144 144 F T T <---
101962024-02-03T17:13:53.108Z 23 460-479 88 88 88 154 154 154 F T T <---
101972024-02-03T17:13:53.108Z 24 480-499 88 88 88 154 154 154 F T T <---
101982024-02-03T17:13:53.108Z 25 500-519 91 91 91 159 159 159 F F F
101992024-02-03T17:13:53.108Z 26 520-539 80 92 92 137 160 160 F T T <---
102002024-02-03T17:13:53.108Z 27 540-559 88 92 92 154 160 160 F T T <---
102012024-02-03T17:13:53.108Z 28 560-579 90 92 92 158 161 161 F F F <---
102022024-02-03T17:13:53.108Z 29 580-599 82 82 82 142 142 142 F F F
102032024-02-03T17:13:53.108ZMax gen: 92, Max flush: 161
102042024-02-03T17:13:53.108ZError: Difference in extent metadata found!
102052024-02-03T17:13:53.108ZA Difference in extent metadata is expected here
102062024-02-03T17:13:53.108ZOn loop 46
102072024-02-03T17:13:53.108Z
102082024-02-03T17:13:53.108ZVerifying data now
102092024-02-03T17:13:59.169ZNo change after verify
102102024-02-03T17:13:59.169ZLoop: 46 Downstairs dump after verify (and repair):
102112024-02-03T17:13:59.193ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
102122024-02-03T17:13:59.193Z 0 000-019 92 92 92 160 160 160 F F F
102132024-02-03T17:13:59.193Z 1 020-039 92 92 92 160 160 160 F F F
102142024-02-03T17:13:59.193Z 2 040-059 88 88 88 154 154 154 F F F
102152024-02-03T17:13:59.196Z 3 060-079 90 90 90 156 156 156 F F F
102162024-02-03T17:13:59.196Z 4 080-099 90 90 90 157 157 157 F F F
102172024-02-03T17:13:59.196Z 5 100-119 90 90 90 157 157 157 F F F
102182024-02-03T17:13:59.196Z 6 120-139 90 90 90 155 155 155 F F F
102192024-02-03T17:13:59.196Z 7 140-159 90 90 90 155 155 155 F F F
102202024-02-03T17:13:59.196Z 8 160-179 92 92 92 160 160 160 F F F
102212024-02-03T17:13:59.196Z 9 180-199 88 88 88 154 154 154 F F F
102222024-02-03T17:13:59.196Z 10 200-219 90 90 90 157 157 157 F F F
102232024-02-03T17:13:59.196Z 11 220-239 82 82 82 141 141 141 F F F
102242024-02-03T17:13:59.196Z 12 240-259 88 88 88 154 154 154 F F F
102252024-02-03T17:13:59.196Z 13 260-279 92 92 92 160 160 160 F F F
102262024-02-03T17:13:59.196Z 14 280-299 91 91 91 159 159 159 F F F
102272024-02-03T17:13:59.197Z 15 300-319 91 91 91 159 159 159 F F F
102282024-02-03T17:13:59.197Z 16 320-339 84 84 84 146 146 146 F F F
102292024-02-03T17:13:59.197Z 17 340-359 91 91 91 159 159 159 F F F
102302024-02-03T17:13:59.197Z 18 360-379 92 92 92 160 160 160 F F F
102312024-02-03T17:13:59.197Z 19 380-399 93 93 93 162 162 162 F F F
102322024-02-03T17:13:59.197Z 20 400-419 87 87 87 153 153 153 F F F
102332024-02-03T17:13:59.197Z 21 420-439 88 88 88 154 154 154 F F F
102342024-02-03T17:13:59.197Z 22 440-459 93 93 93 162 162 162 F F F
102352024-02-03T17:13:59.197Z 23 460-479 93 93 93 162 162 162 F F F
102362024-02-03T17:13:59.197Z 24 480-499 93 93 93 162 162 162 F F F
102372024-02-03T17:13:59.197Z 25 500-519 91 91 91 159 159 159 F F F
102382024-02-03T17:13:59.197Z 26 520-539 93 93 93 162 162 162 F F F
102392024-02-03T17:13:59.197Z 27 540-559 93 93 93 162 162 162 F F F
102402024-02-03T17:13:59.197Z 28 560-579 92 92 92 161 161 161 F F F
102412024-02-03T17:13:59.197Z 29 580-599 82 82 82 142 142 142 F F F
102422024-02-03T17:13:59.197ZMax gen: 93, Max flush: 162
102432024-02-03T17:13:59.200Z
102442024-02-03T17:13:59.200ZBegin loop 47
102452024-02-03T17:13:59.200ZDownstairs to restart: 2
102462024-02-03T17:13:59.230Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.789195432Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508}
102472024-02-03T17:13:59.230Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.789850816Z"Crucible runtime is spawned
102482024-02-03T17:13:59.230Z,"hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508}
102492024-02-03T17:13:59.234Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.789928254Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508}
102502024-02-03T17:13:59.234Z{"msg":"Crucible d81b7adf-d9d4-4f87-a2e3-c18c207d2285 has session id: ca641b9c-f83f-45bd-8668-a1cb60ddff42","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.790040741Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102512024-02-03T17:13:59.235Z{"msg":"Upstairs opts: Upstairs UUID: d81b7adf-d9d4-4f87-a2e3-c18c207d2285, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.790128607Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102522024-02-03T17:13:59.235Z{"msg":"Crucible stats registered with UUID: d81b7adf-d9d4-4f87-a2e3-c18c207d2285","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.79016362Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102532024-02-03T17:13:59.235Z{"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.790215866Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508}
102542024-02-03T17:13:59.235Z{"msg":"d81b7adf-d9d4-4f87-a2e3-c18c207d2285 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.790331839Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102552024-02-03T17:13:59.235Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.790389536Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"client":"0","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102562024-02-03T17:13:59.235Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.790415856Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"client":"0","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102572024-02-03T17:13:59.235Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.790511044Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"client":"1","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102582024-02-03T17:13:59.235Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.790556338Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"client":"1","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102592024-02-03T17:13:59.235Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.790598326Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"client":"2","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102602024-02-03T17:13:59.235Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.790641777Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"client":"2","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102612024-02-03T17:13:59.235Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.790718608Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"":"io task","client":"2","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102622024-02-03T17:13:59.235Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.790765229Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"":"io task","client":"0","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102632024-02-03T17:13:59.235Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.79091874Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"":"io task","client":"1","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102642024-02-03T17:13:59.235Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.790970312Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"":"io task","client":"1","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102652024-02-03T17:13:59.235Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.791053395Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"":"io task","client":"2","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102662024-02-03T17:13:59.235Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.791136068Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"":"io task","client":"0","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102672024-02-03T17:13:59.235Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.792365215Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"client":"0","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102682024-02-03T17:13:59.235Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.792497443Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"client":"0","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102692024-02-03T17:13:59.235Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.79254251Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"client":"2","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102702024-02-03T17:13:59.235Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.792649402Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"client":"2","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102712024-02-03T17:13:59.235Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.792688957Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"client":"1","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102722024-02-03T17:13:59.235Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.792718329Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"client":"1","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102732024-02-03T17:13:59.240Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.793530483Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"client":"1","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102742024-02-03T17:13:59.240Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.793637256Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"client":"2","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102752024-02-03T17:13:59.240Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.79376234Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"client":"0","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102762024-02-03T17:13:59.240Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.794752296Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"client":"1","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102772024-02-03T17:13:59.240Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.794827604Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"client":"1","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102782024-02-03T17:13:59.240Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.794855864Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102792024-02-03T17:13:59.240Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.794876817Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"client":"0","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102802024-02-03T17:13:59.240Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.794924641Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"client":"0","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102812024-02-03T17:13:59.240Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.794949278Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102822024-02-03T17:13:59.240Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.794971635Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"client":"2","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102832024-02-03T17:13:59.240Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.794995423Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"client":"2","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102842024-02-03T17:13:59.241Z{"msg":"[0]R flush_numbers[0..12]: [160, 160, 154, 156, 157, 157, 155, 155, 160, 154, 157, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.795059353Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"":"downstairs","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102852024-02-03T17:13:59.241Z{"msg":"[0]R generation[0..12]: [92, 92, 88, 90, 90, 90, 90, 90, 92, 88, 90, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.795086753Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"":"downstairs","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102862024-02-03T17:13:59.241Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.795136002Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"":"downstairs","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102872024-02-03T17:13:59.241Z{"msg":"[1]R flush_numbers[0..12]: [160, 160, 154, 156, 157, 157, 155, 155, 160, 154, 157, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.795161275Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"":"downstairs","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102882024-02-03T17:13:59.241Z{"msg":"[1]R generation[0..12]: [92, 92, 88, 90, 90, 90, 90, 90, 92, 88, 90, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.795183838Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"":"downstairs","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102892024-02-03T17:13:59.241Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.795207207Z"Wait for a query_work_queue command to finish before sending IO
102902024-02-03T17:13:59.241Z,"hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"":"downstairs","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102912024-02-03T17:13:59.241Z{"msg":"[2]R flush_numbers[0..12]: [160, 160, 154, 156, 157, 157, 155, 155, 160, 154, 157, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.795427693Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"":"downstairs","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102922024-02-03T17:13:59.241Z{"msg":"[2]R generation[0..12]: [92, 92, 88, 90, 90, 90, 90, 90, 92, 88, 90, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.795476409Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"":"downstairs","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102932024-02-03T17:13:59.241Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.795543218Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"":"downstairs","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102942024-02-03T17:13:59.241Z{"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.795586383Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"":"downstairs","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102952024-02-03T17:13:59.241Z{"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.795650103Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"":"downstairs","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102962024-02-03T17:13:59.241Z{"msg":"Next flush: 163","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.795692351Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"":"downstairs","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102972024-02-03T17:13:59.241Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.795756954Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"":"downstairs","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102982024-02-03T17:13:59.241Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.795799977Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
102992024-02-03T17:13:59.241Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.795861634Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"client":"0","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
103002024-02-03T17:13:59.241Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.795904537Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"client":"1","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
103012024-02-03T17:13:59.241Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.795969027Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"client":"2","":"downstairs","session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
103022024-02-03T17:13:59.241Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.796012422Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
103032024-02-03T17:13:59.241Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.79605208Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
103042024-02-03T17:13:59.241ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
103052024-02-03T17:13:59.241Z{"msg":"d81b7adf-d9d4-4f87-a2e3-c18c207d2285 is now active with session: ca641b9c-f83f-45bd-8668-a1cb60ddff42","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.796118746Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
103062024-02-03T17:13:59.241Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.796158775Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508,"session_id":"ca641b9c-f83f-45bd-8668-a1cb60ddff42"}
103072024-02-03T17:13:59.242Z{"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:13:58.79622034Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508}
103082024-02-03T17:13:59.242ZLoading write count information from file "/tmp/test_repair_verify.data"
103092024-02-03T17:13:59.242ZRun Repair workload
103102024-02-03T17:13:59.242Z01/30 Write block 156 len 4096 data: 10
103112024-02-03T17:13:59.245Z02/30 Write block 19 len 12288 data: 13 10 13
103122024-02-03T17:13:59.245Z03/30 Flush
103132024-02-03T17:13:59.245Z04/30 Write block 8 len 12288 data: 8 9 9
103142024-02-03T17:13:59.245Z05/30 Read block 117 len 12288
103152024-02-03T17:13:59.248Z06/30 Write block 255 len 32768 data: 4 6 6 8 7 9 10 9
103162024-02-03T17:13:59.248Z07/30 Flush
103172024-02-03T17:13:59.251Z08/30 Write block 542 len 12288 data: 8 6 5
103182024-02-03T17:13:59.254Z09/30 Read block 524 len 4096
103192024-02-03T17:13:59.254Z10/30 Read block 32 len 24576
103202024-02-03T17:13:59.257Z11/30 Write block 541 len 12288 data: 7 9 7
103212024-02-03T17:13:59.257Z12/30 Write block 112 len 24576 data: 7 5 7 6 7 10
103222024-02-03T17:13:59.257Z13/30 Read block 398 len 20480
103232024-02-03T17:13:59.263Z14/30 Read block 58 len 4096
103242024-02-03T17:13:59.263Z15/30 Write block 309 len 32768 data: 8 10 8 9 10 10 10 10
103252024-02-03T17:13:59.287Z16/30 Read block 468 len 40960
103262024-02-03T17:13:59.287Z17/30 Read block 283 len 12288
103272024-02-03T17:13:59.287Z18/30 Write block 315 len 24576 data: 11 11 12 9 11 9
103282024-02-03T17:13:59.293Z19/30 Read block 435 len 28672
103292024-02-03T17:13:59.293Z20/30 Write block 180 len 16384 data: 6 8 7 10
103302024-02-03T17:13:59.293Z21/30 Read block 544 len 40960
103312024-02-03T17:13:59.293Z22/30 Read block 11 len 20480
103322024-02-03T17:13:59.294Z23/30 Read block 215 len 24576
103332024-02-03T17:13:59.294Z24/30 Read block 392 len 36864
103342024-02-03T17:13:59.294Z25/30 Read block 375 len 20480
103352024-02-03T17:13:59.294Z26/30 Read block 528 len 36864
103362024-02-03T17:13:59.294Z27/30 Write block 188 len 12288 data: 17 17 15
103372024-02-03T17:13:59.295Z28/30 Read block 18 len 8192
103382024-02-03T17:13:59.295Z29/30 Read block 481 len 32768
103392024-02-03T17:13:59.295Z30/30 Read block 561 len 16384
103402024-02-03T17:13:59.295Z----------------------------------------------------------------
103412024-02-03T17:13:59.295Z Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:27
103422024-02-03T17:13:59.295ZStates: Active Active Active
103432024-02-03T17:13:59.295ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
103442024-02-03T17:13:59.295Z 4 Acked 1003 Write 3 Done Done Sent false
103452024-02-03T17:13:59.296Z 5 Acked 1004 Read 3 Done Done Sent false
103462024-02-03T17:13:59.296Z 6 Acked 1005 Write 8 Done Done Sent false
103472024-02-03T17:13:59.296Z 7 Acked 1006 Flush 0 Done Done Sent false
103482024-02-03T17:13:59.296Z 8 Acked 1007 Write 3 Done Done Sent false
103492024-02-03T17:13:59.296Z 9 Acked 1008 Read 1 Done Done Sent false
103502024-02-03T17:13:59.296Z 10 Acked 1009 Read 6 Done Done Sent false
103512024-02-03T17:13:59.297Z 11 Acked 1010 Write 3 Done Done Sent false
103522024-02-03T17:13:59.297Z 12 Acked 1011 Write 6 Done Done Sent false
103532024-02-03T17:13:59.297Z 13 Acked 1012 Read 5 Done Done Sent false
103542024-02-03T17:13:59.297Z 14 Acked 1013 Read 1 Done Done Sent false
103552024-02-03T17:13:59.297Z 15 Acked 1014 Write 8 Done Done Sent false
103562024-02-03T17:13:59.297Z 16 Acked 1015 Read 10 Done Done Sent false
103572024-02-03T17:13:59.297Z 17 Acked 1016 Read 3 Done Done Sent false
103582024-02-03T17:13:59.297Z 18 Acked 1017 Write 6 Done Done Sent false
103592024-02-03T17:13:59.297Z 19 Acked 1018 Read 7 Done Done Sent false
103602024-02-03T17:13:59.297Z 20 Acked 1019 Write 4 Done Done Sent false
103612024-02-03T17:13:59.297Z 21 Acked 1020 Read 10 Done Done Sent false
103622024-02-03T17:13:59.297Z 22 Acked 1021 Read 5 Done Done Sent false
103632024-02-03T17:13:59.297Z 23 Acked 1022 Read 6 Done Done Sent false
103642024-02-03T17:13:59.297Z 24 Acked 1023 Read 9 Done Done Sent false
103652024-02-03T17:13:59.297Z 25 Acked 1024 Read 5 Done Done Sent false
103662024-02-03T17:13:59.297Z 26 Acked 1025 Read 9 Done Done Sent false
103672024-02-03T17:13:59.297Z 27 Acked 1026 Write 3 Done Done Sent false
103682024-02-03T17:13:59.297Z 28 Acked 1027 Read 2 Done Done Sent false
103692024-02-03T17:13:59.297Z 29 Acked 1028 Read 8 Done Done Sent false
103702024-02-03T17:13:59.297Z 30 Acked 1029 Read 4 Done Sent Sent false
103712024-02-03T17:13:59.298Z STATES DS:0 DS:1 DS:2 TOTAL
103722024-02-03T17:13:59.298Z New 0 0 0 0
103732024-02-03T17:13:59.298Z Sent 0 1 27 28
103742024-02-03T17:13:59.298Z Done 27 26 0 53
103752024-02-03T17:13:59.298Z Skipped 0 0 0 0
103762024-02-03T17:13:59.298Z Error 0 0 0 0
103772024-02-03T17:13:59.298ZLast Flush: 1006 1006 1002
103782024-02-03T17:13:59.298ZDownstairs last five completed: 1002 1001 1000
103792024-02-03T17:13:59.298ZUpstairs last five completed: 30 29 28 27 26
103802024-02-03T17:13:59.298Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:13:58.843964658Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1508}
103812024-02-03T17:13:59.298ZWrote out file "/tmp/test_repair_verify.data"
103822024-02-03T17:13:59.301Z
103832024-02-03T17:13:59.301ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
103842024-02-03T17:13:59.324ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
103852024-02-03T17:13:59.324Z 0 000-019 94 94 94 164 164 163 F F F <---
103862024-02-03T17:13:59.324Z 1 020-039 94 94 94 163 163 163 F F F
103872024-02-03T17:13:59.324Z 2 040-059 88 88 88 154 154 154 F F F
103882024-02-03T17:13:59.324Z 3 060-079 90 90 90 156 156 156 F F F
103892024-02-03T17:13:59.324Z 4 080-099 90 90 90 157 157 157 F F F
103902024-02-03T17:13:59.328Z 5 100-119 90 90 90 157 157 157 T T F <---
103912024-02-03T17:13:59.328Z 6 120-139 90 90 90 155 155 155 F F F
103922024-02-03T17:13:59.328Z 7 140-159 94 94 94 163 163 163 F F F
103932024-02-03T17:13:59.328Z 8 160-179 92 92 92 160 160 160 F F F
103942024-02-03T17:13:59.328Z 9 180-199 88 88 88 154 154 154 T T F <---
103952024-02-03T17:13:59.328Z 10 200-219 90 90 90 157 157 157 F F F
103962024-02-03T17:13:59.328Z 11 220-239 82 82 82 141 141 141 F F F
103972024-02-03T17:13:59.328Z 12 240-259 94 94 88 164 164 154 F F F <---
103982024-02-03T17:13:59.328Z 13 260-279 94 94 92 164 164 160 F F F <---
103992024-02-03T17:13:59.328Z 14 280-299 91 91 91 159 159 159 F F F
104002024-02-03T17:13:59.328Z 15 300-319 91 91 91 159 159 159 T T F <---
104012024-02-03T17:13:59.328Z 16 320-339 84 84 84 146 146 146 T T F <---
104022024-02-03T17:13:59.328Z 17 340-359 91 91 91 159 159 159 F F F
104032024-02-03T17:13:59.328Z 18 360-379 92 92 92 160 160 160 F F F
104042024-02-03T17:13:59.328Z 19 380-399 93 93 93 162 162 162 F F F
104052024-02-03T17:13:59.328Z 20 400-419 87 87 87 153 153 153 F F F
104062024-02-03T17:13:59.328Z 21 420-439 88 88 88 154 154 154 F F F
104072024-02-03T17:13:59.328Z 22 440-459 93 93 93 162 162 162 F F F
104082024-02-03T17:13:59.328Z 23 460-479 93 93 93 162 162 162 F F F
104092024-02-03T17:13:59.328Z 24 480-499 93 93 93 162 162 162 F F F
104102024-02-03T17:13:59.328Z 25 500-519 91 91 91 159 159 159 F F F
104112024-02-03T17:13:59.329Z 26 520-539 93 93 93 162 162 162 F F F
104122024-02-03T17:13:59.329Z 27 540-559 93 93 93 162 162 162 T T F <---
104132024-02-03T17:13:59.329Z 28 560-579 92 92 92 161 161 161 F F F
104142024-02-03T17:13:59.329Z 29 580-599 82 82 82 142 142 142 F F F
104152024-02-03T17:13:59.329ZMax gen: 94, Max flush: 164
104162024-02-03T17:13:59.329ZError: Difference in extent metadata found!
104172024-02-03T17:13:59.332ZA Difference in extent metadata is expected here
104182024-02-03T17:13:59.332ZOn loop 47
104192024-02-03T17:13:59.332Z
104202024-02-03T17:13:59.335ZVerifying data now
104212024-02-03T17:14:04.111ZNo change after verify
104222024-02-03T17:14:04.111ZLoop: 47 Downstairs dump after verify (and repair):
104232024-02-03T17:14:04.135ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
104242024-02-03T17:14:04.135Z 0 000-019 94 94 94 164 164 164 F F F
104252024-02-03T17:14:04.139Z 1 020-039 94 94 94 163 163 163 F F F
104262024-02-03T17:14:04.139Z 2 040-059 88 88 88 154 154 154 F F F
104272024-02-03T17:14:04.139Z 3 060-079 90 90 90 156 156 156 F F F
104282024-02-03T17:14:04.139Z 4 080-099 90 90 90 157 157 157 F F F
104292024-02-03T17:14:04.139Z 5 100-119 95 95 95 165 165 165 F F F
104302024-02-03T17:14:04.139Z 6 120-139 90 90 90 155 155 155 F F F
104312024-02-03T17:14:04.140Z 7 140-159 94 94 94 163 163 163 F F F
104322024-02-03T17:14:04.140Z 8 160-179 92 92 92 160 160 160 F F F
104332024-02-03T17:14:04.140Z 9 180-199 95 95 95 165 165 165 F F F
104342024-02-03T17:14:04.140Z 10 200-219 90 90 90 157 157 157 F F F
104352024-02-03T17:14:04.140Z 11 220-239 82 82 82 141 141 141 F F F
104362024-02-03T17:14:04.140Z 12 240-259 94 94 94 164 164 164 F F F
104372024-02-03T17:14:04.140Z 13 260-279 94 94 94 164 164 164 F F F
104382024-02-03T17:14:04.140Z 14 280-299 91 91 91 159 159 159 F F F
104392024-02-03T17:14:04.140Z 15 300-319 95 95 95 165 165 165 F F F
104402024-02-03T17:14:04.140Z 16 320-339 95 95 95 165 165 165 F F F
104412024-02-03T17:14:04.140Z 17 340-359 91 91 91 159 159 159 F F F
104422024-02-03T17:14:04.140Z 18 360-379 92 92 92 160 160 160 F F F
104432024-02-03T17:14:04.140Z 19 380-399 93 93 93 162 162 162 F F F
104442024-02-03T17:14:04.141Z 20 400-419 87 87 87 153 153 153 F F F
104452024-02-03T17:14:04.141Z 21 420-439 88 88 88 154 154 154 F F F
104462024-02-03T17:14:04.141Z 22 440-459 93 93 93 162 162 162 F F F
104472024-02-03T17:14:04.141Z 23 460-479 93 93 93 162 162 162 F F F
104482024-02-03T17:14:04.141Z 24 480-499 93 93 93 162 162 162 F F F
104492024-02-03T17:14:04.141Z 25 500-519 91 91 91 159 159 159 F F F
104502024-02-03T17:14:04.141Z 26 520-539 93 93 93 162 162 162 F F F
104512024-02-03T17:14:04.141Z 27 540-559 95 95 95 165 165 165 F F F
104522024-02-03T17:14:04.141Z 28 560-579 92 92 92 161 161 161 F F F
104532024-02-03T17:14:04.141Z 29 580-599 82 82 82 142 142 142 F F F
104542024-02-03T17:14:04.141ZMax gen: 95, Max flush: 165
104552024-02-03T17:14:04.145Z
104562024-02-03T17:14:04.145ZBegin loop 48
104572024-02-03T17:14:04.145ZDownstairs to restart: 1
104582024-02-03T17:14:04.181Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.729998354Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516}
104592024-02-03T17:14:04.181Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.730481697Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516}
104602024-02-03T17:14:04.181Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.730516302Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516}
104612024-02-03T17:14:04.189Z{"msg":"Crucible 820ef6d6-748b-4580-b7d4-41a30bbaa8f0 has session id: 4c689891-037a-4b49-9ff7-28020e104609","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.730591342Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516Crucible runtime is spawned
104622024-02-03T17:14:04.189Z,"session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104632024-02-03T17:14:04.189Z{"msg":"Upstairs opts: Upstairs UUID: 820ef6d6-748b-4580-b7d4-41a30bbaa8f0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.730899048Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104642024-02-03T17:14:04.189Z{"msg":"Crucible stats registered with UUID: 820ef6d6-748b-4580-b7d4-41a30bbaa8f0","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.730954398Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104652024-02-03T17:14:04.189Z{"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.730991409Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516}
104662024-02-03T17:14:04.189Z{"msg":"820ef6d6-748b-4580-b7d4-41a30bbaa8f0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.731124512Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104672024-02-03T17:14:04.189Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.731194789Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"client":"0","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104682024-02-03T17:14:04.189Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.731223201Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"client":"0","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104692024-02-03T17:14:04.190Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.731246258Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"client":"1","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104702024-02-03T17:14:04.190Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.731270798Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"client":"1","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104712024-02-03T17:14:04.190Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.731310663Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"client":"2","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104722024-02-03T17:14:04.190Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.731355141Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"client":"2","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104732024-02-03T17:14:04.190Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.73142311Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"":"io task","client":"2","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104742024-02-03T17:14:04.190Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.731554073Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"":"io task","client":"1","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104752024-02-03T17:14:04.190Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.731781913Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"":"io task","client":"0","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104762024-02-03T17:14:04.190Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.731829269Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"":"io task","client":"0","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104772024-02-03T17:14:04.190Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.731915573Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"":"io task","client":"2","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104782024-02-03T17:14:04.190Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.732000741Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"":"io task","client":"1","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104792024-02-03T17:14:04.190Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.733214078Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"client":"2","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104802024-02-03T17:14:04.190Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.733346535Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"client":"2","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104812024-02-03T17:14:04.190Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.733561479Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"client":"0","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104822024-02-03T17:14:04.190Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.733633803Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"client":"0","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104832024-02-03T17:14:04.191Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.733660476Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"client":"1","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104842024-02-03T17:14:04.191Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.733682565Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"client":"1","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104852024-02-03T17:14:04.191Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.734162537Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"client":"2","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104862024-02-03T17:14:04.191Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.73445233Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"client":"0","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104872024-02-03T17:14:04.191Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.734883019Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"client":"1","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104882024-02-03T17:14:04.191Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.735121555Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"client":"2","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104892024-02-03T17:14:04.191Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.735317589Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"client":"2","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104902024-02-03T17:14:04.191Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.735391188Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104912024-02-03T17:14:04.191Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.73541592Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"client":"0","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104922024-02-03T17:14:04.191Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.735489329Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"client":"0","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104932024-02-03T17:14:04.191Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.735528196Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104942024-02-03T17:14:04.191Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.735777753Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"client":"1","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104952024-02-03T17:14:04.191Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.735876297Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"client":"1","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104962024-02-03T17:14:04.191Z{"msg":"[0]R flush_numbers[0..12]: [164, 163, 154, 156, 157, 165, 155, 163, 160, 165, 157, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.735962239Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"":"downstairs","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104972024-02-03T17:14:04.192Z{"msg":"[0]R generation[0..12]: [94, 94, 88, 90, 90, 95, 90, 94, 92, 95, 90, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.736045444Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"":"downstairs","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
104982024-02-03T17:14:04.192Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO
104992024-02-03T17:14:04.192Z,"level":30,"time":"2024-02-03T17:14:03.736134265Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"":"downstairs","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
105002024-02-03T17:14:04.192Z{"msg":"[1]R flush_numbers[0..12]: [164, 163, 154, 156, 157, 165, 155, 163, 160, 165, 157, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.736184327Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"":"downstairs","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
105012024-02-03T17:14:04.192Z{"msg":"[1]R generation[0..12]: [94, 94, 88, 90, 90, 95, 90, 94, 92, 95, 90, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.736263771Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"":"downstairs","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
105022024-02-03T17:14:04.192Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.736313345Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"":"downstairs","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
105032024-02-03T17:14:04.192Z{"msg":"[2]R flush_numbers[0..12]: [164, 163, 154, 156, 157, 165, 155, 163, 160, 165, 157, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.736398471Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"":"downstairs","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
105042024-02-03T17:14:04.192Z{"msg":"[2]R generation[0..12]: [94, 94, 88, 90, 90, 95, 90, 94, 92, 95, 90, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.736431306Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"":"downstairs","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
105052024-02-03T17:14:04.192Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.736485222Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"":"downstairs","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
105062024-02-03T17:14:04.192Z{"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.736509404Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"":"downstairs","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
105072024-02-03T17:14:04.192Z{"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.736531312Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"":"downstairs","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
105082024-02-03T17:14:04.192Z{"msg":"Next flush: 166","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.736580175Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"":"downstairs","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
105092024-02-03T17:14:04.192Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.736604342Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"":"downstairs","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
105102024-02-03T17:14:04.192Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.736626343Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
105112024-02-03T17:14:04.192Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.736673575Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"client":"0","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
105122024-02-03T17:14:04.192Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.736697204Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"client":"1","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
105132024-02-03T17:14:04.192Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.736718776Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"client":"2","":"downstairs","session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
105142024-02-03T17:14:04.192Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.736768763Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
105152024-02-03T17:14:04.192Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.736791247Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
105162024-02-03T17:14:04.192Z{"msg":"820ef6d6-748b-4580-b7d4-41a30bbaa8f0 is now active with session: 4c689891-037a-4b49-9ff7-28020e104609","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.736812835Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
105172024-02-03T17:14:04.192Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.736832379Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516,"session_id":"4c689891-037a-4b49-9ff7-28020e104609"}
105182024-02-03T17:14:04.192Z{"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:03.736852591Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516}
105192024-02-03T17:14:04.192ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
105202024-02-03T17:14:04.193ZLoading write count information from file "/tmp/test_repair_verify.data"
105212024-02-03T17:14:04.193ZRun Repair workload
105222024-02-03T17:14:04.193Z01/30 Write block 116 len 8192 data: 8 11
105232024-02-03T17:14:04.193Z02/30 Write block 590 len 20480 data: 7 8 9 9 8
105242024-02-03T17:14:04.193Z03/30 Read block 339 len 40960
105252024-02-03T17:14:04.193Z04/30 Write block 359 len 4096 data: 10
105262024-02-03T17:14:04.197Z05/30 Write block 260 len 12288 data: 10 11 10
105272024-02-03T17:14:04.197Z06/30 Read block 344 len 12288
105282024-02-03T17:14:04.197Z07/30 Write block 1 len 36864 data: 3 5 6 7 7 9 7 9 10
105292024-02-03T17:14:04.198Z08/30 Write block 309 len 16384 data: 9 11 9 10
105302024-02-03T17:14:04.198Z09/30 Read block 83 len 12288
105312024-02-03T17:14:04.198Z10/30 Read block 432 len 24576
105322024-02-03T17:14:04.198Z11/30 Read block 459 len 36864
105332024-02-03T17:14:04.198Z12/30 Read block 57 len 20480
105342024-02-03T17:14:04.202Z13/30 Flush
105352024-02-03T17:14:04.215Z14/30 Write block 101 len 28672 data: 8 5 5 5 5 7 8
105362024-02-03T17:14:04.215Z15/30 Write block 320 len 32768 data: 10 10 10 11 9 8 5 8
105372024-02-03T17:14:04.219Z16/30 Write block 159 len 36864 data: 7 11 10 13 14 14 14 16 16
105382024-02-03T17:14:04.219Z17/30 Write block 208 len 20480 data: 7 7 9 8 8
105392024-02-03T17:14:04.219Z18/30 Flush
105402024-02-03T17:14:04.219Z19/30 Write block 205 len 16384 data: 9 8 8 8
105412024-02-03T17:14:04.220Z20/30 Write block 458 len 8192 data: 6 8
105422024-02-03T17:14:04.220Z21/30 Write block 189 len 32768 data: 18 16 12 12 9 9 9 8
105432024-02-03T17:14:04.220Z22/30 Write block 514 len 28672 data: 8 10 9 8 5 5 6
105442024-02-03T17:14:04.220Z23/30 Read block 551 len 24576
105452024-02-03T17:14:04.220Z24/30 Read block 566 len 16384
105462024-02-03T17:14:04.220Z25/30 Read block 343 len 24576
105472024-02-03T17:14:04.223Z26/30 Write block 90 len 8192 data: 9 8
105482024-02-03T17:14:04.224Z27/30 Write block 185 len 12288 data: 15 12 15
105492024-02-03T17:14:04.224Z28/30 Read block 228 len 20480
105502024-02-03T17:14:04.227Z29/30 Read block 206 len 20480
105512024-02-03T17:14:04.227Z30/30 Write block 59 len 32768 data: 7 7 5 7 5 3 4 5
105522024-02-03T17:14:04.231Z----------------------------------------------------------------
105532024-02-03T17:14:04.231Z Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:30
105542024-02-03T17:14:04.231ZStates: Active Active Active
105552024-02-03T17:14:04.232ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
105562024-02-03T17:14:04.232Z 1 Acked 1000 Write 2 Done Done Done false
105572024-02-03T17:14:04.232Z 2 Acked 1001 Write 5 Done Sent Done false
105582024-02-03T17:14:04.232Z 3 Acked 1002 Read 10 Done Sent Done false
105592024-02-03T17:14:04.232Z 4 Acked 1003 Write 1 Done Sent Done false
105602024-02-03T17:14:04.232Z 5 Acked 1004 Write 3 Done Sent Done false
105612024-02-03T17:14:04.232Z 6 Acked 1005 Read 3 Done Sent Done false
105622024-02-03T17:14:04.232Z 7 Acked 1006 Write 9 Done Sent Done false
105632024-02-03T17:14:04.232Z 8 Acked 1007 Write 4 Done Sent Done false
105642024-02-03T17:14:04.232Z 9 Acked 1008 Read 3 Done Sent Done false
105652024-02-03T17:14:04.232Z 10 Acked 1009 Read 6 Done Sent Done false
105662024-02-03T17:14:04.232Z 11 Acked 1010 Read 9 Done Sent Done false
105672024-02-03T17:14:04.233Z 12 Acked 1011 Read 5 Done Sent Done false
105682024-02-03T17:14:04.233Z 13 Acked 1012 Flush 0 Done Sent Done false
105692024-02-03T17:14:04.233Z 14 Acked 1013 Write 7 Done Sent Done false
105702024-02-03T17:14:04.233Z 15 Acked 1014 Write 8 Done Sent Done false
105712024-02-03T17:14:04.233Z 16 Acked 1015 Write 9 Done Sent Done false
105722024-02-03T17:14:04.233Z 17 Acked 1016 Write 5 Done Sent Done false
105732024-02-03T17:14:04.233Z 18 Acked 1017 Flush 0 Done Sent Done false
105742024-02-03T17:14:04.233Z 19 Acked 1018 Write 4 Done Sent Done false
105752024-02-03T17:14:04.233Z 20 Acked 1019 Write 2 Done Sent Done false
105762024-02-03T17:14:04.233Z 21 Acked 1020 Write 8 Done Sent Done false
105772024-02-03T17:14:04.233Z 22 Acked 1021 Write 7 Done Sent Done false
105782024-02-03T17:14:04.233Z 23 Acked 1022 Read 6 Done Sent Done false
105792024-02-03T17:14:04.233Z 24 Acked 1023 Read 4 Done Sent Done false
105802024-02-03T17:14:04.234Z 25 Acked 1024 Read 6 Done Sent Done false
105812024-02-03T17:14:04.234Z 26 Acked 1025 Write 2 Done Sent Done false
105822024-02-03T17:14:04.234Z 27 Acked 1026 Write 3 Done Sent Done false
105832024-02-03T17:14:04.234Z 28 Acked 1027 Read 5 Done Sent Done false
105842024-02-03T17:14:04.234Z 29 Acked 1028 Read 5 Done Sent Done false
105852024-02-03T17:14:04.234Z 30 Acked 1029 Write 8 Sent Sent Sent false
105862024-02-03T17:14:04.234Z STATES DS:0 DS:1 DS:2 TOTAL
105872024-02-03T17:14:04.234Z New 0 0 0 0
105882024-02-03T17:14:04.234Z Sent 1 29 1 31
105892024-02-03T17:14:04.234Z Done 29 1 29 59
105902024-02-03T17:14:04.234Z Skipped 0 0 0 0
105912024-02-03T17:14:04.234Z Error 0 0 0 0
105922024-02-03T17:14:04.234ZLast Flush: 1017 0 1017
105932024-02-03T17:14:04.235ZDownstairs last five completed:
105942024-02-03T17:14:04.235ZUpstairs last five completed: 30 29 28 27 26
105952024-02-03T17:14:04.235Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:14:03.787798232Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1516}
105962024-02-03T17:14:04.235ZWrote out file "/tmp/test_repair_verify.data"
105972024-02-03T17:14:04.239Z
105982024-02-03T17:14:04.244ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
105992024-02-03T17:14:04.270ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
106002024-02-03T17:14:04.270Z 0 000-019 96 94 96 166 164 166 F F F <---
106012024-02-03T17:14:04.271Z 1 020-039 94 94 94 163 163 163 F F F
106022024-02-03T17:14:04.271Z 2 040-059 88 88 88 154 154 154 T F T <---
106032024-02-03T17:14:04.271Z 3 060-079 90 90 90 156 156 156 T F T <---
106042024-02-03T17:14:04.271Z 4 080-099 90 90 90 157 157 157 T F T <---
106052024-02-03T17:14:04.275Z 5 100-119 96 95 96 167 165 167 F T F <---
106062024-02-03T17:14:04.275Z 6 120-139 90 90 90 155 155 155 F F F
106072024-02-03T17:14:04.275Z 7 140-159 96 94 96 167 163 167 F F F <---
106082024-02-03T17:14:04.275Z 8 160-179 96 92 96 167 160 167 F F F <---
106092024-02-03T17:14:04.276Z 9 180-199 95 95 95 165 165 165 T F T <---
106102024-02-03T17:14:04.276Z 10 200-219 96 90 96 167 157 167 T F T <---
106112024-02-03T17:14:04.276Z 11 220-239 82 82 82 141 141 141 F F F
106122024-02-03T17:14:04.276Z 12 240-259 94 94 94 164 164 164 F F F
106132024-02-03T17:14:04.276Z 13 260-279 96 94 96 166 164 166 F F F <---
106142024-02-03T17:14:04.276Z 14 280-299 91 91 91 159 159 159 F F F
106152024-02-03T17:14:04.276Z 15 300-319 96 95 96 166 165 166 F F F <---
106162024-02-03T17:14:04.276Z 16 320-339 96 95 96 167 165 167 F F F <---
106172024-02-03T17:14:04.276Z 17 340-359 96 91 96 166 159 166 F F F <---
106182024-02-03T17:14:04.276Z 18 360-379 92 92 92 160 160 160 F F F
106192024-02-03T17:14:04.276Z 19 380-399 93 93 93 162 162 162 F F F
106202024-02-03T17:14:04.276Z 20 400-419 87 87 87 153 153 153 F F F
106212024-02-03T17:14:04.276Z 21 420-439 88 88 88 154 154 154 F F F
106222024-02-03T17:14:04.276Z 22 440-459 93 93 93 162 162 162 T F T <---
106232024-02-03T17:14:04.277Z 23 460-479 93 93 93 162 162 162 F F F
106242024-02-03T17:14:04.277Z 24 480-499 93 93 93 162 162 162 F F F
106252024-02-03T17:14:04.277Z 25 500-519 91 91 91 159 159 159 T F T <---
106262024-02-03T17:14:04.277Z 26 520-539 93 93 93 162 162 162 T F T <---
106272024-02-03T17:14:04.277Z 27 540-559 95 95 95 165 165 165 F F F
106282024-02-03T17:14:04.277Z 28 560-579 92 92 92 161 161 161 F F F
106292024-02-03T17:14:04.277Z 29 580-599 96 82 96 166 142 166 F F F <---
106302024-02-03T17:14:04.277ZMax gen: 96, Max flush: 167
106312024-02-03T17:14:04.277ZError: Difference in extent metadata found!
106322024-02-03T17:14:04.281ZA Difference in extent metadata is expected here
106332024-02-03T17:14:04.281ZOn loop 48
106342024-02-03T17:14:04.281Z
106352024-02-03T17:14:04.281ZVerifying data now
106362024-02-03T17:14:09.719ZNo change after verify
106372024-02-03T17:14:09.719ZLoop: 48 Downstairs dump after verify (and repair):
106382024-02-03T17:14:09.744ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
106392024-02-03T17:14:09.745Z 0 000-019 96 96 96 166 166 166 F F F
106402024-02-03T17:14:09.745Z 1 020-039 94 94 94 163 163 163 F F F
106412024-02-03T17:14:09.745Z 2 040-059 97 97 97 168 168 168 F F F
106422024-02-03T17:14:09.745Z 3 060-079 97 97 97 168 168 168 F F F
106432024-02-03T17:14:09.746Z 4 080-099 97 97 97 168 168 168 F F F
106442024-02-03T17:14:09.746Z 5 100-119 96 96 96 167 167 167 F F F
106452024-02-03T17:14:09.746Z 6 120-139 90 90 90 155 155 155 F F F
106462024-02-03T17:14:09.746Z 7 140-159 96 96 96 167 167 167 F F F
106472024-02-03T17:14:09.752Z 8 160-179 96 96 96 167 167 167 F F F
106482024-02-03T17:14:09.753Z 9 180-199 97 97 97 168 168 168 F F F
106492024-02-03T17:14:09.753Z 10 200-219 97 97 97 168 168 168 F F F
106502024-02-03T17:14:09.753Z 11 220-239 82 82 82 141 141 141 F F F
106512024-02-03T17:14:09.753Z 12 240-259 94 94 94 164 164 164 F F F
106522024-02-03T17:14:09.754Z 13 260-279 96 96 96 166 166 166 F F F
106532024-02-03T17:14:09.754Z 14 280-299 91 91 91 159 159 159 F F F
106542024-02-03T17:14:09.754Z 15 300-319 96 96 96 166 166 166 F F F
106552024-02-03T17:14:09.754Z 16 320-339 96 96 96 167 167 167 F F F
106562024-02-03T17:14:09.754Z 17 340-359 96 96 96 166 166 166 F F F
106572024-02-03T17:14:09.755Z 18 360-379 92 92 92 160 160 160 F F F
106582024-02-03T17:14:09.755Z 19 380-399 93 93 93 162 162 162 F F F
106592024-02-03T17:14:09.755Z 20 400-419 87 87 87 153 153 153 F F F
106602024-02-03T17:14:09.755Z 21 420-439 88 88 88 154 154 154 F F F
106612024-02-03T17:14:09.755Z 22 440-459 97 97 97 168 168 168 F F F
106622024-02-03T17:14:09.756Z 23 460-479 93 93 93 162 162 162 F F F
106632024-02-03T17:14:09.756Z 24 480-499 93 93 93 162 162 162 F F F
106642024-02-03T17:14:09.756Z 25 500-519 97 97 97 168 168 168 F F F
106652024-02-03T17:14:09.756Z 26 520-539 97 97 97 168 168 168 F F F
106662024-02-03T17:14:09.757Z 27 540-559 95 95 95 165 165 165 F F F
106672024-02-03T17:14:09.757Z 28 560-579 92 92 92 161 161 161 F F F
106682024-02-03T17:14:09.757Z 29 580-599 96 96 96 166 166 166 F F F
106692024-02-03T17:14:09.757ZMax gen: 97, Max flush: 168
106702024-02-03T17:14:09.758Z
106712024-02-03T17:14:09.758ZBegin loop 49
106722024-02-03T17:14:09.758ZDownstairs to restart: 0
106732024-02-03T17:14:09.781Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.332267001Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524}
106742024-02-03T17:14:09.781Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.332772771Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524}
106752024-02-03T17:14:09.817Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
106762024-02-03T17:14:09.817Z,"time":"2024-02-03T17:14:09.332904285Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524}
106772024-02-03T17:14:09.818Z{"msg":"Crucible f79ea7af-33e4-4b38-a8ce-b25949fcf536 has session id: e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.333212995Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
106782024-02-03T17:14:09.818Z{"msg":"Upstairs opts: Upstairs UUID: f79ea7af-33e4-4b38-a8ce-b25949fcf536, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.333297145Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
106792024-02-03T17:14:09.818Z{"msg":"Crucible stats registered with UUID: f79ea7af-33e4-4b38-a8ce-b25949fcf536","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.333325532Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
106802024-02-03T17:14:09.818Z{"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.333372939Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524}
106812024-02-03T17:14:09.818Z{"msg":"f79ea7af-33e4-4b38-a8ce-b25949fcf536 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.333393752Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
106822024-02-03T17:14:09.818Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.333421559Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"client":"0","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
106832024-02-03T17:14:09.818Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.333493492Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"client":"0","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
106842024-02-03T17:14:09.818Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.333538891Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"client":"1","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
106852024-02-03T17:14:09.819Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.333624565Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"client":"1","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
106862024-02-03T17:14:09.819Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.333665424Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"client":"2","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
106872024-02-03T17:14:09.819Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.333702673Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"client":"2","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
106882024-02-03T17:14:09.819Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.333777861Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"":"io task","client":"0","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
106892024-02-03T17:14:09.819Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.333862627Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"":"io task","client":"1","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
106902024-02-03T17:14:09.819Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.333908449Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"":"io task","client":"2","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
106912024-02-03T17:14:09.819Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.333948958Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"":"io task","client":"0","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
106922024-02-03T17:14:09.820Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.334056487Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"":"io task","client":"2","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
106932024-02-03T17:14:09.820Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.334099108Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"":"io task","client":"1","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
106942024-02-03T17:14:09.820Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.335321381Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"client":"1","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
106952024-02-03T17:14:09.820Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.335445881Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"client":"1","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
106962024-02-03T17:14:09.820Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.335488866Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"client":"2","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
106972024-02-03T17:14:09.820Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.335529836Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"client":"2","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
106982024-02-03T17:14:09.820Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.336351237Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"client":"2","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
106992024-02-03T17:14:09.821Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.336420266Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"client":"0","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107002024-02-03T17:14:09.821Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.33662148Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"client":"0","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107012024-02-03T17:14:09.821Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.336692289Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"client":"1","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107022024-02-03T17:14:09.821Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.337470442Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"client":"2","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107032024-02-03T17:14:09.821Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.337670695Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"client":"2","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107042024-02-03T17:14:09.821Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.337754004Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107052024-02-03T17:14:09.821Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.337797112Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"client":"1","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107062024-02-03T17:14:09.821Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.337868352Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"client":"1","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107072024-02-03T17:14:09.822Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.337912801Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107082024-02-03T17:14:09.822Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.337977721Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"client":"0","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107092024-02-03T17:14:09.822Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.338662726Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"client":"0","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107102024-02-03T17:14:09.822Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.33875588Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"client":"0","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107112024-02-03T17:14:09.822Z{"msg":"[0]R flush_numbers[0..12]: [166, 163, 168, 168, 168, 167, 155, 167, 167, 168, 168, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.338788735Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"":"downstairs","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107122024-02-03T17:14:09.822Z{"msg":"[0]R generation[0..12]: [96, 94, 97, 97, 97, 96, 90, 96, 96, 97, 97, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.338814715Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"":"downstairs","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107132024-02-03T17:14:09.822Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.338876033Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"":"downstairs","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107142024-02-03T17:14:09.822Z{"msg":"[1]R flush_numbers[0..12]: [166, 163, 168, 168, 168, 167, 155, 167, 167, 168, 168, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.338903276Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"":"downstairs","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107152024-02-03T17:14:09.822Z{"msg":"[1]R generation[0..12]: [96, 94, 97, 97, 97, 96, 90, 96, 96, 97, 97, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.338952343Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"":"downstairs","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107162024-02-03T17:14:09.822Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.338978538Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"":"downstairs","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107172024-02-03T17:14:09.823Z{"msg":"[2]R flush_numbers[0..12]: [166, 163, 168, 168, 168, 167, 155, 167, 167, 168, 168, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.339027406Z","hostname"Wait for a query_work_queue command to finish before sending IO
107182024-02-03T17:14:09.823Z:"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"":"downstairs","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107192024-02-03T17:14:09.823Z{"msg":"[2]R generation[0..12]: [96, 94, 97, 97, 97, 96, 90, 96, 96, 97, 97, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.339057093Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"":"downstairs","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107202024-02-03T17:14:09.823Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.339120857Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"":"downstairs","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107212024-02-03T17:14:09.823Z{"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.339187099Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"":"downstairs","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107222024-02-03T17:14:09.823Z{"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.33921333Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"":"downstairs","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107232024-02-03T17:14:09.823Z{"msg":"Next flush: 169","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.339285119Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"":"downstairs","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107242024-02-03T17:14:09.823Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.339314194Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"":"downstairs","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107252024-02-03T17:14:09.824Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.339380055Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107262024-02-03T17:14:09.824Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.339406556Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"client":"0","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107272024-02-03T17:14:09.824Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.339474226Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"client":"1","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107282024-02-03T17:14:09.824Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.339508036Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"client":"2","":"downstairs","session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107292024-02-03T17:14:09.824ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
107302024-02-03T17:14:09.824Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.339575898Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107312024-02-03T17:14:09.824Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.339616991Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107322024-02-03T17:14:09.824Z{"msg":"f79ea7af-33e4-4b38-a8ce-b25949fcf536 is now active with session: e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.339675315Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107332024-02-03T17:14:09.825Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.339698012Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524,"session_id":"e93e2b4d-181f-4759-8f03-ca3d6e4e9ea0"}
107342024-02-03T17:14:09.825Z{"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:09.339744118Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524}
107352024-02-03T17:14:09.825ZLoading write count information from file "/tmp/test_repair_verify.data"
107362024-02-03T17:14:09.825ZRun Repair workload
107372024-02-03T17:14:09.825Z01/30 Flush
107382024-02-03T17:14:09.825Z02/30 Write block 414 len 24576 data: 9 10 9 11 9 7
107392024-02-03T17:14:09.825Z03/30 Read block 281 len 20480
107402024-02-03T17:14:09.825Z04/30 Read block 167 len 20480
107412024-02-03T17:14:09.825Z05/30 Read block 511 len 28672
107422024-02-03T17:14:09.846Z06/30 Read block 253 len 32768
107432024-02-03T17:14:09.846Z07/30 Read block 135 len 40960
107442024-02-03T17:14:09.846Z08/30 Write block 324 len 40960 data: 10 9 6 9 8 9 7 7 7 7
107452024-02-03T17:14:09.846Z09/30 Read block 517 len 40960
107462024-02-03T17:14:09.847Z10/30 Flush
107472024-02-03T17:14:09.847Z11/30 Write block 17 len 8192 data: 8 11
107482024-02-03T17:14:09.847Z12/30 Read block 264 len 36864
107492024-02-03T17:14:09.847Z13/30 Write block 543 len 12288 data: 8 6 7
107502024-02-03T17:14:09.847Z14/30 Write block 122 len 40960 data: 8 7 6 5 6 7 8 9 9 9
107512024-02-03T17:14:09.847Z15/30 Read block 243 len 12288
107522024-02-03T17:14:09.847Z16/30 Read block 5 len 36864
107532024-02-03T17:14:09.848Z17/30 Write block 173 len 12288 data: 9 9 7
107542024-02-03T17:14:09.848Z18/30 Write block 466 len 28672 data: 9 8 7 7 8 6 6
107552024-02-03T17:14:09.848Z19/30 Write block 163 len 40960 data: 15 15 15 17 17 12 12 11 10 8
107562024-02-03T17:14:09.848Z20/30 Flush
107572024-02-03T17:14:09.848Z21/30 Write block 588 len 24576 data: 9 7 8 9 10 10
107582024-02-03T17:14:09.848Z22/30 Read block 44 len 28672
107592024-02-03T17:14:09.848Z23/30 Read block 153 len 28672
107602024-02-03T17:14:09.849Z24/30 Read block 418 len 20480
107612024-02-03T17:14:09.849Z25/30 Read block 337 len 28672
107622024-02-03T17:14:09.849Z26/30 Read block 403 len 28672
107632024-02-03T17:14:09.849Z27/30 Read block 381 len 20480
107642024-02-03T17:14:09.849Z28/30 Write block 345 len 28672 data: 7 6 7 9 9 6 6
107652024-02-03T17:14:09.849Z29/30 Read block 487 len 40960
107662024-02-03T17:14:09.849Z30/30 Read block 416 len 24576
107672024-02-03T17:14:09.857Z----------------------------------------------------------------
107682024-02-03T17:14:09.858Z Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30
107692024-02-03T17:14:09.858ZStates: Active Active Active
107702024-02-03T17:14:09.858ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
107712024-02-03T17:14:09.858Z 1 Acked 1000 Flush 0 Sent Done Done false
107722024-02-03T17:14:09.858Z 2 Acked 1001 Write 6 Sent Done Done false
107732024-02-03T17:14:09.858Z 3 Acked 1002 Read 5 Sent Done Done false
107742024-02-03T17:14:09.858Z 4 Acked 1003 Read 5 Sent Done Done false
107752024-02-03T17:14:09.859Z 5 Acked 1004 Read 7 Sent Done Done false
107762024-02-03T17:14:09.859Z 6 Acked 1005 Read 8 Sent Done Done false
107772024-02-03T17:14:09.859Z 7 Acked 1006 Read 10 Sent Done Done false
107782024-02-03T17:14:09.859Z 8 Acked 1007 Write 10 Sent Done Done false
107792024-02-03T17:14:09.859Z 9 Acked 1008 Read 10 Sent Done Done false
107802024-02-03T17:14:09.860Z 10 Acked 1009 Flush 0 Sent Done Done false
107812024-02-03T17:14:09.860Z 11 Acked 1010 Write 2 Sent Done Done false
107822024-02-03T17:14:09.860Z 12 Acked 1011 Read 9 Sent Done Done false
107832024-02-03T17:14:09.860Z 13 Acked 1012 Write 3 Sent Done Done false
107842024-02-03T17:14:09.860Z 14 Acked 1013 Write 10 Sent Done Done false
107852024-02-03T17:14:09.860Z 15 Acked 1014 Read 3 Sent Done Done false
107862024-02-03T17:14:09.861Z 16 Acked 1015 Read 9 Sent Done Done false
107872024-02-03T17:14:09.861Z 17 Acked 1016 Write 3 Sent Done Done false
107882024-02-03T17:14:09.861Z 18 Acked 1017 Write 7 Sent Done Done false
107892024-02-03T17:14:09.861Z 19 Acked 1018 Write 10 Sent Done Done false
107902024-02-03T17:14:09.862Z 20 Acked 1019 Flush 0 Sent Done Done false
107912024-02-03T17:14:09.862Z 21 Acked 1020 Write 6 Sent Done Done false
107922024-02-03T17:14:09.862Z 22 Acked 1021 Read 7 Sent Done Done false
107932024-02-03T17:14:09.862Z 23 Acked 1022 Read 7 Sent Done Done false
107942024-02-03T17:14:09.862Z 24 Acked 1023 Read 5 Sent Done Done false
107952024-02-03T17:14:09.862Z 25 Acked 1024 Read 7 Sent Done Done false
107962024-02-03T17:14:09.863Z 26 Acked 1025 Read 7 Sent Done Done false
107972024-02-03T17:14:09.863Z 27 Acked 1026 Read 5 Sent Done Done false
107982024-02-03T17:14:09.863Z 28 Acked 1027 Write 7 Sent Done Done false
107992024-02-03T17:14:09.863Z 29 Acked 1028 Read 10 Sent Done Done false
108002024-02-03T17:14:09.864Z 30 Acked 1029 Read 6 Sent Sent Done false
108012024-02-03T17:14:09.864Z STATES DS:0 DS:1 DS:2 TOTAL
108022024-02-03T17:14:09.864Z New 0 0 0 0
108032024-02-03T17:14:09.864Z Sent 30 1 0 31
108042024-02-03T17:14:09.864Z Done 0 29 30 59
108052024-02-03T17:14:09.864Z Skipped 0 0 0 0
108062024-02-03T17:14:09.865Z Error 0 0 0 0
108072024-02-03T17:14:09.865ZLast Flush: 0 1019 1019
108082024-02-03T17:14:09.865ZDownstairs last five completed:
108092024-02-03T17:14:09.865ZUpstairs last five completed: 30 29 28 27 26
108102024-02-03T17:14:09.865Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:14:09.393957136Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1524}
108112024-02-03T17:14:09.866ZWrote out file "/tmp/test_repair_verify.data"
108122024-02-03T17:14:09.866Z
108132024-02-03T17:14:09.866ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
108142024-02-03T17:14:09.876ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
108152024-02-03T17:14:09.876Z 0 000-019 96 98 98 166 171 171 F F F <---
108162024-02-03T17:14:09.876Z 1 020-039 94 94 94 163 163 163 F F F
108172024-02-03T17:14:09.876Z 2 040-059 97 97 97 168 168 168 F F F
108182024-02-03T17:14:09.876Z 3 060-079 97 97 97 168 168 168 F F F
108192024-02-03T17:14:09.877Z 4 080-099 97 97 97 168 168 168 F F F
108202024-02-03T17:14:09.877Z 5 100-119 96 96 96 167 167 167 F F F
108212024-02-03T17:14:09.877Z 6 120-139 90 98 98 155 171 171 F F F <---
108222024-02-03T17:14:09.877Z 7 140-159 96 96 96 167 167 167 F F F
108232024-02-03T17:14:09.877Z 8 160-179 96 98 98 167 171 171 F F F <---
108242024-02-03T17:14:09.877Z 9 180-199 97 97 97 168 168 168 F F F
108252024-02-03T17:14:09.877Z 10 200-219 97 97 97 168 168 168 F F F
108262024-02-03T17:14:09.877Z 11 220-239 82 82 82 141 141 141 F F F
108272024-02-03T17:14:09.878Z 12 240-259 94 94 94 164 164 164 F F F
108282024-02-03T17:14:09.878Z 13 260-279 96 96 96 166 166 166 F F F
108292024-02-03T17:14:09.878Z 14 280-299 91 91 91 159 159 159 F F F
108302024-02-03T17:14:09.878Z 15 300-319 96 96 96 166 166 166 F F F
108312024-02-03T17:14:09.878Z 16 320-339 96 98 98 167 170 170 F F F <---
108322024-02-03T17:14:09.879Z 17 340-359 96 96 96 166 166 166 F T T <---
108332024-02-03T17:14:09.879Z 18 360-379 92 92 92 160 160 160 F F F
108342024-02-03T17:14:09.879Z 19 380-399 93 93 93 162 162 162 F F F
108352024-02-03T17:14:09.879Z 20 400-419 87 98 98 153 170 170 F F F <---
108362024-02-03T17:14:09.879Z 21 420-439 88 88 88 154 154 154 F F F
108372024-02-03T17:14:09.880Z 22 440-459 97 97 97 168 168 168 F F F
108382024-02-03T17:14:09.880Z 23 460-479 93 98 98 162 171 171 F F F <---
108392024-02-03T17:14:09.880Z 24 480-499 93 93 93 162 162 162 F F F
108402024-02-03T17:14:09.880Z 25 500-519 97 97 97 168 168 168 F F F
108412024-02-03T17:14:09.880Z 26 520-539 97 97 97 168 168 168 F F F
108422024-02-03T17:14:09.881Z 27 540-559 95 98 98 165 171 171 F F F <---
108432024-02-03T17:14:09.881Z 28 560-579 92 92 92 161 161 161 F F F
108442024-02-03T17:14:09.881Z 29 580-599 96 96 96 166 166 166 F T T <---
108452024-02-03T17:14:09.881ZMax gen: 98, Max flush: 171
108462024-02-03T17:14:09.881ZError: Difference in extent metadata found!
108472024-02-03T17:14:09.892ZA Difference in extent metadata is expected here
108482024-02-03T17:14:09.892ZOn loop 49
108492024-02-03T17:14:09.893Z
108502024-02-03T17:14:09.893ZVerifying data now
108512024-02-03T17:14:14.714ZNo change after verify
108522024-02-03T17:14:14.714ZLoop: 49 Downstairs dump after verify (and repair):
108532024-02-03T17:14:14.741ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
108542024-02-03T17:14:14.741Z 0 000-019 98 98 98 171 171 171 F F F
108552024-02-03T17:14:14.741Z 1 020-039 94 94 94 163 163 163 F F F
108562024-02-03T17:14:14.741Z 2 040-059 97 97 97 168 168 168 F F F
108572024-02-03T17:14:14.741Z 3 060-079 97 97 97 168 168 168 F F F
108582024-02-03T17:14:14.776Z 4 080-099 97 97 97 168 168 168 F F F
108592024-02-03T17:14:14.776Z 5 100-119 96 96 96 167 167 167 F F F
108602024-02-03T17:14:14.777Z 6 120-139 98 98 98 171 171 171 F F F
108612024-02-03T17:14:14.777Z 7 140-159 96 96 96 167 167 167 F F F
108622024-02-03T17:14:14.777Z 8 160-179 98 98 98 171 171 171 F F F
108632024-02-03T17:14:14.777Z 9 180-199 97 97 97 168 168 168 F F F
108642024-02-03T17:14:14.777Z 10 200-219 97 97 97 168 168 168 F F F
108652024-02-03T17:14:14.777Z 11 220-239 82 82 82 141 141 141 F F F
108662024-02-03T17:14:14.777Z 12 240-259 94 94 94 164 164 164 F F F
108672024-02-03T17:14:14.777Z 13 260-279 96 96 96 166 166 166 F F F
108682024-02-03T17:14:14.777Z 14 280-299 91 91 91 159 159 159 F F F
108692024-02-03T17:14:14.778Z 15 300-319 96 96 96 166 166 166 F F F
108702024-02-03T17:14:14.778Z 16 320-339 98 98 98 170 170 170 F F F
108712024-02-03T17:14:14.778Z 17 340-359 99 99 99 172 172 172 F F F
108722024-02-03T17:14:14.778Z 18 360-379 92 92 92 160 160 160 F F F
108732024-02-03T17:14:14.778Z 19 380-399 93 93 93 162 162 162 F F F
108742024-02-03T17:14:14.778Z 20 400-419 98 98 98 170 170 170 F F F
108752024-02-03T17:14:14.778Z 21 420-439 88 88 88 154 154 154 F F F
108762024-02-03T17:14:14.778Z 22 440-459 97 97 97 168 168 168 F F F
108772024-02-03T17:14:14.779Z 23 460-479 98 98 98 171 171 171 F F F
108782024-02-03T17:14:14.779Z 24 480-499 93 93 93 162 162 162 F F F
108792024-02-03T17:14:14.779Z 25 500-519 97 97 97 168 168 168 F F F
108802024-02-03T17:14:14.779Z 26 520-539 97 97 97 168 168 168 F F F
108812024-02-03T17:14:14.779Z 27 540-559 98 98 98 171 171 171 F F F
108822024-02-03T17:14:14.779Z 28 560-579 92 92 92 161 161 161 F F F
108832024-02-03T17:14:14.779Z 29 580-599 99 99 99 172 172 172 F F F
108842024-02-03T17:14:14.780ZMax gen: 99, Max flush: 172
108852024-02-03T17:14:14.780Z
108862024-02-03T17:14:14.780ZBegin loop 50
108872024-02-03T17:14:14.780ZDownstairs to restart: 1
108882024-02-03T17:14:14.795Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.325862678Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532}
108892024-02-03T17:14:14.795Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.32633335Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532}
108902024-02-03T17:14:14.795Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.326362993Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532}
108912024-02-03T17:14:14.796Z{"msg":"Crucible 0e118d4e-f542-4ed0-ad32-e28936aedd15 has session id: 6912d24c-9ebe-433b-bb42-4c33f7c1ccee","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.326386174Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
108922024-02-03T17:14:14.796Z{"msg":"Crucible runtime is spawned
108932024-02-03T17:14:14.796ZUpstairs opts: Upstairs UUID: 0e118d4e-f542-4ed0-ad32-e28936aedd15, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.326513266Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
108942024-02-03T17:14:14.797Z{"msg":"Crucible stats registered with UUID: 0e118d4e-f542-4ed0-ad32-e28936aedd15","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.326566948Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
108952024-02-03T17:14:14.797Z{"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.326606208Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532}
108962024-02-03T17:14:14.797Z{"msg":"0e118d4e-f542-4ed0-ad32-e28936aedd15 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.326806201Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
108972024-02-03T17:14:14.797Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.326881739Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"client":"0","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
108982024-02-03T17:14:14.797Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.326909765Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"client":"0","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
108992024-02-03T17:14:14.798Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.326944717Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"client":"1","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109002024-02-03T17:14:14.798Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.326982649Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"client":"1","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109012024-02-03T17:14:14.798Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.327119704Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"client":"2","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109022024-02-03T17:14:14.798Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.327156659Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"client":"2","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109032024-02-03T17:14:14.799Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.327180079Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"":"io task","client":"0","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109042024-02-03T17:14:14.799Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.327242344Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"":"io task","client":"2","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109052024-02-03T17:14:14.799Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.327324255Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"":"io task","client":"1","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109062024-02-03T17:14:14.799Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.327426746Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"":"io task","client":"0","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109072024-02-03T17:14:14.800Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.327510334Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"":"io task","client":"2","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109082024-02-03T17:14:14.800Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.327552096Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"":"io task","client":"1","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109092024-02-03T17:14:14.800Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.329023555Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"client":"0","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109102024-02-03T17:14:14.800Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.329140949Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"client":"0","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109112024-02-03T17:14:14.801Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.32918623Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"client":"1","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109122024-02-03T17:14:14.801Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.32926066Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"client":"1","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109132024-02-03T17:14:14.801Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.329336283Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"client":"2","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109142024-02-03T17:14:14.801Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.329382311Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"client":"2","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109152024-02-03T17:14:14.802Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.33059417Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"client":"2","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109162024-02-03T17:14:14.802Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.330669946Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"client":"1","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109172024-02-03T17:14:14.802Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.33069947Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"client":"0","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109182024-02-03T17:14:14.803Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.331670225Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"client":"1","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109192024-02-03T17:14:14.803Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.331777069Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"client":"1","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109202024-02-03T17:14:14.803Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.331908771Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109212024-02-03T17:14:14.804Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.331933228Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"client":"2","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109222024-02-03T17:14:14.804Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.331957203Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"client":"2","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109232024-02-03T17:14:14.804Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.331979296Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109242024-02-03T17:14:14.804Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.331999694Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"client":"0","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109252024-02-03T17:14:14.804Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.332021733Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"client":"0","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109262024-02-03T17:14:14.805Z{"msg":"[0]R flush_numbers[0..12]: [171, 163, 168, 168, 168, 167, 171, 167, 171, 168, 168, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.332045186Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"":"downstairs","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109272024-02-03T17:14:14.805Z{"msg":"[0]R generation[0..12]: [98, 94, 97, 97, 97, 96, 98, 96, 98, 97, 97, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.332094818Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"":"downstairs","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109282024-02-03T17:14:14.805Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.332120682Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"":"downstairs","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109292024-02-03T17:14:14.805Z{"msg":"[1]R flush_numbers[0..12]: [171, 163, 168, 168, 168, 167, 171, 167, 171, 168, 168, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.332197525Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"":"downstairs","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109302024-02-03T17:14:14.806Z{"msg":"[1]R generation[0..12]: [98, 94, 97, 97, 97, 96, 98, 96, 98, 97, 97, 82]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
109312024-02-03T17:14:14.806Z,"time":"2024-02-03T17:14:14.332269718Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"":"downstairs","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109322024-02-03T17:14:14.806Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.332322892Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"":"downstairs","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109332024-02-03T17:14:14.806Z{"msg":"[2]R flush_numbers[0..12]: [171, 163, 168, 168, 168, 167, 171, 167, 171, 168, 168, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.332348164Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"":"downstairs","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109342024-02-03T17:14:14.807Z{"msg":"[2]R generation[0..12]: [98, 94, 97, 97, 97, 96, 98, 96, 98, 97, 97, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.33237073Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"":"downstairs","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109352024-02-03T17:14:14.807Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.332418882Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"":"downstairs","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109362024-02-03T17:14:14.807Z{"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.332442743Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"":"downstairs","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109372024-02-03T17:14:14.807Z{"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.332466468Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"":"downstairs","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109382024-02-03T17:14:14.808Z{"msg":"Next flush: 173","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.332512402Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"":"downstairs","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109392024-02-03T17:14:14.808Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.332535999Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"":"downstairs","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109402024-02-03T17:14:14.808Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.332558449Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109412024-02-03T17:14:14.808Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.332604063Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"client":"0","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109422024-02-03T17:14:14.808Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.332627852Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"client":"1","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109432024-02-03T17:14:14.809Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.332649542Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"client":"2","":"downstairs","session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109442024-02-03T17:14:14.809Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.332696312Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109452024-02-03T17:14:14.809Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.332718877Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109462024-02-03T17:14:14.809Z{"msg":"0e118d4e-f542-4ed0-ad32-e28936aedd15 is now active with session: 6912d24c-9ebe-433b-bb42-4c33f7c1ccee","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.332740319Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109472024-02-03T17:14:14.809Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.332795991Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532,"session_id":"6912d24c-9ebe-433b-bb42-4c33f7c1ccee"}
109482024-02-03T17:14:14.810Z{"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:14.332818253Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532}
109492024-02-03T17:14:14.810ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
109502024-02-03T17:14:14.810ZLoading write count information from file "/tmp/test_repair_verify.data"
109512024-02-03T17:14:14.810ZRun Repair workload
109522024-02-03T17:14:14.811Z01/30 Write block 208 len 12288 data: 9 8 10
109532024-02-03T17:14:14.811Z02/30 Read block 282 len 8192
109542024-02-03T17:14:14.811Z03/30 Write block 96 len 4096 data: 11
109552024-02-03T17:14:14.811Z04/30 Write block 553 len 8192 data: 10 9
109562024-02-03T17:14:14.811Z05/30 Write block 426 len 40960 data: 6 8 8 11 11 9 10 8 8 9
109572024-02-03T17:14:14.812Z06/30 Write block 186 len 20480 data: 13 16 18 19 17
109582024-02-03T17:14:14.812Z07/30 Write block 177 len 28672 data: 9 7 6 7 9 8 11
109592024-02-03T17:14:14.812Z08/30 Read block 444 len 40960
109602024-02-03T17:14:14.827Z09/30 Read block 385 len 12288
109612024-02-03T17:14:14.827Z10/30 Read block 543 len 24576
109622024-02-03T17:14:14.828Z11/30 Write block 544 len 36864 data: 7 8 9 12 9 10 11 9 9
109632024-02-03T17:14:14.828Z12/30 Read block 174 len 24576
109642024-02-03T17:14:14.828Z13/30 Write block 485 len 36864 data: 8 8 5 8 8 5 7 6 6
109652024-02-03T17:14:14.828Z14/30 Write block 7 len 24576 data: 8 10 11 10 6 5
109662024-02-03T17:14:14.829Z15/30 Read block 325 len 32768
109672024-02-03T17:14:14.829Z16/30 Flush
109682024-02-03T17:14:14.829Z17/30 Write block 553 len 16384 data: 11 10 7 6
109692024-02-03T17:14:14.829Z18/30 Write block 228 len 32768 data: 7 5 5 6 7 5 7 7
109702024-02-03T17:14:14.829Z19/30 Flush
109712024-02-03T17:14:14.829Z20/30 Write block 25 len 4096 data: 11
109722024-02-03T17:14:14.830Z21/30 Read block 206 len 8192
109732024-02-03T17:14:14.830Z22/30 Read block 114 len 40960
109742024-02-03T17:14:14.830Z23/30 Read block 171 len 36864
109752024-02-03T17:14:14.830Z24/30 Read block 340 len 4096
109762024-02-03T17:14:14.830Z25/30 Flush
109772024-02-03T17:14:14.830Z26/30 Write block 226 len 12288 data: 4 5 8
109782024-02-03T17:14:14.831Z27/30 Write block 378 len 32768 data: 8 9 7 6 6 6 5 7
109792024-02-03T17:14:14.831Z28/30 Read block 1 len 4096
109802024-02-03T17:14:14.831Z29/30 Read block 232 len 24576
109812024-02-03T17:14:14.846Z30/30 Write block 8 len 28672 data: 11 12 11 7 6 7 9
109822024-02-03T17:14:14.846Z----------------------------------------------------------------
109832024-02-03T17:14:14.846Z Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:30
109842024-02-03T17:14:14.846ZStates: Active Active Active
109852024-02-03T17:14:14.846ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
109862024-02-03T17:14:14.846Z 1 Acked 1000 Write 3 Done Done Done false
109872024-02-03T17:14:14.846Z 2 Acked 1001 Read 2 Done Done Done false
109882024-02-03T17:14:14.847Z 3 Acked 1002 Write 1 Done Done Done false
109892024-02-03T17:14:14.847Z 4 Acked 1003 Write 2 Done Done Done false
109902024-02-03T17:14:14.847Z 5 Acked 1004 Write 10 Done Done Done false
109912024-02-03T17:14:14.847Z 6 Acked 1005 Write 5 Done Sent Done false
109922024-02-03T17:14:14.847Z 7 Acked 1006 Write 7 Done Sent Done false
109932024-02-03T17:14:14.847Z 8 Acked 1007 Read 10 Done Sent Done false
109942024-02-03T17:14:14.848Z 9 Acked 1008 Read 3 Done Sent Done false
109952024-02-03T17:14:14.848Z 10 Acked 1009 Read 6 Done Sent Done false
109962024-02-03T17:14:14.848Z 11 Acked 1010 Write 9 Done Sent Done false
109972024-02-03T17:14:14.848Z 12 Acked 1011 Read 6 Done Sent Done false
109982024-02-03T17:14:14.848Z 13 Acked 1012 Write 9 Done Sent Done false
109992024-02-03T17:14:14.849Z 14 Acked 1013 Write 6 Done Sent Done false
110002024-02-03T17:14:14.849Z 15 Acked 1014 Read 8 Done Sent Done false
110012024-02-03T17:14:14.849Z 16 Acked 1015 Flush 0 Done Sent Done false
110022024-02-03T17:14:14.849Z 17 Acked 1016 Write 4 Done Sent Done false
110032024-02-03T17:14:14.849Z 18 Acked 1017 Write 8 Done Sent Done false
110042024-02-03T17:14:14.849Z 19 Acked 1018 Flush 0 Done Sent Done false
110052024-02-03T17:14:14.849Z 20 Acked 1019 Write 1 Done Sent Done false
110062024-02-03T17:14:14.850Z 21 Acked 1020 Read 2 Done Sent Done false
110072024-02-03T17:14:14.850Z 22 Acked 1021 Read 10 Done Sent Done false
110082024-02-03T17:14:14.850Z 23 Acked 1022 Read 9 Done Sent Done false
110092024-02-03T17:14:14.850Z 24 Acked 1023 Read 1 Done Sent Done false
110102024-02-03T17:14:14.850Z 25 Acked 1024 Flush 0 Done Sent Done false
110112024-02-03T17:14:14.850Z 26 Acked 1025 Write 3 Done Sent Done false
110122024-02-03T17:14:14.851Z 27 Acked 1026 Write 8 Done Sent Done false
110132024-02-03T17:14:14.851Z 28 Acked 1027 Read 1 Done Sent Done false
110142024-02-03T17:14:14.851Z 29 Acked 1028 Read 6 Done Sent Done false
110152024-02-03T17:14:14.851Z 30 Acked 1029 Write 7 Sent Sent Sent false
110162024-02-03T17:14:14.851Z STATES DS:0 DS:1 DS:2 TOTAL
110172024-02-03T17:14:14.851Z New 0 0 0 0
110182024-02-03T17:14:14.851Z Sent 1 25 1 27
110192024-02-03T17:14:14.852Z Done 29 5 29 63
110202024-02-03T17:14:14.852Z Skipped 0 0 0 0
110212024-02-03T17:14:14.852Z Error 0 0 0 0
110222024-02-03T17:14:14.852ZLast Flush: 1024 0 1024
110232024-02-03T17:14:14.852ZDownstairs last five completed:
110242024-02-03T17:14:14.852ZUpstairs last five completed: 30 29 28 27 26
110252024-02-03T17:14:14.852Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:14:14.385085141Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1532}
110262024-02-03T17:14:14.853ZWrote out file "/tmp/test_repair_verify.data"
110272024-02-03T17:14:14.853Z
110282024-02-03T17:14:14.863ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
110292024-02-03T17:14:15.273ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
110302024-02-03T17:14:15.273Z 0 000-019 100 98 100 173 171 173 T F T <---
110312024-02-03T17:14:15.273Z 1 020-039 100 94 100 175 163 175 F F F <---
110322024-02-03T17:14:15.273Z 2 040-059 97 97 97 168 168 168 F F F
110332024-02-03T17:14:15.273Z 3 060-079 97 97 97 168 168 168 F F F
110342024-02-03T17:14:15.273Z 4 080-099 100 97 100 173 168 173 F T F <---
110352024-02-03T17:14:15.273Z 5 100-119 96 96 96 167 167 167 F F F
110362024-02-03T17:14:15.273Z 6 120-139 98 98 98 171 171 171 F F F
110372024-02-03T17:14:15.273Z 7 140-159 96 96 96 167 167 167 F F F
110382024-02-03T17:14:15.273Z 8 160-179 100 98 100 173 171 173 F F F <---
110392024-02-03T17:14:15.273Z 9 180-199 100 97 100 173 168 173 F F F <---
110402024-02-03T17:14:15.273Z 10 200-219 100 97 100 173 168 173 F T F <---
110412024-02-03T17:14:15.273Z 11 220-239 100 82 100 174 141 174 T F T <---
110422024-02-03T17:14:15.273Z 12 240-259 94 94 94 164 164 164 F F F
110432024-02-03T17:14:15.273Z 13 260-279 96 96 96 166 166 166 F F F
110442024-02-03T17:14:15.273Z 14 280-299 91 91 91 159 159 159 F F F
110452024-02-03T17:14:15.274Z 15 300-319 96 96 96 166 166 166 F F F
110462024-02-03T17:14:15.274Z 16 320-339 98 98 98 170 170 170 F F F
110472024-02-03T17:14:15.274Z 17 340-359 99 99 99 172 172 172 F F F
110482024-02-03T17:14:15.274Z 18 360-379 92 92 92 160 160 160 T F T <---
110492024-02-03T17:14:15.274Z 19 380-399 93 93 93 162 162 162 T F T <---
110502024-02-03T17:14:15.274Z 20 400-419 98 98 98 170 170 170 F F F
110512024-02-03T17:14:15.274Z 21 420-439 100 88 100 173 154 173 F T F <---
110522024-02-03T17:14:15.274Z 22 440-459 97 97 97 168 168 168 F F F
110532024-02-03T17:14:15.274Z 23 460-479 98 98 98 171 171 171 F F F
110542024-02-03T17:14:15.274Z 24 480-499 100 93 100 173 162 173 F F F <---
110552024-02-03T17:14:15.274Z 25 500-519 97 97 97 168 168 168 F F F
110562024-02-03T17:14:15.274Z 26 520-539 97 97 97 168 168 168 F F F
110572024-02-03T17:14:15.274Z 27 540-559 100 98 100 174 171 174 F T F <---
110582024-02-03T17:14:15.274Z 28 560-579 92 92 92 161 161 161 F F F
110592024-02-03T17:14:15.274Z 29 580-599 99 99 99 172 172 172 F F F
110602024-02-03T17:14:15.274ZMax gen: 100, Max flush: 175
110612024-02-03T17:14:15.274ZError: Difference in extent metadata found!
110622024-02-03T17:14:15.274ZA Difference in extent metadata is expected here
110632024-02-03T17:14:15.274ZOn loop 50
110642024-02-03T17:14:15.274Z
110652024-02-03T17:14:15.278ZVerifying data now
110662024-02-03T17:14:20.932ZNo change after verify
110672024-02-03T17:14:20.933ZLoop: 50 Downstairs dump after verify (and repair):
110682024-02-03T17:14:20.960ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
110692024-02-03T17:14:20.960Z 0 000-019 101 101 101 176 176 176 F F F
110702024-02-03T17:14:20.960Z 1 020-039 100 100 100 175 175 175 F F F
110712024-02-03T17:14:20.960Z 2 040-059 97 97 97 168 168 168 F F F
110722024-02-03T17:14:20.960Z 3 060-079 97 97 97 168 168 168 F F F
110732024-02-03T17:14:21.373Z 4 080-099 100 100 100 173 173 173 F F F
110742024-02-03T17:14:21.373Z 5 100-119 96 96 96 167 167 167 F F F
110752024-02-03T17:14:21.373Z 6 120-139 98 98 98 171 171 171 F F F
110762024-02-03T17:14:21.373Z 7 140-159 96 96 96 167 167 167 F F F
110772024-02-03T17:14:21.373Z 8 160-179 100 100 100 173 173 173 F F F
110782024-02-03T17:14:21.373Z 9 180-199 100 100 100 173 173 173 F F F
110792024-02-03T17:14:21.373Z 10 200-219 100 100 100 173 173 173 F F F
110802024-02-03T17:14:21.373Z 11 220-239 101 101 101 176 176 176 F F F
110812024-02-03T17:14:21.373Z 12 240-259 94 94 94 164 164 164 F F F
110822024-02-03T17:14:21.373Z 13 260-279 96 96 96 166 166 166 F F F
110832024-02-03T17:14:21.373Z 14 280-299 91 91 91 159 159 159 F F F
110842024-02-03T17:14:21.373Z 15 300-319 96 96 96 166 166 166 F F F
110852024-02-03T17:14:21.373Z 16 320-339 98 98 98 170 170 170 F F F
110862024-02-03T17:14:21.373Z 17 340-359 99 99 99 172 172 172 F F F
110872024-02-03T17:14:21.373Z 18 360-379 101 101 101 176 176 176 F F F
110882024-02-03T17:14:21.373Z 19 380-399 101 101 101 176 176 176 F F F
110892024-02-03T17:14:21.373Z 20 400-419 98 98 98 170 170 170 F F F
110902024-02-03T17:14:21.373Z 21 420-439 100 100 100 173 173 173 F F F
110912024-02-03T17:14:21.373Z 22 440-459 97 97 97 168 168 168 F F F
110922024-02-03T17:14:21.373Z 23 460-479 98 98 98 171 171 171 F F F
110932024-02-03T17:14:21.373Z 24 480-499 100 100 100 173 173 173 F F F
110942024-02-03T17:14:21.373Z 25 500-519 97 97 97 168 168 168 F F F
110952024-02-03T17:14:21.374Z 26 520-539 97 97 97 168 168 168 F F F
110962024-02-03T17:14:21.374Z 27 540-559 100 100 100 174 174 174 F F F
110972024-02-03T17:14:21.374Z 28 560-579 92 92 92 161 161 161 F F F
110982024-02-03T17:14:21.374Z 29 580-599 99 99 99 172 172 172 F F F
110992024-02-03T17:14:21.374ZMax gen: 101, Max flush: 176
111002024-02-03T17:14:21.374Z
111012024-02-03T17:14:21.374ZBegin loop 51
111022024-02-03T17:14:21.374ZDownstairs to restart: 1
111032024-02-03T17:14:21.383Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.551945642Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540}
111042024-02-03T17:14:21.384Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.552535356Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540}
111052024-02-03T17:14:21.384ZCrucible runtime is spawned
111062024-02-03T17:14:21.384Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.552694193Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540}
111072024-02-03T17:14:21.385Z{"msg":"Crucible 40b8e73a-4941-4dbe-a41f-666d8af6f1bb has session id: 3b1485bc-6148-4349-bc12-a3d33db0aa5d","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.552800285Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111082024-02-03T17:14:21.385Z{"msg":"Upstairs opts: Upstairs UUID: 40b8e73a-4941-4dbe-a41f-666d8af6f1bb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.552842559Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111092024-02-03T17:14:21.385Z{"msg":"Crucible stats registered with UUID: 40b8e73a-4941-4dbe-a41f-666d8af6f1bb","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.552866226Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111102024-02-03T17:14:21.385Z{"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.5529174Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540}
111112024-02-03T17:14:21.385Z{"msg":"40b8e73a-4941-4dbe-a41f-666d8af6f1bb active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.552938914Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111122024-02-03T17:14:21.385Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.552959254Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"client":"0","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111132024-02-03T17:14:21.385Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.552982223Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"client":"0","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111142024-02-03T17:14:21.385Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.553004102Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"client":"1","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111152024-02-03T17:14:21.385Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.553027192Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"client":"1","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111162024-02-03T17:14:21.385Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.553048836Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"client":"2","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111172024-02-03T17:14:21.385Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.553074477Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"client":"2","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111182024-02-03T17:14:21.386Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.553238203Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"":"io task","client":"1","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111192024-02-03T17:14:21.386Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.553369559Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"":"io task","client":"0","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111202024-02-03T17:14:21.386Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.55347723Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"":"io task","client":"2","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111212024-02-03T17:14:21.386Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.553569252Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"":"io task","client":"0","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111222024-02-03T17:14:21.386Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.553706223Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"":"io task","client":"2","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111232024-02-03T17:14:21.386Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.553801421Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"":"io task","client":"1","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111242024-02-03T17:14:21.386Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.55500991Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"client":"2","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111252024-02-03T17:14:21.386Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.55514254Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"client":"2","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111262024-02-03T17:14:21.386Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.555261753Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"client":"0","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111272024-02-03T17:14:21.386Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.555308702Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"client":"0","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111282024-02-03T17:14:21.386Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.555929015Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"client":"1","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111292024-02-03T17:14:21.386Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.556090795Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"client":"1","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111302024-02-03T17:14:21.386Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.556145589Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"client":"2","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111312024-02-03T17:14:21.387Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.556237431Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"client":"0","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111322024-02-03T17:14:21.387Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.557077433Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"client":"2","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111332024-02-03T17:14:21.387Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.557161939Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"client":"2","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111342024-02-03T17:14:21.387Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.557193807Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111352024-02-03T17:14:21.387Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.557265883Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"client":"1","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111362024-02-03T17:14:21.387Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.557294476Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"client":"0","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111372024-02-03T17:14:21.387Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.55734684Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"client":"0","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111382024-02-03T17:14:21.387Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.557439265Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111392024-02-03T17:14:21.387Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.558199804Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"client":"1","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111402024-02-03T17:14:21.387Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.558284079Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"client":"1","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111412024-02-03T17:14:21.387Z{"msg":"[0]R flush_numbers[0..12]: [176, 175, 168, 168, 173, 167, 171, 167, 173, 173, 173, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.558310543Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"":"downstairs","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111422024-02-03T17:14:21.387Z{"msg":"[0]R generation[0..12]: [101, 100, 97, 97, 100, 96, 98, 96, 100, 100, 100, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.558333697Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"":"downstairs","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111432024-02-03T17:14:21.388Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.558357422Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"":"downstairs","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111442024-02-03T17:14:21.388Z{"msg":"[1]R flush_numbers[0..12]: [176, 175, 168, 168, 173, 167, 171, 167, 173, 173, 173, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.55838183Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"":"downstairs","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111452024-02-03T17:14:21.388Z{"msg":"[1]R generation[0..12]: [101, 100, 97, 97, 100, 96, 98, 96, 100, 100, 100, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.558432908Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"":"downstairs","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111462024-02-03T17:14:21.388Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.558458732Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"":"downstairs","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111472024-02-03T17:14:21.388Z{"msg":"[2]R flush_numbers[0..12]: [176, 175, 168, 168, 173, 167, 171, 167, 173, 173, 173, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.558509239Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"":"downstairs","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111482024-02-03T17:14:21.388Z{"msg":"[2]R generation[0..12]: [101, 100, 97, 97, 100, 96, 98, 96, 100, 100, 100, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.558539807Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"":"downstairs","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111492024-02-03T17:14:21.388Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","vWait for a query_work_queue command to finish before sending IO
111502024-02-03T17:14:21.388Z":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.558727191Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"":"downstairs","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111512024-02-03T17:14:21.388Z{"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.558808905Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"":"downstairs","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111522024-02-03T17:14:21.388Z{"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.558836774Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"":"downstairs","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111532024-02-03T17:14:21.388Z{"msg":"Next flush: 177","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.558887006Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"":"downstairs","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111542024-02-03T17:14:21.388Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.558910839Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"":"downstairs","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111552024-02-03T17:14:21.388Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.558990101Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111562024-02-03T17:14:21.389Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.559018498Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"client":"0","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111572024-02-03T17:14:21.389Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.559085565Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"client":"1","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111582024-02-03T17:14:21.389Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.559166563Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"client":"2","":"downstairs","session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111592024-02-03T17:14:21.389Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.559196336Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111602024-02-03T17:14:21.389Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.559246181Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111612024-02-03T17:14:21.389Z{"msg":"40b8e73a-4941-4dbe-a41f-666d8af6f1bb is now active with session: 3b1485bc-6148-4349-bc12-a3d33db0aa5d","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.559276991Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111622024-02-03T17:14:21.389Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.5593494Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540,"session_id":"3b1485bc-6148-4349-bc12-a3d33db0aa5d"}
111632024-02-03T17:14:21.389Z{"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:20.559425629Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540}
111642024-02-03T17:14:21.389ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
111652024-02-03T17:14:21.389ZLoading write count information from file "/tmp/test_repair_verify.data"
111662024-02-03T17:14:21.389ZRun Repair workload
111672024-02-03T17:14:21.389Z01/30 Flush
111682024-02-03T17:14:21.390Z02/30 Flush
111692024-02-03T17:14:21.390Z03/30 Write block 564 len 28672 data: 9 8 7 6 9 8 8
111702024-02-03T17:14:21.390Z04/30 Read block 289 len 12288
111712024-02-03T17:14:21.390Z05/30 Read block 491 len 32768
111722024-02-03T17:14:21.390Z06/30 Write block 570 len 20480 data: 9 8 6 8 8
111732024-02-03T17:14:21.390Z07/30 Read block 504 len 8192
111742024-02-03T17:14:21.390Z08/30 Read block 453 len 36864
111752024-02-03T17:14:21.390Z09/30 Read block 47 len 32768
111762024-02-03T17:14:21.390Z10/30 Read block 202 len 24576
111772024-02-03T17:14:21.390Z11/30 Read block 592 len 4096
111782024-02-03T17:14:21.390Z12/30 Write block 47 len 24576 data: 6 5 5 6 7 7
111792024-02-03T17:14:21.390Z13/30 Read block 130 len 28672
111802024-02-03T17:14:21.390Z14/30 Write block 13 len 4096 data: 8
111812024-02-03T17:14:21.391Z15/30 Write block 148 len 28672 data: 6 6 5 9 8 8 7
111822024-02-03T17:14:21.391Z16/30 Write block 336 len 8192 data: 6 7
111832024-02-03T17:14:21.391Z17/30 Read block 83 len 16384
111842024-02-03T17:14:21.391Z18/30 Read block 327 len 20480
111852024-02-03T17:14:21.391Z19/30 Read block 234 len 4096
111862024-02-03T17:14:21.391Z20/30 Read block 369 len 40960
111872024-02-03T17:14:21.391Z21/30 Write block 338 len 36864 data: 9 8 10 8 8 6 7 8 7
111882024-02-03T17:14:21.391Z22/30 Write block 382 len 12288 data: 7 7 6
111892024-02-03T17:14:21.391Z23/30 Read block 491 len 24576
111902024-02-03T17:14:21.391Z24/30 Read block 169 len 16384
111912024-02-03T17:14:21.391Z25/30 Read block 130 len 40960
111922024-02-03T17:14:21.391Z26/30 Read block 443 len 20480
111932024-02-03T17:14:21.391Z27/30 Write block 321 len 4096 data: 11
111942024-02-03T17:14:21.392Z28/30 Write block 87 len 36864 data: 8 7 9 10 9 8 11 11 9
111952024-02-03T17:14:21.392Z29/30 Write block 422 len 4096 data: 7
111962024-02-03T17:14:21.392Z30/30 Read block 553 len 40960
111972024-02-03T17:14:21.392Z----------------------------------------------------------------
111982024-02-03T17:14:21.392Z Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:28
111992024-02-03T17:14:21.392ZStates: Active Active Active
112002024-02-03T17:14:21.392ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
112012024-02-03T17:14:21.392Z 3 Acked 1002 Write 7 Done Sent Done false
112022024-02-03T17:14:21.392Z 4 Acked 1003 Read 3 Done Sent Done false
112032024-02-03T17:14:21.404Z 5 Acked 1004 Read 8 Done Sent Done false
112042024-02-03T17:14:21.404Z 6 Acked 1005 Write 5 Done Sent Done false
112052024-02-03T17:14:21.404Z 7 Acked 1006 Read 2 Done Sent Done false
112062024-02-03T17:14:21.404Z 8 Acked 1007 Read 9 Done Sent Done false
112072024-02-03T17:14:21.404Z 9 Acked 1008 Read 8 Done Sent Done false
112082024-02-03T17:14:21.404Z 10 Acked 1009 Read 6 Done Sent Done false
112092024-02-03T17:14:21.404Z 11 Acked 1010 Read 1 Done Sent Done false
112102024-02-03T17:14:21.404Z 12 Acked 1011 Write 6 Done Sent Done false
112112024-02-03T17:14:21.404Z 13 Acked 1012 Read 7 Done Sent Done false
112122024-02-03T17:14:21.405Z 14 Acked 1013 Write 1 Done Sent Done false
112132024-02-03T17:14:21.405Z 15 Acked 1014 Write 7 Done Sent Done false
112142024-02-03T17:14:21.405Z 16 Acked 1015 Write 2 Done Sent Done false
112152024-02-03T17:14:21.405Z 17 Acked 1016 Read 4 Done Sent Done false
112162024-02-03T17:14:21.405Z 18 Acked 1017 Read 5 Done Sent Done false
112172024-02-03T17:14:21.405Z 19 Acked 1018 Read 1 Done Sent Done false
112182024-02-03T17:14:21.405Z 20 Acked 1019 Read 10 Done Sent Done false
112192024-02-03T17:14:21.405Z 21 Acked 1020 Write 9 Done Sent Done false
112202024-02-03T17:14:21.405Z 22 Acked 1021 Write 3 Done Sent Done false
112212024-02-03T17:14:21.405Z 23 Acked 1022 Read 6 Done Sent Done false
112222024-02-03T17:14:21.405Z 24 Acked 1023 Read 4 Done Sent Done false
112232024-02-03T17:14:21.405Z 25 Acked 1024 Read 10 Done Sent Done false
112242024-02-03T17:14:21.406Z 26 Acked 1025 Read 5 Done Sent Done false
112252024-02-03T17:14:21.406Z 27 Acked 1026 Write 1 Done Sent Done false
112262024-02-03T17:14:21.406Z 28 Acked 1027 Write 9 Done Sent Done false
112272024-02-03T17:14:21.406Z 29 Acked 1028 Write 1 Done Sent Done false
112282024-02-03T17:14:21.406Z 30 Acked 1029 Read 10 Done Sent Done false
112292024-02-03T17:14:21.406Z STATES DS:0 DS:1 DS:2 TOTAL
112302024-02-03T17:14:21.406Z New 0 0 0 0
112312024-02-03T17:14:21.406Z Sent 0 28 0 28
112322024-02-03T17:14:21.406Z Done 28 0 28 56
112332024-02-03T17:14:21.406Z Skipped 0 0 0 0
112342024-02-03T17:14:21.406Z Error 0 0 0 0
112352024-02-03T17:14:21.406ZLast Flush: 1001 1001 1001
112362024-02-03T17:14:21.406ZDownstairs last five completed: 1001 1000
112372024-02-03T17:14:21.406ZUpstairs last five completed: 30 29 28 27 26
112382024-02-03T17:14:21.407Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:14:20.604483833Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1540}
112392024-02-03T17:14:21.407ZWrote out file "/tmp/test_repair_verify.data"
112402024-02-03T17:14:21.407Z
112412024-02-03T17:14:21.407ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
112422024-02-03T17:14:21.407ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
112432024-02-03T17:14:21.407Z 0 000-019 101 101 101 176 176 176 T F T <---
112442024-02-03T17:14:21.407Z 1 020-039 100 100 100 175 175 175 F F F
112452024-02-03T17:14:21.407Z 2 040-059 97 97 97 168 168 168 T F T <---
112462024-02-03T17:14:21.407Z 3 060-079 97 97 97 168 168 168 F F F
112472024-02-03T17:14:21.407Z 4 080-099 100 100 100 173 173 173 T F T <---
112482024-02-03T17:14:21.407Z 5 100-119 96 96 96 167 167 167 F F F
112492024-02-03T17:14:21.407Z 6 120-139 98 98 98 171 171 171 F F F
112502024-02-03T17:14:21.407Z 7 140-159 96 96 96 167 167 167 T F T <---
112512024-02-03T17:14:21.407Z 8 160-179 100 100 100 173 173 173 F F F
112522024-02-03T17:14:21.407Z 9 180-199 100 100 100 173 173 173 F F F
112532024-02-03T17:14:21.408Z 10 200-219 100 100 100 173 173 173 F F F
112542024-02-03T17:14:21.408Z 11 220-239 101 101 101 176 176 176 F F F
112552024-02-03T17:14:21.408Z 12 240-259 94 94 94 164 164 164 F F F
112562024-02-03T17:14:21.408Z 13 260-279 96 96 96 166 166 166 F F F
112572024-02-03T17:14:21.408Z 14 280-299 91 91 91 159 159 159 F F F
112582024-02-03T17:14:21.408Z 15 300-319 96 96 96 166 166 166 F F F
112592024-02-03T17:14:21.408Z 16 320-339 98 98 98 170 170 170 T F T <---
112602024-02-03T17:14:21.408Z 17 340-359 99 99 99 172 172 172 T F T <---
112612024-02-03T17:14:21.408Z 18 360-379 101 101 101 176 176 176 F F F
112622024-02-03T17:14:21.408Z 19 380-399 101 101 101 176 176 176 T F T <---
112632024-02-03T17:14:21.408Z 20 400-419 98 98 98 170 170 170 F F F
112642024-02-03T17:14:21.408Z 21 420-439 100 100 100 173 173 173 T F T <---
112652024-02-03T17:14:21.408Z 22 440-459 97 97 97 168 168 168 F F F
112662024-02-03T17:14:21.408Z 23 460-479 98 98 98 171 171 171 F F F
112672024-02-03T17:14:21.408Z 24 480-499 100 100 100 173 173 173 F F F
112682024-02-03T17:14:21.408Z 25 500-519 97 97 97 168 168 168 F F F
112692024-02-03T17:14:21.408Z 26 520-539 97 97 97 168 168 168 F F F
112702024-02-03T17:14:21.408Z 27 540-559 100 100 100 174 174 174 F F F
112712024-02-03T17:14:21.408Z 28 560-579 92 92 92 161 161 161 T F T <---
112722024-02-03T17:14:21.408Z 29 580-599 99 99 99 172 172 172 F F F
112732024-02-03T17:14:21.408ZMax gen: 101, Max flush: 176
112742024-02-03T17:14:21.408ZError: Difference in extent metadata found!
112752024-02-03T17:14:21.408ZA Difference in extent metadata is expected here
112762024-02-03T17:14:21.409ZOn loop 51
112772024-02-03T17:14:21.409Z
112782024-02-03T17:14:21.409ZVerifying data now
112792024-02-03T17:14:25.947ZNo change after verify
112802024-02-03T17:14:25.947ZLoop: 51 Downstairs dump after verify (and repair):
112812024-02-03T17:14:25.970ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
112822024-02-03T17:14:25.970Z 0 000-019 102 102 102 177 177 177 F F F
112832024-02-03T17:14:25.970Z 1 020-039 100 100 100 175 175 175 F F F
112842024-02-03T17:14:25.970Z 2 040-059 102 102 102 177 177 177 F F F
112852024-02-03T17:14:25.970Z 3 060-079 97 97 97 168 168 168 F F F
112862024-02-03T17:14:25.974Z 4 080-099 102 102 102 177 177 177 F F F
112872024-02-03T17:14:25.974Z 5 100-119 96 96 96 167 167 167 F F F
112882024-02-03T17:14:25.974Z 6 120-139 98 98 98 171 171 171 F F F
112892024-02-03T17:14:25.974Z 7 140-159 102 102 102 177 177 177 F F F
112902024-02-03T17:14:25.974Z 8 160-179 100 100 100 173 173 173 F F F
112912024-02-03T17:14:25.974Z 9 180-199 100 100 100 173 173 173 F F F
112922024-02-03T17:14:25.974Z 10 200-219 100 100 100 173 173 173 F F F
112932024-02-03T17:14:25.974Z 11 220-239 101 101 101 176 176 176 F F F
112942024-02-03T17:14:25.974Z 12 240-259 94 94 94 164 164 164 F F F
112952024-02-03T17:14:25.974Z 13 260-279 96 96 96 166 166 166 F F F
112962024-02-03T17:14:25.974Z 14 280-299 91 91 91 159 159 159 F F F
112972024-02-03T17:14:25.974Z 15 300-319 96 96 96 166 166 166 F F F
112982024-02-03T17:14:25.974Z 16 320-339 102 102 102 177 177 177 F F F
112992024-02-03T17:14:25.974Z 17 340-359 102 102 102 177 177 177 F F F
113002024-02-03T17:14:25.974Z 18 360-379 101 101 101 176 176 176 F F F
113012024-02-03T17:14:25.974Z 19 380-399 102 102 102 177 177 177 F F F
113022024-02-03T17:14:25.974Z 20 400-419 98 98 98 170 170 170 F F F
113032024-02-03T17:14:25.974Z 21 420-439 102 102 102 177 177 177 F F F
113042024-02-03T17:14:25.974Z 22 440-459 97 97 97 168 168 168 F F F
113052024-02-03T17:14:25.974Z 23 460-479 98 98 98 171 171 171 F F F
113062024-02-03T17:14:25.974Z 24 480-499 100 100 100 173 173 173 F F F
113072024-02-03T17:14:25.974Z 25 500-519 97 97 97 168 168 168 F F F
113082024-02-03T17:14:25.975Z 26 520-539 97 97 97 168 168 168 F F F
113092024-02-03T17:14:25.975Z 27 540-559 100 100 100 174 174 174 F F F
113102024-02-03T17:14:25.975Z 28 560-579 102 102 102 177 177 177 F F F
113112024-02-03T17:14:25.975Z 29 580-599 99 99 99 172 172 172 F F F
113122024-02-03T17:14:25.975ZMax gen: 102, Max flush: 177
113132024-02-03T17:14:25.978Z
113142024-02-03T17:14:25.978ZBegin loop 52
113152024-02-03T17:14:25.978ZDownstairs to restart: 1
113162024-02-03T17:14:26.006Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.566965165Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548}
113172024-02-03T17:14:26.006Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.567459259Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548}
113182024-02-03T17:14:26.006Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.56749333Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548}
113192024-02-03T17:14:26.010Z{"msg":"Crucible b5f294dc-dcd6-40a2-b8bf-98a09d95b0f3 has session id: 022759b5-2316-4e4c-93d4-93623f5a6a97","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.567570287Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"session_id"Crucible runtime is spawned
113202024-02-03T17:14:26.010Z:"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113212024-02-03T17:14:26.010Z{"msg":"Upstairs opts: Upstairs UUID: b5f294dc-dcd6-40a2-b8bf-98a09d95b0f3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.567893091Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113222024-02-03T17:14:26.010Z{"msg":"Crucible stats registered with UUID: b5f294dc-dcd6-40a2-b8bf-98a09d95b0f3","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.567984764Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113232024-02-03T17:14:26.011Z{"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.568027892Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548}
113242024-02-03T17:14:26.011Z{"msg":"b5f294dc-dcd6-40a2-b8bf-98a09d95b0f3 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.568059714Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113252024-02-03T17:14:26.011Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.568111124Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"client":"0","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113262024-02-03T17:14:26.011Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.568138176Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"client":"0","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113272024-02-03T17:14:26.011Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.568160682Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"client":"1","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113282024-02-03T17:14:26.011Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.568183144Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"client":"1","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113292024-02-03T17:14:26.011Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.56820723Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"client":"2","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113302024-02-03T17:14:26.011Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.56825615Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"client":"2","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113312024-02-03T17:14:26.011Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.568280747Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"":"io task","client":"2","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113322024-02-03T17:14:26.011Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.568431983Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"":"io task","client":"0","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113332024-02-03T17:14:26.011Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.568675211Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"":"io task","client":"1","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113342024-02-03T17:14:26.011Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.568757553Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"":"io task","client":"0","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113352024-02-03T17:14:26.011Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.568847211Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"":"io task","client":"2","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113362024-02-03T17:14:26.011Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.569090724Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"":"io task","client":"1","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113372024-02-03T17:14:26.011Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.570271531Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"client":"2","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113382024-02-03T17:14:26.011Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.570345132Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"client":"2","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113392024-02-03T17:14:26.011Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.570372834Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"client":"0","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113402024-02-03T17:14:26.011Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.570395124Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"client":"0","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113412024-02-03T17:14:26.011Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.570542202Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"client":"1","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113422024-02-03T17:14:26.011Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.570601021Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"client":"1","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113432024-02-03T17:14:26.016Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.571453586Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"client":"0","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113442024-02-03T17:14:26.016Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.571555892Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"client":"2","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113452024-02-03T17:14:26.016Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.571714246Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"client":"1","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113462024-02-03T17:14:26.016Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.572419246Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"client":"0","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113472024-02-03T17:14:26.016Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.57258597Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"client":"0","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113482024-02-03T17:14:26.017Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.572631978Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113492024-02-03T17:14:26.017Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.572700473Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"client":"1","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113502024-02-03T17:14:26.017Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.572729482Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"client":"1","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113512024-02-03T17:14:26.017Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.572752278Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113522024-02-03T17:14:26.017Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.572772736Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"client":"2","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113532024-02-03T17:14:26.017Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.572795352Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"client":"2","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113542024-02-03T17:14:26.017Z{"msg":"[0]R flush_numbers[0..12]: [177, 175, 177, 168, 177, 167, 171, 177, 173, 173, 173, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.572855534Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"":"downstairs","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113552024-02-03T17:14:26.017Z{"msg":"[0]R generation[0..12]: [102, 100, 102, 97, 102, 96, 98, 102, 100, 100, 100, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.572884038Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"":"downstairs","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113562024-02-03T17:14:26.017Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.57293428Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"":"downstairs","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113572024-02-03T17:14:26.017Z{"msg":"[1]R flush_numbers[0..12]: [177, 175, 177, 168, 177, 167, 171, 177, 173, 173, 173, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.572960215Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"":"downstairs","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113582024-02-03T17:14:26.017Z{"msg":"[1]R generation[0..12]: [102, 100, 102, 97, 102, 96, 98, 102, 100, 100, 100, 101]","v":0,"name":"Wait for a query_work_queue command to finish before sending IO
113592024-02-03T17:14:26.017Zcrucible","level":30,"time":"2024-02-03T17:14:25.573020964Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"":"downstairs","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113602024-02-03T17:14:26.017Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.57305277Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"":"downstairs","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113612024-02-03T17:14:26.017Z{"msg":"[2]R flush_numbers[0..12]: [177, 175, 177, 168, 177, 167, 171, 177, 173, 173, 173, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.573104076Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"":"downstairs","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113622024-02-03T17:14:26.017Z{"msg":"[2]R generation[0..12]: [102, 100, 102, 97, 102, 96, 98, 102, 100, 100, 100, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.57312954Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"":"downstairs","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113632024-02-03T17:14:26.017Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.573178815Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"":"downstairs","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113642024-02-03T17:14:26.017Z{"msg":"Max found gen is 103","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.573202679Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"":"downstairs","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113652024-02-03T17:14:26.017Z{"msg":"Generation requested: 104 >= found:103","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.573225071Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"":"downstairs","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113662024-02-03T17:14:26.017Z{"msg":"Next flush: 178","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.573270421Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"":"downstairs","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113672024-02-03T17:14:26.017Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.573294052Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"":"downstairs","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113682024-02-03T17:14:26.017Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.573347701Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113692024-02-03T17:14:26.017Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.573372443Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"client":"0","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113702024-02-03T17:14:26.017Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.573421415Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"client":"1","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113712024-02-03T17:14:26.018Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.573445648Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"client":"2","":"downstairs","session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113722024-02-03T17:14:26.018Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.573469369Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113732024-02-03T17:14:26.018Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.573490076Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113742024-02-03T17:14:26.018Z{"msg":"b5f294dc-dcd6-40a2-b8bf-98a09d95b0f3 is now active with session: 022759b5-2316-4e4c-93d4-93623f5a6a97","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.573511406Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113752024-02-03T17:14:26.018Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.573561398Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548,"session_id":"022759b5-2316-4e4c-93d4-93623f5a6a97"}
113762024-02-03T17:14:26.018Z{"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:25.573585342Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548}
113772024-02-03T17:14:26.018ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
113782024-02-03T17:14:26.018ZLoading write count information from file "/tmp/test_repair_verify.data"
113792024-02-03T17:14:26.018ZRun Repair workload
113802024-02-03T17:14:26.018Z01/30 Write block 303 len 24576 data: 8 9 8 9 7 7
113812024-02-03T17:14:26.043Z02/30 Read block 486 len 24576
113822024-02-03T17:14:26.044Z03/30 Read block 36 len 28672
113832024-02-03T17:14:26.044Z04/30 Write block 103 len 20480 data: 6 6 6 8 9
113842024-02-03T17:14:26.044Z05/30 Write block 450 len 8192 data: 6 5
113852024-02-03T17:14:26.044Z06/30 Write block 452 len 16384 data: 5 6 6 5
113862024-02-03T17:14:26.044Z07/30 Write block 562 len 32768 data: 9 9 10 9 8 7 10 9
113872024-02-03T17:14:26.044Z08/30 Write block 576 len 16384 data: 6 5 5 6
113882024-02-03T17:14:26.044Z09/30 Read block 589 len 12288
113892024-02-03T17:14:26.044Z10/30 Flush
113902024-02-03T17:14:26.044Z11/30 Write block 179 len 28672 data: 7 8 10 9 12 12 16
113912024-02-03T17:14:26.044Z12/30 Read block 118 len 12288
113922024-02-03T17:14:26.044Z13/30 Read block 581 len 24576
113932024-02-03T17:14:26.044Z14/30 Write block 8 len 12288 data: 12 13 12
113942024-02-03T17:14:26.044Z15/30 Write block 527 len 16384 data: 9 8 8 10
113952024-02-03T17:14:26.044Z16/30 Write block 12 len 32768 data: 7 9 10 7 7 9 12 14
113962024-02-03T17:14:26.044Z17/30 Write block 478 len 4096 data: 8
113972024-02-03T17:14:26.044Z18/30 Read block 293 len 40960
113982024-02-03T17:14:26.044Z19/30 Read block 307 len 32768
113992024-02-03T17:14:26.048Z20/30 Read block 88 len 8192
114002024-02-03T17:14:26.048Z21/30 Write block 425 len 16384 data: 6 7 9 9
114012024-02-03T17:14:26.048Z22/30 Write block 551 len 40960 data: 10 10 12 11 8 7 6 5 6 7
114022024-02-03T17:14:26.049Z23/30 Read block 264 len 20480
114032024-02-03T17:14:26.051Z24/30 Write block 157 len 12288 data: 9 9 8
114042024-02-03T17:14:26.051Z25/30 Flush
114052024-02-03T17:14:26.055Z26/30 Write block 168 len 16384 data: 13 13 12 11
114062024-02-03T17:14:26.058Z27/30 Write block 298 len 12288 data: 9 9 7
114072024-02-03T17:14:26.058Z28/30 Write block 46 len 16384 data: 6 7 6 6
114082024-02-03T17:14:26.058Z29/30 Read block 490 len 24576
114092024-02-03T17:14:26.061Z30/30 Read block 115 len 24576
114102024-02-03T17:14:26.065Z----------------------------------------------------------------
114112024-02-03T17:14:26.065Z Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:30
114122024-02-03T17:14:26.065ZStates: Active Active Active
114132024-02-03T17:14:26.065ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
114142024-02-03T17:14:26.065Z 1 Acked 1000 Write 6 Done Done Done false
114152024-02-03T17:14:26.065Z 2 Acked 1001 Read 6 Done Done Done false
114162024-02-03T17:14:26.065Z 3 Acked 1002 Read 7 Done Sent Done false
114172024-02-03T17:14:26.065Z 4 Acked 1003 Write 5 Done Sent Done false
114182024-02-03T17:14:26.065Z 5 Acked 1004 Write 2 Done Sent Done false
114192024-02-03T17:14:26.065Z 6 Acked 1005 Write 4 Done Sent Done false
114202024-02-03T17:14:26.065Z 7 Acked 1006 Write 8 Done Sent Done false
114212024-02-03T17:14:26.065Z 8 Acked 1007 Write 4 Done Sent Done false
114222024-02-03T17:14:26.065Z 9 Acked 1008 Read 3 Done Sent Done false
114232024-02-03T17:14:26.065Z 10 Acked 1009 Flush 0 Done Sent Done false
114242024-02-03T17:14:26.065Z 11 Acked 1010 Write 7 Done Sent Done false
114252024-02-03T17:14:26.065Z 12 Acked 1011 Read 3 Done Sent Done false
114262024-02-03T17:14:26.065Z 13 Acked 1012 Read 6 Done Sent Done false
114272024-02-03T17:14:26.065Z 14 Acked 1013 Write 3 Done Sent Done false
114282024-02-03T17:14:26.065Z 15 Acked 1014 Write 4 Done Sent Done false
114292024-02-03T17:14:26.065Z 16 Acked 1015 Write 8 Done Sent Done false
114302024-02-03T17:14:26.065Z 17 Acked 1016 Write 1 Done Sent Done false
114312024-02-03T17:14:26.066Z 18 Acked 1017 Read 10 Done Sent Done false
114322024-02-03T17:14:26.066Z 19 Acked 1018 Read 8 Done Sent Done false
114332024-02-03T17:14:26.066Z 20 Acked 1019 Read 2 Done Sent Done false
114342024-02-03T17:14:26.066Z 21 Acked 1020 Write 4 Done Sent Done false
114352024-02-03T17:14:26.066Z 22 Acked 1021 Write 10 Done Sent Done false
114362024-02-03T17:14:26.066Z 23 Acked 1022 Read 5 Done Sent Done false
114372024-02-03T17:14:26.066Z 24 Acked 1023 Write 3 Done Sent Done false
114382024-02-03T17:14:26.066Z 25 Acked 1024 Flush 0 Done Sent Done false
114392024-02-03T17:14:26.066Z 26 Acked 1025 Write 4 Done Sent Done false
114402024-02-03T17:14:26.066Z 27 Acked 1026 Write 3 Done Sent Done false
114412024-02-03T17:14:26.066Z 28 Acked 1027 Write 4 Done Sent Done false
114422024-02-03T17:14:26.066Z 29 Acked 1028 Read 6 Done Sent Done false
114432024-02-03T17:14:26.066Z 30 Acked 1029 Read 6 Done Sent Sent false
114442024-02-03T17:14:26.066Z STATES DS:0 DS:1 DS:2 TOTAL
114452024-02-03T17:14:26.066Z New 0 0 0 0
114462024-02-03T17:14:26.066Z Sent 0 28 1 29
114472024-02-03T17:14:26.066Z Done 30 2 29 61
114482024-02-03T17:14:26.066Z Skipped 0 0 0 0
114492024-02-03T17:14:26.066Z Error 0 0 0 0
114502024-02-03T17:14:26.066ZLast Flush: 1024 0 1024
114512024-02-03T17:14:26.066ZDownstairs last five completed:
114522024-02-03T17:14:26.066ZUpstairs last five completed: 30 29 28 27 26
114532024-02-03T17:14:26.066Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:14:25.624343997Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1548}
114542024-02-03T17:14:26.066ZWrote out file "/tmp/test_repair_verify.data"
114552024-02-03T17:14:26.070Z
114562024-02-03T17:14:26.075ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
114572024-02-03T17:14:26.100ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
114582024-02-03T17:14:26.100Z 0 000-019 104 102 104 179 177 179 F F F <---
114592024-02-03T17:14:26.100Z 1 020-039 100 100 100 175 175 175 F F F
114602024-02-03T17:14:26.100Z 2 040-059 102 102 102 177 177 177 T F T <---
114612024-02-03T17:14:26.100Z 3 060-079 97 97 97 168 168 168 F F F
114622024-02-03T17:14:26.104Z 4 080-099 102 102 102 177 177 177 F F F
114632024-02-03T17:14:26.104Z 5 100-119 104 96 104 178 167 178 F F F <---
114642024-02-03T17:14:26.104Z 6 120-139 98 98 98 171 171 171 F F F
114652024-02-03T17:14:26.104Z 7 140-159 104 102 104 179 177 179 F F F <---
114662024-02-03T17:14:26.104Z 8 160-179 104 100 104 179 173 179 T F T <---
114672024-02-03T17:14:26.104Z 9 180-199 104 100 104 179 173 179 F F F <---
114682024-02-03T17:14:26.104Z 10 200-219 100 100 100 173 173 173 F F F
114692024-02-03T17:14:26.104Z 11 220-239 101 101 101 176 176 176 F F F
114702024-02-03T17:14:26.104Z 12 240-259 94 94 94 164 164 164 F F F
114712024-02-03T17:14:26.104Z 13 260-279 96 96 96 166 166 166 F F F
114722024-02-03T17:14:26.104Z 14 280-299 91 91 91 159 159 159 T F T <---
114732024-02-03T17:14:26.104Z 15 300-319 104 96 104 178 166 178 T T T <---
114742024-02-03T17:14:26.104Z 16 320-339 102 102 102 177 177 177 F F F
114752024-02-03T17:14:26.104Z 17 340-359 102 102 102 177 177 177 F F F
114762024-02-03T17:14:26.104Z 18 360-379 101 101 101 176 176 176 F F F
114772024-02-03T17:14:26.104Z 19 380-399 102 102 102 177 177 177 F F F
114782024-02-03T17:14:26.104Z 20 400-419 98 98 98 170 170 170 F F F
114792024-02-03T17:14:26.104Z 21 420-439 104 102 104 179 177 179 F F F <---
114802024-02-03T17:14:26.104Z 22 440-459 104 97 104 178 168 178 F F F <---
114812024-02-03T17:14:26.104Z 23 460-479 104 98 104 179 171 179 F F F <---
114822024-02-03T17:14:26.104Z 24 480-499 100 100 100 173 173 173 F F F
114832024-02-03T17:14:26.104Z 25 500-519 97 97 97 168 168 168 F F F
114842024-02-03T17:14:26.105Z 26 520-539 104 97 104 179 168 179 F F F <---
114852024-02-03T17:14:26.105Z 27 540-559 104 100 104 179 174 179 F F F <---
114862024-02-03T17:14:26.105Z 28 560-579 104 102 104 179 177 179 F F F <---
114872024-02-03T17:14:26.105Z 29 580-599 99 99 99 172 172 172 F F F
114882024-02-03T17:14:26.105ZMax gen: 104, Max flush: 179
114892024-02-03T17:14:26.105ZError: Difference in extent metadata found!
114902024-02-03T17:14:26.108ZA Difference in extent metadata is expected here
114912024-02-03T17:14:26.108ZOn loop 52
114922024-02-03T17:14:26.108Z
114932024-02-03T17:14:26.111ZVerifying data now
114942024-02-03T17:14:31.295ZNo change after verify
114952024-02-03T17:14:31.295ZLoop: 52 Downstairs dump after verify (and repair):
114962024-02-03T17:14:31.319ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
114972024-02-03T17:14:31.319Z 0 000-019 104 104 104 179 179 179 F F F
114982024-02-03T17:14:31.319Z 1 020-039 100 100 100 175 175 175 F F F
114992024-02-03T17:14:31.319Z 2 040-059 105 105 105 180 180 180 F F F
115002024-02-03T17:14:31.319Z 3 060-079 97 97 97 168 168 168 F F F
115012024-02-03T17:14:31.319Z 4 080-099 102 102 102 177 177 177 F F F
115022024-02-03T17:14:31.322Z 5 100-119 104 104 104 178 178 178 F F F
115032024-02-03T17:14:31.322Z 6 120-139 98 98 98 171 171 171 F F F
115042024-02-03T17:14:31.322Z 7 140-159 104 104 104 179 179 179 F F F
115052024-02-03T17:14:31.322Z 8 160-179 105 105 105 180 180 180 F F F
115062024-02-03T17:14:31.322Z 9 180-199 104 104 104 179 179 179 F F F
115072024-02-03T17:14:31.322Z 10 200-219 100 100 100 173 173 173 F F F
115082024-02-03T17:14:31.322Z 11 220-239 101 101 101 176 176 176 F F F
115092024-02-03T17:14:31.323Z 12 240-259 94 94 94 164 164 164 F F F
115102024-02-03T17:14:31.323Z 13 260-279 96 96 96 166 166 166 F F F
115112024-02-03T17:14:31.323Z 14 280-299 105 105 105 180 180 180 F F F
115122024-02-03T17:14:31.323Z 15 300-319 105 105 105 180 180 180 F F F
115132024-02-03T17:14:31.323Z 16 320-339 102 102 102 177 177 177 F F F
115142024-02-03T17:14:31.323Z 17 340-359 102 102 102 177 177 177 F F F
115152024-02-03T17:14:31.323Z 18 360-379 101 101 101 176 176 176 F F F
115162024-02-03T17:14:31.323Z 19 380-399 102 102 102 177 177 177 F F F
115172024-02-03T17:14:31.323Z 20 400-419 98 98 98 170 170 170 F F F
115182024-02-03T17:14:31.323Z 21 420-439 104 104 104 179 179 179 F F F
115192024-02-03T17:14:31.323Z 22 440-459 104 104 104 178 178 178 F F F
115202024-02-03T17:14:31.323Z 23 460-479 104 104 104 179 179 179 F F F
115212024-02-03T17:14:31.323Z 24 480-499 100 100 100 173 173 173 F F F
115222024-02-03T17:14:31.323Z 25 500-519 97 97 97 168 168 168 F F F
115232024-02-03T17:14:31.323Z 26 520-539 104 104 104 179 179 179 F F F
115242024-02-03T17:14:31.323Z 27 540-559 104 104 104 179 179 179 F F F
115252024-02-03T17:14:31.323Z 28 560-579 104 104 104 179 179 179 F F F
115262024-02-03T17:14:31.323Z 29 580-599 99 99 99 172 172 172 F F F
115272024-02-03T17:14:31.323ZMax gen: 105, Max flush: 180
115282024-02-03T17:14:31.326Z
115292024-02-03T17:14:31.326ZBegin loop 53
115302024-02-03T17:14:31.326ZDownstairs to restart: 2
115312024-02-03T17:14:31.355Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.917162177Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556}
115322024-02-03T17:14:31.360Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
115332024-02-03T17:14:31.360Z,"time":"2024-02-03T17:14:30.917695525Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556}
115342024-02-03T17:14:31.360Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.917913827Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556}
115352024-02-03T17:14:31.360Z{"msg":"Crucible a5d276f3-5fa5-4ea4-bd69-f985cf1f7939 has session id: 1a37a48d-b490-4829-976e-eb72ce8e181c","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.917951313Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115362024-02-03T17:14:31.360Z{"msg":"Upstairs opts: Upstairs UUID: a5d276f3-5fa5-4ea4-bd69-f985cf1f7939, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.918008292Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115372024-02-03T17:14:31.360Z{"msg":"Crucible stats registered with UUID: a5d276f3-5fa5-4ea4-bd69-f985cf1f7939","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.918143449Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115382024-02-03T17:14:31.360Z{"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.918170182Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556}
115392024-02-03T17:14:31.360Z{"msg":"a5d276f3-5fa5-4ea4-bd69-f985cf1f7939 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.918189837Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115402024-02-03T17:14:31.361Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.91821497Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"client":"0","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115412024-02-03T17:14:31.361Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.918265924Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"client":"0","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115422024-02-03T17:14:31.361Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.918290797Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"client":"1","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115432024-02-03T17:14:31.361Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.918313513Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"client":"1","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115442024-02-03T17:14:31.361Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.918335699Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"client":"2","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115452024-02-03T17:14:31.361Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.91837232Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"client":"2","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115462024-02-03T17:14:31.361Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.91843162Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"":"io task","client":"2","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115472024-02-03T17:14:31.361Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.918471516Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"":"io task","client":"1","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115482024-02-03T17:14:31.361Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.918800776Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"":"io task","client":"0","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115492024-02-03T17:14:31.361Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.918962792Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"":"io task","client":"1","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115502024-02-03T17:14:31.361Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.919025708Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"":"io task","client":"2","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115512024-02-03T17:14:31.361Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.919113522Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"":"io task","client":"0","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115522024-02-03T17:14:31.361Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.920141346Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"client":"0","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115532024-02-03T17:14:31.361Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.920216771Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"client":"0","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115542024-02-03T17:14:31.361Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.92038975Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"client":"1","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115552024-02-03T17:14:31.361Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.920597557Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"client":"1","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115562024-02-03T17:14:31.361Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.920732281Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"client":"2","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115572024-02-03T17:14:31.361Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.920777081Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"client":"2","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115582024-02-03T17:14:31.361Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.921026324Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"client":"0","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115592024-02-03T17:14:31.366Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.921673997Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"client":"1","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115602024-02-03T17:14:31.366Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.921836594Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"client":"2","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115612024-02-03T17:14:31.366Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.922205656Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"client":"0","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115622024-02-03T17:14:31.366Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.922302154Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"client":"0","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115632024-02-03T17:14:31.366Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.922348993Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115642024-02-03T17:14:31.366Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.922900987Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"client":"1","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115652024-02-03T17:14:31.366Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.923039882Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"client":"1","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115662024-02-03T17:14:31.366Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.923124398Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115672024-02-03T17:14:31.366Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.92316506Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"client":"2","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115682024-02-03T17:14:31.366Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.92324434Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"client":"2","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115692024-02-03T17:14:31.366Z{"msg":"[0]R flush_numbers[0..12]: [179, 175, 180, 168, 177, 178, 171, 179, 180, 179, 173, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.923327574Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"":"downstairs","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115702024-02-03T17:14:31.367Z{"msg":"[0]R generation[0..12]: [104, 100, 105, 97, 102, 104, 98, 104, 105, 104, 100, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.923410394Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"":"downstairs","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115712024-02-03T17:14:31.367Z{Wait for a query_work_queue command to finish before sending IO
115722024-02-03T17:14:31.367Z"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.923496598Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"":"downstairs","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115732024-02-03T17:14:31.367Z{"msg":"[1]R flush_numbers[0..12]: [179, 175, 180, 168, 177, 178, 171, 179, 180, 179, 173, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.923581045Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"":"downstairs","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115742024-02-03T17:14:31.367Z{"msg":"[1]R generation[0..12]: [104, 100, 105, 97, 102, 104, 98, 104, 105, 104, 100, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.923668554Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"":"downstairs","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115752024-02-03T17:14:31.367Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.923759616Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"":"downstairs","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115762024-02-03T17:14:31.367Z{"msg":"[2]R flush_numbers[0..12]: [179, 175, 180, 168, 177, 178, 171, 179, 180, 179, 173, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.923842524Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"":"downstairs","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115772024-02-03T17:14:31.367Z{"msg":"[2]R generation[0..12]: [104, 100, 105, 97, 102, 104, 98, 104, 105, 104, 100, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.923925023Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"":"downstairs","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115782024-02-03T17:14:31.367Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.923974524Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"":"downstairs","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115792024-02-03T17:14:31.367Z{"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.924051999Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"":"downstairs","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115802024-02-03T17:14:31.367Z{"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.924137166Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"":"downstairs","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115812024-02-03T17:14:31.367Z{"msg":"Next flush: 181","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.924245158Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"":"downstairs","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115822024-02-03T17:14:31.367Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.92428797Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"":"downstairs","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115832024-02-03T17:14:31.367Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.924363911Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115842024-02-03T17:14:31.367Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.924408103Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"client":"0","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115852024-02-03T17:14:31.367Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.924488626Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"client":"1","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115862024-02-03T17:14:31.367Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
115872024-02-03T17:14:31.367Z,"time":"2024-02-03T17:14:30.924535289Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"client":"2","":"downstairs","session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115882024-02-03T17:14:31.367Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.924661693Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115892024-02-03T17:14:31.367Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.924698808Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115902024-02-03T17:14:31.367Z{"msg":"a5d276f3-5fa5-4ea4-bd69-f985cf1f7939 is now active with session: 1a37a48d-b490-4829-976e-eb72ce8e181c","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.924722354Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115912024-02-03T17:14:31.367Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.924768609Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556,"session_id":"1a37a48d-b490-4829-976e-eb72ce8e181c"}
115922024-02-03T17:14:31.367Z{"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:30.924792215Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556}
115932024-02-03T17:14:31.367ZLoading write count information from file "/tmp/test_repair_verify.data"
115942024-02-03T17:14:31.368ZRun Repair workload
115952024-02-03T17:14:31.368Z01/30 Flush
115962024-02-03T17:14:31.368Z02/30 Flush
115972024-02-03T17:14:31.371Z03/30 Write block 349 len 24576 data: 10 7 7 7 7 7
115982024-02-03T17:14:31.371Z04/30 Read block 483 len 32768
115992024-02-03T17:14:31.371Z05/30 Read block 194 len 36864
116002024-02-03T17:14:31.374Z06/30 Read block 28 len 16384
116012024-02-03T17:14:31.374Z07/30 Write block 12 len 40960 data: 8 10 11 8 8 10 13 15 11 14
116022024-02-03T17:14:31.377Z08/30 Write block 31 len 32768 data: 9 9 7 8 6 8 8 8
116032024-02-03T17:14:31.377Z09/30 Write block 110 len 16384 data: 5 6 8 6
116042024-02-03T17:14:31.377Z10/30 Flush
116052024-02-03T17:14:31.381Z11/30 Write block 435 len 28672 data: 10 8 7 7 7 8 10
116062024-02-03T17:14:31.384Z12/30 Write block 201 len 12288 data: 3 7 7
116072024-02-03T17:14:31.384Z13/30 Write block 131 len 32768 data: 10 9 7 5 3 3 3 4
116082024-02-03T17:14:31.390Z14/30 Write block 561 len 32768 data: 8 10 10 11 10 9 8 11
116092024-02-03T17:14:31.390Z15/30 Write block 30 len 40960 data: 8 10 10 8 9 7 9 9 9 9
116102024-02-03T17:14:31.393Z16/30 Write block 98 len 24576 data: 9 9 9 9 6 7
116112024-02-03T17:14:31.393Z17/30 Read block 190 len 24576
116122024-02-03T17:14:31.393Z18/30 Read block 322 len 36864
116132024-02-03T17:14:31.393Z19/30 Write block 369 len 4096 data: 8
116142024-02-03T17:14:31.396Z20/30 Write block 247 len 8192 data: 5 9
116152024-02-03T17:14:31.396Z21/30 Read block 250 len 40960
116162024-02-03T17:14:31.399Z22/30 Read block 169 len 4096
116172024-02-03T17:14:31.399Z23/30 Write block 472 len 12288 data: 7 6 7
116182024-02-03T17:14:31.402Z24/30 Write block 352 len 20480 data: 8 8 8 9 7
116192024-02-03T17:14:31.402Z25/30 Write block 251 len 8192 data: 6 6
116202024-02-03T17:14:31.402Z26/30 Read block 326 len 36864
116212024-02-03T17:14:31.405Z27/30 Write block 398 len 16384 data: 10 11 9 7
116222024-02-03T17:14:31.405Z28/30 Read block 421 len 24576
116232024-02-03T17:14:31.408Z29/30 Write block 177 len 12288 data: 10 8 8
116242024-02-03T17:14:31.408Z30/30 Write block 346 len 16384 data: 8 8 10 11
116252024-02-03T17:14:31.408Z----------------------------------------------------------------
116262024-02-03T17:14:31.408Z Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:30
116272024-02-03T17:14:31.408ZStates: Active Active Active
116282024-02-03T17:14:31.408ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
116292024-02-03T17:14:31.409Z 1 Acked 1000 Flush 0 Done Done Sent false
116302024-02-03T17:14:31.409Z 2 Acked 1001 Flush 0 Done Done Sent false
116312024-02-03T17:14:31.409Z 3 Acked 1002 Write 6 Done Done Sent false
116322024-02-03T17:14:31.409Z 4 Acked 1003 Read 8 Done Done Sent false
116332024-02-03T17:14:31.409Z 5 Acked 1004 Read 9 Done Done Sent false
116342024-02-03T17:14:31.409Z 6 Acked 1005 Read 4 Done Done Sent false
116352024-02-03T17:14:31.409Z 7 Acked 1006 Write 10 Done Done Sent false
116362024-02-03T17:14:31.409Z 8 Acked 1007 Write 8 Done Done Sent false
116372024-02-03T17:14:31.409Z 9 Acked 1008 Write 4 Done Done Sent false
116382024-02-03T17:14:31.409Z 10 Acked 1009 Flush 0 Done Done Sent false
116392024-02-03T17:14:31.409Z 11 Acked 1010 Write 7 Done Done Sent false
116402024-02-03T17:14:31.409Z 12 Acked 1011 Write 3 Done Done Sent false
116412024-02-03T17:14:31.409Z 13 Acked 1012 Write 8 Done Done Sent false
116422024-02-03T17:14:31.409Z 14 Acked 1013 Write 8 Done Done Sent false
116432024-02-03T17:14:31.409Z 15 Acked 1014 Write 10 Done Done Sent false
116442024-02-03T17:14:31.409Z 16 Acked 1015 Write 6 Done Done Sent false
116452024-02-03T17:14:31.409Z 17 Acked 1016 Read 6 Done Done Sent false
116462024-02-03T17:14:31.409Z 18 Acked 1017 Read 9 Done Done Sent false
116472024-02-03T17:14:31.409Z 19 Acked 1018 Write 1 Done Done Sent false
116482024-02-03T17:14:31.409Z 20 Acked 1019 Write 2 Done Done Sent false
116492024-02-03T17:14:31.409Z 21 Acked 1020 Read 10 Done Done Sent false
116502024-02-03T17:14:31.409Z 22 Acked 1021 Read 1 Done Done Sent false
116512024-02-03T17:14:31.409Z 23 Acked 1022 Write 3 Done Done Sent false
116522024-02-03T17:14:31.409Z 24 Acked 1023 Write 5 Done Done Sent false
116532024-02-03T17:14:31.409Z 25 Acked 1024 Write 2 Done Done Sent false
116542024-02-03T17:14:31.410Z 26 Acked 1025 Read 9 Done Done Sent false
116552024-02-03T17:14:31.410Z 27 Acked 1026 Write 4 Done Done Sent false
116562024-02-03T17:14:31.410Z 28 Acked 1027 Read 6 Done Done Sent false
116572024-02-03T17:14:31.410Z 29 Acked 1028 Write 3 Done Done Sent false
116582024-02-03T17:14:31.410Z 30 Acked 1029 Write 4 Sent Sent Sent false
116592024-02-03T17:14:31.410Z STATES DS:0 DS:1 DS:2 TOTAL
116602024-02-03T17:14:31.410Z New 0 0 0 0
116612024-02-03T17:14:31.410Z Sent 1 1 30 32
116622024-02-03T17:14:31.410Z Done 29 29 0 58
116632024-02-03T17:14:31.410Z Skipped 0 0 0 0
116642024-02-03T17:14:31.410Z Error 0 0 0 0
116652024-02-03T17:14:31.410ZLast Flush: 1009 1009 0
116662024-02-03T17:14:31.410ZDownstairs last five completed:
116672024-02-03T17:14:31.410ZUpstairs last five completed: 30 29 28 27 26
116682024-02-03T17:14:31.410Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:14:30.970222589Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1556}
116692024-02-03T17:14:31.413ZWrote out file "/tmp/test_repair_verify.data"
116702024-02-03T17:14:31.417Z
116712024-02-03T17:14:31.427ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
116722024-02-03T17:14:31.453ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
116732024-02-03T17:14:31.453Z 0 000-019 106 106 104 183 183 179 F F F <---
116742024-02-03T17:14:31.453Z 1 020-039 106 106 100 183 183 175 T T F <---
116752024-02-03T17:14:31.453Z 2 040-059 105 105 105 180 180 180 F F F
116762024-02-03T17:14:31.453Z 3 060-079 97 97 97 168 168 168 F F F
116772024-02-03T17:14:31.453Z 4 080-099 102 102 102 177 177 177 T T F <---
116782024-02-03T17:14:31.453Z 5 100-119 106 106 104 183 183 178 T T F <---
116792024-02-03T17:14:31.456Z 6 120-139 98 98 98 171 171 171 T T F <---
116802024-02-03T17:14:31.457Z 7 140-159 104 104 104 179 179 179 F F F
116812024-02-03T17:14:31.457Z 8 160-179 105 105 105 180 180 180 T T F <---
116822024-02-03T17:14:31.457Z 9 180-199 104 104 104 179 179 179 F F F
116832024-02-03T17:14:31.457Z 10 200-219 100 100 100 173 173 173 T T F <---
116842024-02-03T17:14:31.457Z 11 220-239 101 101 101 176 176 176 F F F
116852024-02-03T17:14:31.457Z 12 240-259 94 94 94 164 164 164 T T F <---
116862024-02-03T17:14:31.457Z 13 260-279 96 96 96 166 166 166 F F F
116872024-02-03T17:14:31.457Z 14 280-299 105 105 105 180 180 180 F F F
116882024-02-03T17:14:31.457Z 15 300-319 105 105 105 180 180 180 F F F
116892024-02-03T17:14:31.457Z 16 320-339 102 102 102 177 177 177 F F F
116902024-02-03T17:14:31.457Z 17 340-359 106 106 102 183 183 177 T T F <---
116912024-02-03T17:14:31.457Z 18 360-379 101 101 101 176 176 176 T T F <---
116922024-02-03T17:14:31.457Z 19 380-399 102 102 102 177 177 177 T T F <---
116932024-02-03T17:14:31.457Z 20 400-419 98 98 98 170 170 170 T T F <---
116942024-02-03T17:14:31.457Z 21 420-439 104 104 104 179 179 179 T T F <---
116952024-02-03T17:14:31.457Z 22 440-459 104 104 104 178 178 178 T T F <---
116962024-02-03T17:14:31.457Z 23 460-479 104 104 104 179 179 179 T T F <---
116972024-02-03T17:14:31.457Z 24 480-499 100 100 100 173 173 173 F F F
116982024-02-03T17:14:31.457Z 25 500-519 97 97 97 168 168 168 F F F
116992024-02-03T17:14:31.457Z 26 520-539 104 104 104 179 179 179 F F F
117002024-02-03T17:14:31.457Z 27 540-559 104 104 104 179 179 179 F F F
117012024-02-03T17:14:31.457Z 28 560-579 104 104 104 179 179 179 T T F <---
117022024-02-03T17:14:31.457Z 29 580-599 99 99 99 172 172 172 F F F
117032024-02-03T17:14:31.458ZMax gen: 106, Max flush: 183
117042024-02-03T17:14:31.458ZError: Difference in extent metadata found!
117052024-02-03T17:14:31.461ZA Difference in extent metadata is expected here
117062024-02-03T17:14:31.461ZOn loop 53
117072024-02-03T17:14:31.461Z
117082024-02-03T17:14:31.467ZVerifying data now
117092024-02-03T17:14:36.845ZNo change after verify
117102024-02-03T17:14:36.845ZLoop: 53 Downstairs dump after verify (and repair):
117112024-02-03T17:14:36.871ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
117122024-02-03T17:14:36.871Z 0 000-019 106 106 106 183 183 183 F F F
117132024-02-03T17:14:36.871Z 1 020-039 107 107 107 184 184 184 F F F
117142024-02-03T17:14:36.871Z 2 040-059 105 105 105 180 180 180 F F F
117152024-02-03T17:14:36.871Z 3 060-079 97 97 97 168 168 168 F F F
117162024-02-03T17:14:36.875Z 4 080-099 107 107 107 184 184 184 F F F
117172024-02-03T17:14:36.875Z 5 100-119 107 107 107 184 184 184 F F F
117182024-02-03T17:14:36.875Z 6 120-139 107 107 107 184 184 184 F F F
117192024-02-03T17:14:36.875Z 7 140-159 104 104 104 179 179 179 F F F
117202024-02-03T17:14:36.875Z 8 160-179 107 107 107 184 184 184 F F F
117212024-02-03T17:14:36.875Z 9 180-199 104 104 104 179 179 179 F F F
117222024-02-03T17:14:36.875Z 10 200-219 107 107 107 184 184 184 F F F
117232024-02-03T17:14:36.875Z 11 220-239 101 101 101 176 176 176 F F F
117242024-02-03T17:14:36.875Z 12 240-259 107 107 107 184 184 184 F F F
117252024-02-03T17:14:36.875Z 13 260-279 96 96 96 166 166 166 F F F
117262024-02-03T17:14:36.875Z 14 280-299 105 105 105 180 180 180 F F F
117272024-02-03T17:14:36.876Z 15 300-319 105 105 105 180 180 180 F F F
117282024-02-03T17:14:36.876Z 16 320-339 102 102 102 177 177 177 F F F
117292024-02-03T17:14:36.876Z 17 340-359 107 107 107 184 184 184 F F F
117302024-02-03T17:14:36.876Z 18 360-379 107 107 107 184 184 184 F F F
117312024-02-03T17:14:36.876Z 19 380-399 107 107 107 184 184 184 F F F
117322024-02-03T17:14:36.876Z 20 400-419 107 107 107 184 184 184 F F F
117332024-02-03T17:14:36.876Z 21 420-439 107 107 107 184 184 184 F F F
117342024-02-03T17:14:36.876Z 22 440-459 107 107 107 184 184 184 F F F
117352024-02-03T17:14:36.876Z 23 460-479 107 107 107 184 184 184 F F F
117362024-02-03T17:14:36.876Z 24 480-499 100 100 100 173 173 173 F F F
117372024-02-03T17:14:36.876Z 25 500-519 97 97 97 168 168 168 F F F
117382024-02-03T17:14:36.876Z 26 520-539 104 104 104 179 179 179 F F F
117392024-02-03T17:14:36.876Z 27 540-559 104 104 104 179 179 179 F F F
117402024-02-03T17:14:36.876Z 28 560-579 107 107 107 184 184 184 F F F
117412024-02-03T17:14:36.876Z 29 580-599 99 99 99 172 172 172 F F F
117422024-02-03T17:14:36.876ZMax gen: 107, Max flush: 184
117432024-02-03T17:14:36.876Z
117442024-02-03T17:14:36.876ZBegin loop 54
117452024-02-03T17:14:36.876ZDownstairs to restart: 1
117462024-02-03T17:14:36.905Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.466867663Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564}
117472024-02-03T17:14:36.905Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.467343999Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564}
117482024-02-03T17:14:36.905Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.467374003Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564}
117492024-02-03T17:14:36.910Z{"msg":"Crucible c3173fbf-e160-4596-93fe-e63e28addd92 has session id: cd594812-5a10-4a8e-887a-bb109b03e4c7","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.467444111Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564Crucible runtime is spawned
117502024-02-03T17:14:36.910Z,"session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117512024-02-03T17:14:36.910Z{"msg":"Upstairs opts: Upstairs UUID: c3173fbf-e160-4596-93fe-e63e28addd92, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.467621205Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117522024-02-03T17:14:36.910Z{"msg":"Crucible stats registered with UUID: c3173fbf-e160-4596-93fe-e63e28addd92","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.467702745Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117532024-02-03T17:14:36.910Z{"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.467746746Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564}
117542024-02-03T17:14:36.910Z{"msg":"c3173fbf-e160-4596-93fe-e63e28addd92 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.467810215Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117552024-02-03T17:14:36.910Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.467851853Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"client":"0","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117562024-02-03T17:14:36.910Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.467929219Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"client":"0","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117572024-02-03T17:14:36.910Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.467973316Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"client":"1","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117582024-02-03T17:14:36.910Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.468013139Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"client":"1","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117592024-02-03T17:14:36.910Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.468055686Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"client":"2","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117602024-02-03T17:14:36.910Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.468095486Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"client":"2","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117612024-02-03T17:14:36.910Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.468126345Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"":"io task","client":"2","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117622024-02-03T17:14:36.910Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.468150296Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"":"io task","client":"1","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117632024-02-03T17:14:36.910Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.468217592Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"":"io task","client":"0","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117642024-02-03T17:14:36.910Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.468461642Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"":"io task","client":"2","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117652024-02-03T17:14:36.910Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.46859621Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"":"io task","client":"0","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117662024-02-03T17:14:36.911Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.468647648Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"":"io task","client":"1","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117672024-02-03T17:14:36.911Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.470020841Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"client":"0","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117682024-02-03T17:14:36.911Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.47011816Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"client":"0","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117692024-02-03T17:14:36.911Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.47016731Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"client":"2","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117702024-02-03T17:14:36.911Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.470328175Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"client":"2","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117712024-02-03T17:14:36.911Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.470538556Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"client":"1","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117722024-02-03T17:14:36.911Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.470685309Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"client":"1","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117732024-02-03T17:14:36.917Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.471312624Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"client":"0","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117742024-02-03T17:14:36.917Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.471498253Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"client":"2","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117752024-02-03T17:14:36.917Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.471549756Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"client":"1","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117762024-02-03T17:14:36.917Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.472118292Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"client":"0","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117772024-02-03T17:14:36.917Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.472187359Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"client":"0","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117782024-02-03T17:14:36.917Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.472317878Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117792024-02-03T17:14:36.917Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.472362514Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"client":"1","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117802024-02-03T17:14:36.917Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.47242352Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"client":"1","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117812024-02-03T17:14:36.917Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.47244976Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117822024-02-03T17:14:36.917Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.472495729Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"client":"2","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117832024-02-03T17:14:36.917Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.472520091Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"client":"2","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117842024-02-03T17:14:36.918Z{"msg":"[0]R flush_numbers[0..12]: [183, 184, 180, 168, 184, 184, 184, 179, 184, 179, 184, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.472543215Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"":"downstairs","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117852024-02-03T17:14:36.918Z{"msg":"[0]R generation[0..12]: [106, 107, 105, 97, 107, 107, 107, 104, 107, 104, 107, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.472566553Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"":"downstairs","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117862024-02-03T17:14:36.918Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.472614838Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"":"downstairs","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117872024-02-03T17:14:36.918Z{"msg":"[1]R flush_numbers[0..12]: [183, 184, 180, 168, 184, 184, 184, 179, 184, 179, 184, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.472639904Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"":"downstairs","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117882024-02-03T17:14:36.918Z{"msg":"[1]R generation[0..12]: [106, 107, 105, 97, 107, 107, 107, 104, 107, 104, 107, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.472663055Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"":"downstairs","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117892024-02-03T17:14:36.918Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible",Wait for a query_work_queue command to finish before sending IO
117902024-02-03T17:14:36.918Z"level":30,"time":"2024-02-03T17:14:36.472737178Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"":"downstairs","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117912024-02-03T17:14:36.918Z{"msg":"[2]R flush_numbers[0..12]: [183, 184, 180, 168, 184, 184, 184, 179, 184, 179, 184, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.472762826Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"":"downstairs","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117922024-02-03T17:14:36.918Z{"msg":"[2]R generation[0..12]: [106, 107, 105, 97, 107, 107, 107, 104, 107, 104, 107, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.472803842Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"":"downstairs","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117932024-02-03T17:14:36.918Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.472829584Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"":"downstairs","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117942024-02-03T17:14:36.918Z{"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.472851908Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"":"downstairs","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117952024-02-03T17:14:36.918Z{"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.472901339Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"":"downstairs","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117962024-02-03T17:14:36.918Z{"msg":"Next flush: 185","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.472925109Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"":"downstairs","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117972024-02-03T17:14:36.918Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.472946837Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"":"downstairs","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117982024-02-03T17:14:36.918Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.472968909Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
117992024-02-03T17:14:36.918Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.472988648Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"client":"0","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
118002024-02-03T17:14:36.918Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.473029656Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"client":"1","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
118012024-02-03T17:14:36.918Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.473052299Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"client":"2","":"downstairs","session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
118022024-02-03T17:14:36.918Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.473074Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
118032024-02-03T17:14:36.918Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.473094726Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
118042024-02-03T17:14:36.918Z{"msg":"c3173fbf-e160-4596-93fe-e63e28addd92 is now active with session: cd594812-5a10-4a8e-887a-bb109b03e4c7","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.47311786Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
118052024-02-03T17:14:36.918Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.473138141Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564,"session_id":"cd594812-5a10-4a8e-887a-bb109b03e4c7"}
118062024-02-03T17:14:36.918Z{"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:36.473158392Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564}
118072024-02-03T17:14:36.918ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
118082024-02-03T17:14:36.919ZLoading write count information from file "/tmp/test_repair_verify.data"
118092024-02-03T17:14:36.919ZRun Repair workload
118102024-02-03T17:14:36.919Z01/30 Write block 447 len 16384 data: 12 12 9 7
118112024-02-03T17:14:36.919Z02/30 Flush
118122024-02-03T17:14:36.922Z03/30 Write block 557 len 40960 data: 7 6 7 8 9 11 11 12 11 10
118132024-02-03T17:14:36.922Z04/30 Write block 335 len 16384 data: 6 7 8 10
118142024-02-03T17:14:36.922Z05/30 Read block 49 len 8192
118152024-02-03T17:14:36.922Z06/30 Read block 52 len 28672
118162024-02-03T17:14:36.925Z07/30 Flush
118172024-02-03T17:14:36.928Z08/30 Write block 542 len 8192 data: 10 9
118182024-02-03T17:14:36.928Z09/30 Read block 31 len 16384
118192024-02-03T17:14:36.931Z10/30 Write block 90 len 16384 data: 11 10 9 12
118202024-02-03T17:14:36.931Z11/30 Read block 167 len 32768
118212024-02-03T17:14:36.934Z12/30 Write block 117 len 24576 data: 12 10 9 8 7 9
118222024-02-03T17:14:36.935Z13/30 Write block 387 len 32768 data: 8 7 5 10 11 15 15 12
118232024-02-03T17:14:36.937Z14/30 Write block 516 len 40960 data: 10 9 6 6 7 5 6 7 6 8
118242024-02-03T17:14:36.937Z15/30 Flush
118252024-02-03T17:14:36.942Z16/30 Write block 427 len 16384 data: 10 10 12 12
118262024-02-03T17:14:36.945Z17/30 Read block 588 len 36864
118272024-02-03T17:14:36.945Z18/30 Write block 327 len 20480 data: 10 9 10 8 8
118282024-02-03T17:14:36.948Z19/30 Write block 326 len 28672 data: 7 11 10 11 9 9 8
118292024-02-03T17:14:36.948Z20/30 Write block 505 len 40960 data: 10 11 12 12 12 10 12 12 9 9
118302024-02-03T17:14:36.951Z21/30 Read block 407 len 12288
118312024-02-03T17:14:36.954Z22/30 Write block 511 len 32768 data: 13 13 10 10 11 11 10 7
118322024-02-03T17:14:36.954Z23/30 Read block 139 len 8192
118332024-02-03T17:14:36.956Z24/30 Flush
118342024-02-03T17:14:36.959Z25/30 Flush
118352024-02-03T17:14:36.959Z26/30 Write block 137 len 24576 data: 4 5 5 5 8 8
118362024-02-03T17:14:36.963Z27/30 Read block 512 len 20480
118372024-02-03T17:14:36.963Z28/30 Write block 350 len 28672 data: 8 8 9 9 9 10 8
118382024-02-03T17:14:36.968Z29/30 Read block 568 len 28672
118392024-02-03T17:14:36.972Z30/30 Write block 251 len 40960 data: 7 7 5 4 5 7 7 9 8 11
118402024-02-03T17:14:36.972Z----------------------------------------------------------------
118412024-02-03T17:14:36.972Z Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:28
118422024-02-03T17:14:36.972ZStates: Active Active Active
118432024-02-03T17:14:36.972ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
118442024-02-03T17:14:36.972Z 3 Acked 1002 Write 10 Done Done Done false
118452024-02-03T17:14:36.972Z 4 Acked 1003 Write 4 Done Done Done false
118462024-02-03T17:14:36.972Z 5 Acked 1004 Read 2 Done Done Done false
118472024-02-03T17:14:36.972Z 6 Acked 1005 Read 7 Done Done Done false
118482024-02-03T17:14:36.972Z 7 Acked 1006 Flush 0 Done Sent Done false
118492024-02-03T17:14:36.972Z 8 Acked 1007 Write 2 Done Sent Done false
118502024-02-03T17:14:36.972Z 9 Acked 1008 Read 4 Done Sent Done false
118512024-02-03T17:14:36.972Z 10 Acked 1009 Write 4 Done Sent Done false
118522024-02-03T17:14:36.972Z 11 Acked 1010 Read 8 Done Sent Done false
118532024-02-03T17:14:36.972Z 12 Acked 1011 Write 6 Done Sent Done false
118542024-02-03T17:14:36.972Z 13 Acked 1012 Write 8 Done Sent Done false
118552024-02-03T17:14:36.972Z 14 Acked 1013 Write 10 Done Sent Done false
118562024-02-03T17:14:36.972Z 15 Acked 1014 Flush 0 Done Sent Done false
118572024-02-03T17:14:36.972Z 16 Acked 1015 Write 4 Done Sent Done false
118582024-02-03T17:14:36.972Z 17 Acked 1016 Read 9 Done Sent Done false
118592024-02-03T17:14:36.972Z 18 Acked 1017 Write 5 Done Sent Done false
118602024-02-03T17:14:36.972Z 19 Acked 1018 Write 7 Done Sent Done false
118612024-02-03T17:14:36.973Z 20 Acked 1019 Write 10 Done Sent Done false
118622024-02-03T17:14:36.973Z 21 Acked 1020 Read 3 Done Sent Done false
118632024-02-03T17:14:36.973Z 22 Acked 1021 Write 8 Done Sent Done false
118642024-02-03T17:14:36.973Z 23 Acked 1022 Read 2 Done Sent Done false
118652024-02-03T17:14:36.973Z 24 Acked 1023 Flush 0 Done Sent Done false
118662024-02-03T17:14:36.973Z 25 Acked 1024 Flush 0 Done Sent Done false
118672024-02-03T17:14:36.973Z 26 Acked 1025 Write 6 Done Sent Done false
118682024-02-03T17:14:36.973Z 27 Acked 1026 Read 5 Done Sent Done false
118692024-02-03T17:14:36.973Z 28 Acked 1027 Write 7 Done Sent Done false
118702024-02-03T17:14:36.973Z 29 Acked 1028 Read 7 Done Sent Done false
118712024-02-03T17:14:36.973Z 30 Acked 1029 Write 10 Sent Sent Sent false
118722024-02-03T17:14:36.973Z STATES DS:0 DS:1 DS:2 TOTAL
118732024-02-03T17:14:36.973Z New 0 0 0 0
118742024-02-03T17:14:36.973Z Sent 1 24 1 26
118752024-02-03T17:14:36.973Z Done 27 4 27 58
118762024-02-03T17:14:36.973Z Skipped 0 0 0 0
118772024-02-03T17:14:36.973Z Error 0 0 0 0
118782024-02-03T17:14:36.973ZLast Flush: 1024 1001 1024
118792024-02-03T17:14:36.973ZDownstairs last five completed: 1001 1000
118802024-02-03T17:14:36.973ZUpstairs last five completed: 30 29 28 27 26
118812024-02-03T17:14:36.973Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:14:36.530449101Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1564}
118822024-02-03T17:14:36.973ZWrote out file "/tmp/test_repair_verify.data"
118832024-02-03T17:14:36.977Z
118842024-02-03T17:14:36.981ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
118852024-02-03T17:14:37.008ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
118862024-02-03T17:14:37.009Z 0 000-019 106 106 106 183 183 183 F F F
118872024-02-03T17:14:37.009Z 1 020-039 107 107 107 184 184 184 F F F
118882024-02-03T17:14:37.009Z 2 040-059 105 105 105 180 180 180 F F F
118892024-02-03T17:14:37.009Z 3 060-079 97 97 97 168 168 168 F F F
118902024-02-03T17:14:37.012Z 4 080-099 108 107 108 187 184 187 F F F <---
118912024-02-03T17:14:37.012Z 5 100-119 108 107 108 187 184 187 F F F <---
118922024-02-03T17:14:37.012Z 6 120-139 108 107 108 187 184 187 T F T <---
118932024-02-03T17:14:37.012Z 7 140-159 104 104 104 179 179 179 T F T <---
118942024-02-03T17:14:37.012Z 8 160-179 107 107 107 184 184 184 F F F
118952024-02-03T17:14:37.012Z 9 180-199 104 104 104 179 179 179 F F F
118962024-02-03T17:14:37.012Z 10 200-219 107 107 107 184 184 184 F F F
118972024-02-03T17:14:37.012Z 11 220-239 101 101 101 176 176 176 F F F
118982024-02-03T17:14:37.012Z 12 240-259 107 107 107 184 184 184 T F T <---
118992024-02-03T17:14:37.012Z 13 260-279 96 96 96 166 166 166 T F T <---
119002024-02-03T17:14:37.012Z 14 280-299 105 105 105 180 180 180 F F F
119012024-02-03T17:14:37.012Z 15 300-319 105 105 105 180 180 180 F F F
119022024-02-03T17:14:37.012Z 16 320-339 108 102 108 188 177 188 F T F <---
119032024-02-03T17:14:37.012Z 17 340-359 107 107 107 184 184 184 T F T <---
119042024-02-03T17:14:37.012Z 18 360-379 107 107 107 184 184 184 F F F
119052024-02-03T17:14:37.012Z 19 380-399 108 107 108 187 184 187 F F F <---
119062024-02-03T17:14:37.013Z 20 400-419 107 107 107 184 184 184 F F F
119072024-02-03T17:14:37.013Z 21 420-439 108 107 108 188 184 188 F F F <---
119082024-02-03T17:14:37.013Z 22 440-459 108 108 108 185 185 185 F F F
119092024-02-03T17:14:37.013Z 23 460-479 107 107 107 184 184 184 F F F
119102024-02-03T17:14:37.013Z 24 480-499 100 100 100 173 173 173 F F F
119112024-02-03T17:14:37.013Z 25 500-519 108 97 108 188 168 188 F F F <---
119122024-02-03T17:14:37.013Z 26 520-539 108 104 108 187 179 187 F F F <---
119132024-02-03T17:14:37.013Z 27 540-559 108 104 108 187 179 187 F T F <---
119142024-02-03T17:14:37.013Z 28 560-579 108 107 108 186 184 186 F T F <---
119152024-02-03T17:14:37.013Z 29 580-599 99 99 99 172 172 172 F F F
119162024-02-03T17:14:37.013ZMax gen: 108, Max flush: 188
119172024-02-03T17:14:37.013ZError: Difference in extent metadata found!
119182024-02-03T17:14:37.016ZA Difference in extent metadata is expected here
119192024-02-03T17:14:37.016ZOn loop 54
119202024-02-03T17:14:37.016Z
119212024-02-03T17:14:37.019ZVerifying data now
119222024-02-03T17:14:43.234ZNo change after verify
119232024-02-03T17:14:43.234ZLoop: 54 Downstairs dump after verify (and repair):
119242024-02-03T17:14:43.261ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
119252024-02-03T17:14:43.261Z 0 000-019 106 106 106 183 183 183 F F F
119262024-02-03T17:14:43.261Z 1 020-039 107 107 107 184 184 184 F F F
119272024-02-03T17:14:43.261Z 2 040-059 105 105 105 180 180 180 F F F
119282024-02-03T17:14:43.261Z 3 060-079 97 97 97 168 168 168 F F F
119292024-02-03T17:14:43.261Z 4 080-099 108 108 108 187 187 187 F F F
119302024-02-03T17:14:43.261Z 5 100-119 108 108 108 187 187 187 F F F
119312024-02-03T17:14:43.261Z 6 120-139 109 109 109 189 189 189 F F F
119322024-02-03T17:14:43.265Z 7 140-159 109 109 109 189 189 189 F F F
119332024-02-03T17:14:43.265Z 8 160-179 107 107 107 184 184 184 F F F
119342024-02-03T17:14:43.265Z 9 180-199 104 104 104 179 179 179 F F F
119352024-02-03T17:14:43.265Z 10 200-219 107 107 107 184 184 184 F F F
119362024-02-03T17:14:43.265Z 11 220-239 101 101 101 176 176 176 F F F
119372024-02-03T17:14:43.265Z 12 240-259 109 109 109 189 189 189 F F F
119382024-02-03T17:14:43.265Z 13 260-279 109 109 109 189 189 189 F F F
119392024-02-03T17:14:43.265Z 14 280-299 105 105 105 180 180 180 F F F
119402024-02-03T17:14:43.265Z 15 300-319 105 105 105 180 180 180 F F F
119412024-02-03T17:14:43.265Z 16 320-339 108 108 108 188 188 188 F F F
119422024-02-03T17:14:43.265Z 17 340-359 109 109 109 189 189 189 F F F
119432024-02-03T17:14:43.266Z 18 360-379 107 107 107 184 184 184 F F F
119442024-02-03T17:14:43.266Z 19 380-399 108 108 108 187 187 187 F F F
119452024-02-03T17:14:43.266Z 20 400-419 107 107 107 184 184 184 F F F
119462024-02-03T17:14:43.266Z 21 420-439 108 108 108 188 188 188 F F F
119472024-02-03T17:14:43.266Z 22 440-459 108 108 108 185 185 185 F F F
119482024-02-03T17:14:43.266Z 23 460-479 107 107 107 184 184 184 F F F
119492024-02-03T17:14:43.266Z 24 480-499 100 100 100 173 173 173 F F F
119502024-02-03T17:14:43.266Z 25 500-519 108 108 108 188 188 188 F F F
119512024-02-03T17:14:43.266Z 26 520-539 108 108 108 187 187 187 F F F
119522024-02-03T17:14:43.266Z 27 540-559 108 108 108 187 187 187 F F F
119532024-02-03T17:14:43.266Z 28 560-579 108 108 108 186 186 186 F F F
119542024-02-03T17:14:43.266Z 29 580-599 99 99 99 172 172 172 F F F
119552024-02-03T17:14:43.266ZMax gen: 109, Max flush: 189
119562024-02-03T17:14:43.266Z
119572024-02-03T17:14:43.266ZBegin loop 55
119582024-02-03T17:14:43.266ZDownstairs to restart: 0
119592024-02-03T17:14:43.296Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.859038835Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572}
119602024-02-03T17:14:43.302Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
119612024-02-03T17:14:43.302Z,"time":"2024-02-03T17:14:42.859624947Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572}
119622024-02-03T17:14:43.302Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.859774217Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572}
119632024-02-03T17:14:43.302Z{"msg":"Crucible 2a24fe24-189e-4cd6-850a-57221aa65f07 has session id: 80bc05bd-032a-4e21-bb06-66f0eadd4d95","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.859889474Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119642024-02-03T17:14:43.302Z{"msg":"Upstairs opts: Upstairs UUID: 2a24fe24-189e-4cd6-850a-57221aa65f07, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.859948812Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119652024-02-03T17:14:43.302Z{"msg":"Crucible stats registered with UUID: 2a24fe24-189e-4cd6-850a-57221aa65f07","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.86002339Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119662024-02-03T17:14:43.302Z{"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.860127695Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572}
119672024-02-03T17:14:43.302Z{"msg":"2a24fe24-189e-4cd6-850a-57221aa65f07 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.860234458Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119682024-02-03T17:14:43.302Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.860309228Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"client":"0","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119692024-02-03T17:14:43.302Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.860392726Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"client":"0","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119702024-02-03T17:14:43.302Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.860436472Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"client":"1","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119712024-02-03T17:14:43.302Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.860478648Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"client":"1","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119722024-02-03T17:14:43.302Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.860560062Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"client":"2","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119732024-02-03T17:14:43.302Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.860601505Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"client":"2","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119742024-02-03T17:14:43.302Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.860679205Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"":"io task","client":"2","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119752024-02-03T17:14:43.302Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.860713406Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"":"io task","client":"0","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119762024-02-03T17:14:43.302Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.860740151Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"":"io task","client":"1","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119772024-02-03T17:14:43.302Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.860918778Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"":"io task","client":"1","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119782024-02-03T17:14:43.302Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.861043925Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"":"io task","client":"2","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119792024-02-03T17:14:43.303Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.86109204Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"":"io task","client":"0","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119802024-02-03T17:14:43.303Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.862602333Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"client":"1","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119812024-02-03T17:14:43.303Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.862704983Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"client":"1","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119822024-02-03T17:14:43.303Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.862750971Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"client":"2","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119832024-02-03T17:14:43.303Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.862902719Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"client":"2","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119842024-02-03T17:14:43.303Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.863415643Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"client":"0","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119852024-02-03T17:14:43.303Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.863472127Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"client":"0","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119862024-02-03T17:14:43.308Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.863804446Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"client":"1","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119872024-02-03T17:14:43.308Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.864064473Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"client":"2","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119882024-02-03T17:14:43.308Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.864509857Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"client":"0","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119892024-02-03T17:14:43.308Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.864977347Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"client":"1","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119902024-02-03T17:14:43.308Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.865046575Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"client":"1","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119912024-02-03T17:14:43.308Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.86507263Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119922024-02-03T17:14:43.308Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.865093969Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"client":"2","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119932024-02-03T17:14:43.309Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.865146983Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"client":"2","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119942024-02-03T17:14:43.309Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.865171451Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119952024-02-03T17:14:43.309Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.86543532Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"client":"0","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119962024-02-03T17:14:43.309Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.865528773Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"client":"0","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119972024-02-03T17:14:43.309Z{"msg":"[0]R flush_numbers[0..12]: [183, 184, 180, 168, 187, 187, 189, 189, 184, 179, 184, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.865573877Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"":"downstairs","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119982024-02-03T17:14:43.309Z{"msg":"[0]R generation[0..12]: [106, 107, 105, 97, 108, 108, 109, 109, 107, 104, 107, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.865656197Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"":"downstairs","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
119992024-02-03T17:14:43.309Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.865738664Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"":"downstairs","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
120002024-02-03T17:14:43.309Z{"msg":"[1]R flush_numbers[0..12]: [183, 184, 180, 168, 187, 187, 189, 189, 184, 179, 184, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.865827574Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"":"downstairs","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
120012024-02-03T17:14:43.309ZWait for a query_work_queue command to finish before sending IO
120022024-02-03T17:14:43.309Z{"msg":"[1]R generation[0..12]: [106, 107, 105, 97, 108, 108, 109, 109, 107, 104, 107, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.865882307Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"":"downstairs","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
120032024-02-03T17:14:43.309Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.865931166Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"":"downstairs","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
120042024-02-03T17:14:43.309Z{"msg":"[2]R flush_numbers[0..12]: [183, 184, 180, 168, 187, 187, 189, 189, 184, 179, 184, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.866016661Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"":"downstairs","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
120052024-02-03T17:14:43.309Z{"msg":"[2]R generation[0..12]: [106, 107, 105, 97, 108, 108, 109, 109, 107, 104, 107, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.866102155Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"":"downstairs","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
120062024-02-03T17:14:43.309Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.866149783Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"":"downstairs","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
120072024-02-03T17:14:43.309Z{"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.866230502Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"":"downstairs","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
120082024-02-03T17:14:43.309Z{"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.86627479Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"":"downstairs","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
120092024-02-03T17:14:43.309Z{"msg":"Next flush: 190","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.866353965Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"":"downstairs","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
120102024-02-03T17:14:43.309Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.866397483Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"":"downstairs","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
120112024-02-03T17:14:43.309Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.866477447Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
120122024-02-03T17:14:43.309Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.866519905Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"client":"0","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
120132024-02-03T17:14:43.309Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.866600747Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"client":"1"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
120142024-02-03T17:14:43.309Z,"":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
120152024-02-03T17:14:43.309Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.86665301Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"client":"2","":"downstairs","session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
120162024-02-03T17:14:43.310Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.866714652Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
120172024-02-03T17:14:43.310Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.866738381Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
120182024-02-03T17:14:43.310Z{"msg":"2a24fe24-189e-4cd6-850a-57221aa65f07 is now active with session: 80bc05bd-032a-4e21-bb06-66f0eadd4d95","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.866760409Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
120192024-02-03T17:14:43.310Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.866808353Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572,"session_id":"80bc05bd-032a-4e21-bb06-66f0eadd4d95"}
120202024-02-03T17:14:43.310Z{"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:42.86683114Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572}
120212024-02-03T17:14:43.310ZLoading write count information from file "/tmp/test_repair_verify.data"
120222024-02-03T17:14:43.310ZRun Repair workload
120232024-02-03T17:14:43.310Z01/30 Write block 52 len 8192 data: 8 6
120242024-02-03T17:14:43.314Z02/30 Write block 405 len 36864 data: 8 9 9 7 7 8 8 9 8
120252024-02-03T17:14:43.314Z03/30 Read block 426 len 8192
120262024-02-03T17:14:43.314Z04/30 Read block 4 len 32768
120272024-02-03T17:14:43.314Z05/30 Write block 379 len 24576 data: 10 8 7 8 8 7
120282024-02-03T17:14:43.314Z06/30 Read block 293 len 32768
120292024-02-03T17:14:43.317Z07/30 Write block 187 len 36864 data: 17 19 20 18 13 13 10 10 10
120302024-02-03T17:14:43.320Z08/30 Write block 141 len 12288 data: 9 9 8
120312024-02-03T17:14:43.320Z09/30 Read block 249 len 8192
120322024-02-03T17:14:43.320Z10/30 Read block 184 len 36864
120332024-02-03T17:14:43.323Z11/30 Write block 575 len 4096 data: 7
120342024-02-03T17:14:43.323Z12/30 Read block 435 len 8192
120352024-02-03T17:14:43.323Z13/30 Read block 408 len 20480
120362024-02-03T17:14:43.326Z14/30 Read block 533 len 4096
120372024-02-03T17:14:43.326Z15/30 Read block 482 len 24576
120382024-02-03T17:14:43.333Z16/30 Read block 277 len 36864
120392024-02-03T17:14:43.333Z17/30 Read block 471 len 20480
120402024-02-03T17:14:43.337Z18/30 Write block 319 len 24576 data: 12 11 12 11 12 11
120412024-02-03T17:14:43.337Z19/30 Read block 66 len 36864
120422024-02-03T17:14:43.337Z20/30 Read block 171 len 40960
120432024-02-03T17:14:43.337Z21/30 Write block 135 len 36864 data: 4 4 5 6 6 6 10 10 9
120442024-02-03T17:14:43.340Z22/30 Write block 131 len 28672 data: 11 10 8 6 5 5 6
120452024-02-03T17:14:43.340Z23/30 Read block 414 len 20480
120462024-02-03T17:14:43.340Z24/30 Read block 317 len 36864
120472024-02-03T17:14:43.343Z25/30 Read block 317 len 24576
120482024-02-03T17:14:43.346Z26/30 Read block 387 len 16384
120492024-02-03T17:14:43.346Z27/30 Write block 404 len 12288 data: 7 9 10
120502024-02-03T17:14:43.350Z28/30 Write block 56 len 24576 data: 8 7 7 8 8 6
120512024-02-03T17:14:43.351Z29/30 Write block 146 len 16384 data: 5 4 7 7
120522024-02-03T17:14:43.351Z30/30 Read block 10 len 16384
120532024-02-03T17:14:43.357Z----------------------------------------------------------------
120542024-02-03T17:14:43.357Z Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30
120552024-02-03T17:14:43.357ZStates: Active Active Active
120562024-02-03T17:14:43.357ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
120572024-02-03T17:14:43.357Z 1 Acked 1000 Write 2 Done Done Done false
120582024-02-03T17:14:43.357Z 2 Acked 1001 Write 9 Done Done Done false
120592024-02-03T17:14:43.357Z 3 Acked 1002 Read 2 Done Done Done false
120602024-02-03T17:14:43.357Z 4 Acked 1003 Read 8 Done Done Done false
120612024-02-03T17:14:43.357Z 5 Acked 1004 Write 6 Sent Done Done false
120622024-02-03T17:14:43.357Z 6 Acked 1005 Read 8 Sent Done Done false
120632024-02-03T17:14:43.357Z 7 Acked 1006 Write 9 Sent Done Done false
120642024-02-03T17:14:43.357Z 8 Acked 1007 Write 3 Sent Done Done false
120652024-02-03T17:14:43.357Z 9 Acked 1008 Read 2 Sent Done Done false
120662024-02-03T17:14:43.357Z 10 Acked 1009 Read 9 Sent Done Done false
120672024-02-03T17:14:43.357Z 11 Acked 1010 Write 1 Sent Done Done false
120682024-02-03T17:14:43.357Z 12 Acked 1011 Read 2 Sent Done Done false
120692024-02-03T17:14:43.357Z 13 Acked 1012 Read 5 Sent Done Done false
120702024-02-03T17:14:43.358Z 14 Acked 1013 Read 1 Sent Done Done false
120712024-02-03T17:14:43.358Z 15 Acked 1014 Read 6 Sent Done Done false
120722024-02-03T17:14:43.358Z 16 Acked 1015 Read 9 Sent Done Done false
120732024-02-03T17:14:43.358Z 17 Acked 1016 Read 5 Sent Done Done false
120742024-02-03T17:14:43.358Z 18 Acked 1017 Write 6 Sent Done Done false
120752024-02-03T17:14:43.358Z 19 Acked 1018 Read 9 Sent Done Done false
120762024-02-03T17:14:43.358Z 20 Acked 1019 Read 10 Sent Done Done false
120772024-02-03T17:14:43.358Z 21 Acked 1020 Write 9 Sent Done Done false
120782024-02-03T17:14:43.358Z 22 Acked 1021 Write 7 Sent Done Done false
120792024-02-03T17:14:43.358Z 23 Acked 1022 Read 5 Sent Done Done false
120802024-02-03T17:14:43.358Z 24 Acked 1023 Read 9 Sent Done Done false
120812024-02-03T17:14:43.358Z 25 Acked 1024 Read 6 Sent Done Done false
120822024-02-03T17:14:43.358Z 26 Acked 1025 Read 4 Sent Done Done false
120832024-02-03T17:14:43.358Z 27 Acked 1026 Write 3 Sent Done Done false
120842024-02-03T17:14:43.358Z 28 Acked 1027 Write 6 Sent Done Done false
120852024-02-03T17:14:43.358Z 29 Acked 1028 Write 4 Sent Done Done false
120862024-02-03T17:14:43.358Z 30 Acked 1029 Read 4 Sent Done Sent false
120872024-02-03T17:14:43.358Z STATES DS:0 DS:1 DS:2 TOTAL
120882024-02-03T17:14:43.358Z New 0 0 0 0
120892024-02-03T17:14:43.358Z Sent 26 0 1 27
120902024-02-03T17:14:43.358Z Done 4 30 29 63
120912024-02-03T17:14:43.358Z Skipped 0 0 0 0
120922024-02-03T17:14:43.358Z Error 0 0 0 0
120932024-02-03T17:14:43.358ZLast Flush: 0 0 0
120942024-02-03T17:14:43.358ZDownstairs last five completed:
120952024-02-03T17:14:43.359ZUpstairs last five completed: 30 29 28 27 26
120962024-02-03T17:14:43.359Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:14:42.914064572Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1572}
120972024-02-03T17:14:43.359ZWrote out file "/tmp/test_repair_verify.data"
120982024-02-03T17:14:43.362Z
120992024-02-03T17:14:43.366ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
121002024-02-03T17:14:43.396ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
121012024-02-03T17:14:43.397Z 0 000-019 106 106 106 183 183 183 F F F
121022024-02-03T17:14:43.397Z 1 020-039 107 107 107 184 184 184 F F F
121032024-02-03T17:14:43.397Z 2 040-059 105 105 105 180 180 180 T T T
121042024-02-03T17:14:43.397Z 3 060-079 97 97 97 168 168 168 F T T <---
121052024-02-03T17:14:43.403Z 4 080-099 108 108 108 187 187 187 F F F
121062024-02-03T17:14:43.403Z 5 100-119 108 108 108 187 187 187 F F F
121072024-02-03T17:14:43.403Z 6 120-139 109 109 109 189 189 189 F T T <---
121082024-02-03T17:14:43.403Z 7 140-159 109 109 109 189 189 189 F T T <---
121092024-02-03T17:14:43.403Z 8 160-179 107 107 107 184 184 184 F F F
121102024-02-03T17:14:43.403Z 9 180-199 104 104 104 179 179 179 F T T <---
121112024-02-03T17:14:43.403Z 10 200-219 107 107 107 184 184 184 F F F
121122024-02-03T17:14:43.403Z 11 220-239 101 101 101 176 176 176 F F F
121132024-02-03T17:14:43.403Z 12 240-259 109 109 109 189 189 189 F F F
121142024-02-03T17:14:43.403Z 13 260-279 109 109 109 189 189 189 F F F
121152024-02-03T17:14:43.403Z 14 280-299 105 105 105 180 180 180 F F F
121162024-02-03T17:14:43.404Z 15 300-319 105 105 105 180 180 180 F T T <---
121172024-02-03T17:14:43.404Z 16 320-339 108 108 108 188 188 188 F T T <---
121182024-02-03T17:14:43.404Z 17 340-359 109 109 109 189 189 189 F F F
121192024-02-03T17:14:43.404Z 18 360-379 107 107 107 184 184 184 F T T <---
121202024-02-03T17:14:43.404Z 19 380-399 108 108 108 187 187 187 F T T <---
121212024-02-03T17:14:43.404Z 20 400-419 107 107 107 184 184 184 T T T
121222024-02-03T17:14:43.404Z 21 420-439 108 108 108 188 188 188 F F F
121232024-02-03T17:14:43.404Z 22 440-459 108 108 108 185 185 185 F F F
121242024-02-03T17:14:43.404Z 23 460-479 107 107 107 184 184 184 F F F
121252024-02-03T17:14:43.404Z 24 480-499 100 100 100 173 173 173 F F F
121262024-02-03T17:14:43.404Z 25 500-519 108 108 108 188 188 188 F F F
121272024-02-03T17:14:43.404Z 26 520-539 108 108 108 187 187 187 F F F
121282024-02-03T17:14:43.404Z 27 540-559 108 108 108 187 187 187 F F F
121292024-02-03T17:14:43.404Z 28 560-579 108 108 108 186 186 186 F T T <---
121302024-02-03T17:14:43.404Z 29 580-599 99 99 99 172 172 172 F F F
121312024-02-03T17:14:43.404ZMax gen: 109, Max flush: 189
121322024-02-03T17:14:43.404ZError: Difference in extent metadata found!
121332024-02-03T17:14:43.408ZA Difference in extent metadata is expected here
121342024-02-03T17:14:43.408ZOn loop 55
121352024-02-03T17:14:43.408Z
121362024-02-03T17:14:43.408ZVerifying data now
121372024-02-03T17:14:49.507ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
121382024-02-03T17:14:49.521Z58a59
121392024-02-03T17:14:49.521Z> 8,
121402024-02-03T17:14:49.521Z60,61d60
121412024-02-03T17:14:49.522Z< 6,
121422024-02-03T17:14:49.522Z< 6,
121432024-02-03T17:14:49.522Z63a63
121442024-02-03T17:14:49.522Z> 8,
121452024-02-03T17:14:49.522Z407,408c407
121462024-02-03T17:14:49.522Z< 6,
121472024-02-03T17:14:49.522Z< 8,
121482024-02-03T17:14:49.522Z---
121492024-02-03T17:14:49.522Z> 7,
121502024-02-03T17:14:49.522Z409a409
121512024-02-03T17:14:49.522Z> 10,
121522024-02-03T17:14:49.522Zdiff found after verify
121532024-02-03T17:14:49.526ZLoop: 55 Downstairs dump after verify (and repair):
121542024-02-03T17:14:49.537ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
121552024-02-03T17:14:49.537Z 0 000-019 106 106 106 183 183 183 F F F
121562024-02-03T17:14:49.537Z 1 020-039 107 107 107 184 184 184 F F F
121572024-02-03T17:14:49.538Z 2 040-059 110 110 110 190 190 190 F F F
121582024-02-03T17:14:49.538Z 3 060-079 110 110 110 190 190 190 F F F
121592024-02-03T17:14:49.541Z 4 080-099 108 108 108 187 187 187 F F F
121602024-02-03T17:14:49.542Z 5 100-119 108 108 108 187 187 187 F F F
121612024-02-03T17:14:49.542Z 6 120-139 110 110 110 190 190 190 F F F
121622024-02-03T17:14:49.542Z 7 140-159 110 110 110 190 190 190 F F F
121632024-02-03T17:14:49.542Z 8 160-179 107 107 107 184 184 184 F F F
121642024-02-03T17:14:49.542Z 9 180-199 110 110 110 190 190 190 F F F
121652024-02-03T17:14:49.542Z 10 200-219 107 107 107 184 184 184 F F F
121662024-02-03T17:14:49.542Z 11 220-239 101 101 101 176 176 176 F F F
121672024-02-03T17:14:49.542Z 12 240-259 109 109 109 189 189 189 F F F
121682024-02-03T17:14:49.543Z 13 260-279 109 109 109 189 189 189 F F F
121692024-02-03T17:14:49.543Z 14 280-299 105 105 105 180 180 180 F F F
121702024-02-03T17:14:49.543Z 15 300-319 110 110 110 190 190 190 F F F
121712024-02-03T17:14:49.543Z 16 320-339 110 110 110 190 190 190 F F F
121722024-02-03T17:14:49.543Z 17 340-359 109 109 109 189 189 189 F F F
121732024-02-03T17:14:49.543Z 18 360-379 110 110 110 190 190 190 F F F
121742024-02-03T17:14:49.544Z 19 380-399 110 110 110 190 190 190 F F F
121752024-02-03T17:14:49.544Z 20 400-419 110 110 110 190 190 190 F F F
121762024-02-03T17:14:49.544Z 21 420-439 108 108 108 188 188 188 F F F
121772024-02-03T17:14:49.544Z 22 440-459 108 108 108 185 185 185 F F F
121782024-02-03T17:14:49.544Z 23 460-479 107 107 107 184 184 184 F F F
121792024-02-03T17:14:49.545Z 24 480-499 100 100 100 173 173 173 F F F
121802024-02-03T17:14:49.545Z 25 500-519 108 108 108 188 188 188 F F F
121812024-02-03T17:14:49.545Z 26 520-539 108 108 108 187 187 187 F F F
121822024-02-03T17:14:49.545Z 27 540-559 108 108 108 187 187 187 F F F
121832024-02-03T17:14:49.546Z 28 560-579 110 110 110 190 190 190 F F F
121842024-02-03T17:14:49.546Z 29 580-599 99 99 99 172 172 172 F F F
121852024-02-03T17:14:49.546ZMax gen: 110, Max flush: 190
121862024-02-03T17:14:49.557Z
121872024-02-03T17:14:49.557ZBegin loop 56
121882024-02-03T17:14:49.557ZDownstairs to restart: 0
121892024-02-03T17:14:49.573Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.135720116Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583}
121902024-02-03T17:14:49.573Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.136247851Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583}
121912024-02-03T17:14:49.573Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.136279817Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583}
121922024-02-03T17:14:49.577Z{"msg":"Crucible 7cb90494-97da-4e9c-b65d-1a19fffb47d2 has session id: 370849e7-99c2-465c-a6d8-1674e0be855a","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.136357094Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583Crucible runtime is spawned
121932024-02-03T17:14:49.578Z,"session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
121942024-02-03T17:14:49.578Z{"msg":"Upstairs opts: Upstairs UUID: 7cb90494-97da-4e9c-b65d-1a19fffb47d2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.136472179Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
121952024-02-03T17:14:49.578Z{"msg":"Crucible stats registered with UUID: 7cb90494-97da-4e9c-b65d-1a19fffb47d2","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.136559316Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
121962024-02-03T17:14:49.578Z{"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.136630445Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583}
121972024-02-03T17:14:49.578Z{"msg":"7cb90494-97da-4e9c-b65d-1a19fffb47d2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.136863813Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
121982024-02-03T17:14:49.578Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.136923935Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
121992024-02-03T17:14:49.578Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.13700326Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122002024-02-03T17:14:49.578Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.137028507Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122012024-02-03T17:14:49.578Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.137058888Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122022024-02-03T17:14:49.578Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.137118346Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122032024-02-03T17:14:49.578Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.137141148Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122042024-02-03T17:14:49.578Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.13727569Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"":"io task","client":"2","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122052024-02-03T17:14:49.578Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.137395007Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"":"io task","client":"0","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122062024-02-03T17:14:49.578Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.137440255Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"":"io task","client":"1","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122072024-02-03T17:14:49.578Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.137518999Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"":"io task","client":"0","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122082024-02-03T17:14:49.578Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.13763313Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"":"io task","client":"1","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122092024-02-03T17:14:49.578Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.137823745Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"":"io task","client":"2","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122102024-02-03T17:14:49.578Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.138689983Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122112024-02-03T17:14:49.578Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.138758976Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122122024-02-03T17:14:49.578Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.139044857Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122132024-02-03T17:14:49.578Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.139165103Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122142024-02-03T17:14:49.578Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.139729626Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122152024-02-03T17:14:49.579Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.139806795Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122162024-02-03T17:14:49.579Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.13985484Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122172024-02-03T17:14:49.579Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.140009094Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122182024-02-03T17:14:49.584Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.140813639Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122192024-02-03T17:14:49.584Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.140890587Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122202024-02-03T17:14:49.584Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.14102003Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122212024-02-03T17:14:49.584Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.141093022Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122222024-02-03T17:14:49.584Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.141242364Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122232024-02-03T17:14:49.584Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.141336839Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122242024-02-03T17:14:49.584Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.141382033Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122252024-02-03T17:14:49.584Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.142086215Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122262024-02-03T17:14:49.584Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.142218051Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122272024-02-03T17:14:49.584Z{"msg":"[0]R flush_numbers[0..12]: [183, 184, 190, 190, 187, 187, 190, 190, 184, 190, 184, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.142278918Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122282024-02-03T17:14:49.584Z{"msg":"[0]R generation[0..12]: [106, 107, 110, 110, 108, 108, 110, 110, 107, 110, 107, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.142306023Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122292024-02-03T17:14:49.584Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.14235775Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122302024-02-03T17:14:49.585Z{"msg":"[1]R flush_numbers[0..12]: [183, 184, 190, 190, 187, 187, 190, 190, 184, 190, 184, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.142384109Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122312024-02-03T17:14:49.585Z{"msg":"[1]R generation[0..12]: [106, 107, 110, 110, 108, 108, 110, 110, 107, 110, 107, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.142407526Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122322024-02-03T17:14:49.585Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.142448589Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122332024-02-03T17:14:49.585Z{"msg":"[2]R flush_numbers[0..12]: [183, 184, 190, 190, 187, 187, 190, 190, 184, 190, 184, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.142491943Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122342024-02-03T17:14:49.585Z{"msg":"[2]R generation[0..12]: [106, 107, 110, 110, 108, 108, 110, 110, 107, 110, 107, 101]","v":0,"name":"crucible",Wait for a query_work_queue command to finish before sending IO
122352024-02-03T17:14:49.585Z"level":30,"time":"2024-02-03T17:14:49.142541544Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122362024-02-03T17:14:49.585Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.142634135Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122372024-02-03T17:14:49.585Z{"msg":"Max found gen is 111","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.142719502Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122382024-02-03T17:14:49.585Z{"msg":"Generation requested: 112 >= found:111","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.142764299Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122392024-02-03T17:14:49.585Z{"msg":"Next flush: 191","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.142834226Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122402024-02-03T17:14:49.585Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.142860106Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122412024-02-03T17:14:49.585Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.14291429Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122422024-02-03T17:14:49.586Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.142936959Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122432024-02-03T17:14:49.586Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.14298789Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122442024-02-03T17:14:49.586Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.143039648Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122452024-02-03T17:14:49.586Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.143063974Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122462024-02-03T17:14:49.586Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.143112167Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122472024-02-03T17:14:49.586Z{"msg":"7cb90494-97da-4e9c-b65d-1a19fffb47d2 is now active with session: 370849e7-99c2-465c-a6d8-1674e0be855a","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.143135828Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122482024-02-03T17:14:49.586Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.143155686Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583,"session_id":"370849e7-99c2-465c-a6d8-1674e0be855a"}
122492024-02-03T17:14:49.586Z{"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:49.143206385Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583}
122502024-02-03T17:14:49.586ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
122512024-02-03T17:14:49.586ZLoading write count information from file "/tmp/test_repair_verify.data"
122522024-02-03T17:14:49.586ZRun Repair workload
122532024-02-03T17:14:49.586Z01/30 Write block 549 len 36864 data: 11 12 11 11 13 12 9 8 8
122542024-02-03T17:14:49.590Z02/30 Flush
122552024-02-03T17:14:49.590Z03/30 Write block 105 len 20480 data: 7 9 10 5 5
122562024-02-03T17:14:49.590Z04/30 Read block 203 len 8192
122572024-02-03T17:14:49.591Z05/30 Read block 580 len 12288
122582024-02-03T17:14:49.594Z06/30 Read block 445 len 20480
122592024-02-03T17:14:49.594Z07/30 Write block 259 len 40960 data: 9 12 12 11 11 12 13 12 12 14
122602024-02-03T17:14:49.597Z08/30 Read block 41 len 16384
122612024-02-03T17:14:49.597Z09/30 Read block 221 len 28672
122622024-02-03T17:14:49.601Z10/30 Read block 48 len 32768
122632024-02-03T17:14:49.601Z11/30 Write block 238 len 8192 data: 3 3
122642024-02-03T17:14:49.604Z12/30 Read block 575 len 40960
122652024-02-03T17:14:49.605Z13/30 Write block 372 len 24576 data: 7 8 8 7 9 10
122662024-02-03T17:14:49.605Z14/30 Flush
122672024-02-03T17:14:49.609Z15/30 Write block 572 len 4096 data: 7
122682024-02-03T17:14:49.612Z16/30 Write block 239 len 28672 data: 4 3 5 5 5 6 6
122692024-02-03T17:14:49.612Z17/30 Read block 570 len 32768
122702024-02-03T17:14:49.616Z18/30 Read block 194 len 12288
122712024-02-03T17:14:49.616Z19/30 Write block 460 len 8192 data: 10 8
122722024-02-03T17:14:49.616Z20/30 Write block 519 len 28672 data: 7 8 6 7 8 7 9
122732024-02-03T17:14:49.619Z21/30 Write block 379 len 40960 data: 11 9 8 9 9 8 8 8 9 8
122742024-02-03T17:14:49.619Z22/30 Write block 45 len 32768 data: 7 7 8 7 7 7 8 9
122752024-02-03T17:14:49.623Z23/30 Flush
122762024-02-03T17:14:49.628Z24/30 Write block 570 len 16384 data: 10 9 8 9
122772024-02-03T17:14:49.631Z25/30 Write block 161 len 4096 data: 11
122782024-02-03T17:14:49.631Z26/30 Write block 62 len 12288 data: 8 6 4
122792024-02-03T17:14:49.631Z27/30 Read block 505 len 8192
122802024-02-03T17:14:49.631Z28/30 Write block 283 len 32768 data: 16 12 11 11 10 8 9 8
122812024-02-03T17:14:49.635Z29/30 Write block 381 len 20480 data: 9 10 10 9 9
122822024-02-03T17:14:49.635Z30/30 Read block 574 len 40960
122832024-02-03T17:14:49.639Z----------------------------------------------------------------
122842024-02-03T17:14:49.639Z Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:28
122852024-02-03T17:14:49.639ZStates: Active Active Active
122862024-02-03T17:14:49.639ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
122872024-02-03T17:14:49.639Z 3 Acked 1002 Write 5 Sent Done Done false
122882024-02-03T17:14:49.639Z 4 Acked 1003 Read 2 Sent Done Done false
122892024-02-03T17:14:49.639Z 5 Acked 1004 Read 3 Sent Done Done false
122902024-02-03T17:14:49.639Z 6 Acked 1005 Read 5 Sent Done Done false
122912024-02-03T17:14:49.639Z 7 Acked 1006 Write 10 Sent Done Done false
122922024-02-03T17:14:49.639Z 8 Acked 1007 Read 4 Sent Done Done false
122932024-02-03T17:14:49.640Z 9 Acked 1008 Read 7 Sent Done Done false
122942024-02-03T17:14:49.640Z 10 Acked 1009 Read 8 Sent Done Done false
122952024-02-03T17:14:49.640Z 11 Acked 1010 Write 2 Sent Done Done false
122962024-02-03T17:14:49.640Z 12 Acked 1011 Read 10 Sent Done Done false
122972024-02-03T17:14:49.640Z 13 Acked 1012 Write 6 Sent Done Done false
122982024-02-03T17:14:49.640Z 14 Acked 1013 Flush 0 Sent Done Done false
122992024-02-03T17:14:49.640Z 15 Acked 1014 Write 1 Sent Done Done false
123002024-02-03T17:14:49.640Z 16 Acked 1015 Write 7 Sent Done Done false
123012024-02-03T17:14:49.640Z 17 Acked 1016 Read 8 Sent Done Done false
123022024-02-03T17:14:49.640Z 18 Acked 1017 Read 3 Sent Done Done false
123032024-02-03T17:14:49.640Z 19 Acked 1018 Write 2 Sent Done Done false
123042024-02-03T17:14:49.640Z 20 Acked 1019 Write 7 Sent Done Done false
123052024-02-03T17:14:49.640Z 21 Acked 1020 Write 10 Sent Done Done false
123062024-02-03T17:14:49.641Z 22 Acked 1021 Write 8 Sent Done Done false
123072024-02-03T17:14:49.641Z 23 Acked 1022 Flush 0 Sent Done Done false
123082024-02-03T17:14:49.641Z 24 Acked 1023 Write 4 Sent Done Done false
123092024-02-03T17:14:49.641Z 25 Acked 1024 Write 1 Sent Done Done false
123102024-02-03T17:14:49.641Z 26 Acked 1025 Write 3 Sent Done Done false
123112024-02-03T17:14:49.641Z 27 Acked 1026 Read 2 Sent Done Done false
123122024-02-03T17:14:49.641Z 28 Acked 1027 Write 8 Sent Done Done false
123132024-02-03T17:14:49.641Z 29 Acked 1028 Write 5 Sent Done Done false
123142024-02-03T17:14:49.641Z 30 Acked 1029 Read 10 Sent Done Done false
123152024-02-03T17:14:49.641Z STATES DS:0 DS:1 DS:2 TOTAL
123162024-02-03T17:14:49.641Z New 0 0 0 0
123172024-02-03T17:14:49.641Z Sent 28 0 0 28
123182024-02-03T17:14:49.642Z Done 0 28 28 56
123192024-02-03T17:14:49.642Z Skipped 0 0 0 0
123202024-02-03T17:14:49.642Z Error 0 0 0 0
123212024-02-03T17:14:49.642ZLast Flush: 1001 1022 1022
123222024-02-03T17:14:49.642ZDownstairs last five completed: 1001 1000
123232024-02-03T17:14:49.642ZUpstairs last five completed: 30 29 28 27 26
123242024-02-03T17:14:49.642Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:14:49.19957455Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1583}
123252024-02-03T17:14:49.642ZWrote out file "/tmp/test_repair_verify.data"
123262024-02-03T17:14:49.646Z
123272024-02-03T17:14:49.652ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
123282024-02-03T17:14:49.679ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
123292024-02-03T17:14:49.679Z 0 000-019 106 106 106 183 183 183 F F F
123302024-02-03T17:14:49.679Z 1 020-039 107 107 107 184 184 184 F F F
123312024-02-03T17:14:49.679Z 2 040-059 110 112 112 190 193 193 F F F <---
123322024-02-03T17:14:49.683Z 3 060-079 110 110 110 190 190 190 F T T <---
123332024-02-03T17:14:49.683Z 4 080-099 108 108 108 187 187 187 F F F
123342024-02-03T17:14:49.683Z 5 100-119 108 112 112 187 192 192 F F F <---
123352024-02-03T17:14:49.683Z 6 120-139 110 110 110 190 190 190 F F F
123362024-02-03T17:14:49.683Z 7 140-159 110 110 110 190 190 190 F F F
123372024-02-03T17:14:49.684Z 8 160-179 107 107 107 184 184 184 F T T <---
123382024-02-03T17:14:49.684Z 9 180-199 110 110 110 190 190 190 F F F
123392024-02-03T17:14:49.684Z 10 200-219 107 107 107 184 184 184 F F F
123402024-02-03T17:14:49.684Z 11 220-239 101 112 112 176 193 193 F F F <---
123412024-02-03T17:14:49.684Z 12 240-259 109 112 112 189 193 193 F F F <---
123422024-02-03T17:14:49.684Z 13 260-279 109 112 112 189 192 192 F F F <---
123432024-02-03T17:14:49.684Z 14 280-299 105 105 105 180 180 180 F T T <---
123442024-02-03T17:14:49.684Z 15 300-319 110 110 110 190 190 190 F F F
123452024-02-03T17:14:49.684Z 16 320-339 110 110 110 190 190 190 F F F
123462024-02-03T17:14:49.684Z 17 340-359 109 109 109 189 189 189 F F F
123472024-02-03T17:14:49.684Z 18 360-379 110 112 112 190 193 193 F F F <---
123482024-02-03T17:14:49.684Z 19 380-399 110 112 112 190 193 193 F T T <---
123492024-02-03T17:14:49.684Z 20 400-419 110 110 110 190 190 190 F F F
123502024-02-03T17:14:49.685Z 21 420-439 108 108 108 188 188 188 F F F
123512024-02-03T17:14:49.685Z 22 440-459 108 108 108 185 185 185 F F F
123522024-02-03T17:14:49.685Z 23 460-479 107 112 112 184 193 193 F F F <---
123532024-02-03T17:14:49.685Z 24 480-499 100 100 100 173 173 173 F F F
123542024-02-03T17:14:49.685Z 25 500-519 108 112 112 188 193 193 F F F <---
123552024-02-03T17:14:49.685Z 26 520-539 108 112 112 187 193 193 F F F <---
123562024-02-03T17:14:49.685Z 27 540-559 112 112 112 191 191 191 F F F
123572024-02-03T17:14:49.685Z 28 560-579 110 112 112 190 193 193 F T T <---
123582024-02-03T17:14:49.685Z 29 580-599 99 99 99 172 172 172 F F F
123592024-02-03T17:14:49.685ZMax gen: 112, Max flush: 193
123602024-02-03T17:14:49.685ZError: Difference in extent metadata found!
123612024-02-03T17:14:49.685ZA Difference in extent metadata is expected here
123622024-02-03T17:14:49.685ZOn loop 56
123632024-02-03T17:14:49.685Z
123642024-02-03T17:14:49.692ZVerifying data now
123652024-02-03T17:14:54.887ZNo change after verify
123662024-02-03T17:14:54.887ZLoop: 56 Downstairs dump after verify (and repair):
123672024-02-03T17:14:54.915ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
123682024-02-03T17:14:54.915Z 0 000-019 106 106 106 183 183 183 F F F
123692024-02-03T17:14:54.915Z 1 020-039 107 107 107 184 184 184 F F F
123702024-02-03T17:14:54.915Z 2 040-059 112 112 112 193 193 193 F F F
123712024-02-03T17:14:54.915Z 3 060-079 113 113 113 194 194 194 F F F
123722024-02-03T17:14:54.915Z 4 080-099 108 108 108 187 187 187 F F F
123732024-02-03T17:14:54.915Z 5 100-119 112 112 112 192 192 192 F F F
123742024-02-03T17:14:54.915Z 6 120-139 110 110 110 190 190 190 F F F
123752024-02-03T17:14:54.915Z 7 140-159 110 110 110 190 190 190 F F F
123762024-02-03T17:14:54.915Z 8 160-179 113 113 113 194 194 194 F F F
123772024-02-03T17:14:54.915Z 9 180-199 110 110 110 190 190 190 F F F
123782024-02-03T17:14:54.916Z 10 200-219 107 107 107 184 184 184 F F F
123792024-02-03T17:14:54.916Z 11 220-239 112 112 112 193 193 193 F F F
123802024-02-03T17:14:54.916Z 12 240-259 112 112 112 193 193 193 F F F
123812024-02-03T17:14:54.916Z 13 260-279 112 112 112 192 192 192 F F F
123822024-02-03T17:14:54.916Z 14 280-299 113 113 113 194 194 194 F F F
123832024-02-03T17:14:54.916Z 15 300-319 110 110 110 190 190 190 F F F
123842024-02-03T17:14:54.922Z 16 320-339 110 110 110 190 190 190 F F F
123852024-02-03T17:14:54.922Z 17 340-359 109 109 109 189 189 189 F F F
123862024-02-03T17:14:54.922Z 18 360-379 112 112 112 193 193 193 F F F
123872024-02-03T17:14:54.922Z 19 380-399 113 113 113 194 194 194 F F F
123882024-02-03T17:14:54.922Z 20 400-419 110 110 110 190 190 190 F F F
123892024-02-03T17:14:54.922Z 21 420-439 108 108 108 188 188 188 F F F
123902024-02-03T17:14:54.922Z 22 440-459 108 108 108 185 185 185 F F F
123912024-02-03T17:14:54.922Z 23 460-479 112 112 112 193 193 193 F F F
123922024-02-03T17:14:54.922Z 24 480-499 100 100 100 173 173 173 F F F
123932024-02-03T17:14:54.922Z 25 500-519 112 112 112 193 193 193 F F F
123942024-02-03T17:14:54.922Z 26 520-539 112 112 112 193 193 193 F F F
123952024-02-03T17:14:54.922Z 27 540-559 112 112 112 191 191 191 F F F
123962024-02-03T17:14:54.922Z 28 560-579 113 113 113 194 194 194 F F F
123972024-02-03T17:14:54.922Z 29 580-599 99 99 99 172 172 172 F F F
123982024-02-03T17:14:54.922ZMax gen: 113, Max flush: 194
123992024-02-03T17:14:54.922Z
124002024-02-03T17:14:54.922ZBegin loop 57
124012024-02-03T17:14:54.922ZDownstairs to restart: 2
124022024-02-03T17:14:54.945Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.50846331Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591}
124032024-02-03T17:14:54.949Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.508981499Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591Crucible runtime is spawned
124042024-02-03T17:14:54.949Z}
124052024-02-03T17:14:54.949Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.509261468Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591}
124062024-02-03T17:14:54.950Z{"msg":"Crucible 110ac515-f95e-4b80-85a6-05e6711ea4ad has session id: 23151e91-b00e-4484-9fbd-63260959d5b3","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.509335995Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124072024-02-03T17:14:54.950Z{"msg":"Upstairs opts: Upstairs UUID: 110ac515-f95e-4b80-85a6-05e6711ea4ad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.50940134Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124082024-02-03T17:14:54.950Z{"msg":"Crucible stats registered with UUID: 110ac515-f95e-4b80-85a6-05e6711ea4ad","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.509464858Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124092024-02-03T17:14:54.950Z{"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.509506699Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591}
124102024-02-03T17:14:54.950Z{"msg":"110ac515-f95e-4b80-85a6-05e6711ea4ad active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.509541832Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124112024-02-03T17:14:54.950Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.509574427Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124122024-02-03T17:14:54.950Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.509615046Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124132024-02-03T17:14:54.950Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.509654335Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124142024-02-03T17:14:54.950Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.509723432Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124152024-02-03T17:14:54.950Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.509888064Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124162024-02-03T17:14:54.950Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.509968982Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124172024-02-03T17:14:54.950Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.510157794Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"":"io task","client":"2","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124182024-02-03T17:14:54.950Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.510227604Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"":"io task","client":"0","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124192024-02-03T17:14:54.950Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.510304983Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"":"io task","client":"1","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124202024-02-03T17:14:54.950Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.51035157Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"":"io task","client":"2","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124212024-02-03T17:14:54.950Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.51042627Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"":"io task","client":"0","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124222024-02-03T17:14:54.950Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.510472952Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"":"io task","client":"1","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124232024-02-03T17:14:54.950Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.511366776Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124242024-02-03T17:14:54.950Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.5114659Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124252024-02-03T17:14:54.950Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.511701469Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124262024-02-03T17:14:54.950Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.51180086Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124272024-02-03T17:14:54.958Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.512369989Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124282024-02-03T17:14:54.958Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.512670115Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124292024-02-03T17:14:54.958Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.512861365Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124302024-02-03T17:14:54.958Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.513079969Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124312024-02-03T17:14:54.958Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.513339507Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124322024-02-03T17:14:54.958Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.513462184Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124332024-02-03T17:14:54.958Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.513511122Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124342024-02-03T17:14:54.958Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.513628021Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124352024-02-03T17:14:54.958Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.513689911Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124362024-02-03T17:14:54.958Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.513715878Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124372024-02-03T17:14:54.958Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.514076933Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124382024-02-03T17:14:54.958Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.514858121Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124392024-02-03T17:14:54.958Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.514933936Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124402024-02-03T17:14:54.958Z{"msg":"[0]R flush_numbers[0..12]: [183, 184, 193, 194, 187, 192, 190, 190, 194, 190, 184, 193]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.514980001Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124412024-02-03T17:14:54.958Z{"msg":"[0]R generation[0..12]: [106, 107, 112, 113, 108, 112, 110, 110, 113, 110, 107, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.515040693Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124422024-02-03T17:14:54.958Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.515068829Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124432024-02-03T17:14:54.958Z{"msg":"[1]R flush_numbers[0..12]: [183, 184, 193, 194, 187, 192, 190, 190, 194, 190, 184, 193]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.515092909Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124442024-02-03T17:14:54.958Z{"msg":"[1]R generation[0..12]: [106, 107, 112, 113, 108, 112, 110, 110, 113, 110, 107, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.515141795Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124452024-02-03T17:14:54.958Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.515168276Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124462024-02-03T17:14:54.958Z{"msg":"[2]R flush_numbers[0..12]: [183, 184, 193, 194, 187, 192, 190, 190, 194, 190, 184, 193]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.51519223Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124472024-02-03T17:14:54.959Z{"msg":"[2]R generation[0..12]: [106, 107, 112, 113, 108, 112, 110, 110, 113, 110, 107, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.515215601Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124482024-02-03T17:14:54.959Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.515264786Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124492024-02-03T17:14:54.959Z{"msg":"Max found gen is 114","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
124502024-02-03T17:14:54.959Z,"time":"2024-02-03T17:14:54.515336445Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124512024-02-03T17:14:54.959Z{"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.515388026Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124522024-02-03T17:14:54.959Z{"msg":"Next flush: 195","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.515412064Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124532024-02-03T17:14:54.959Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.515460486Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124542024-02-03T17:14:54.959Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.515484992Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124552024-02-03T17:14:54.959Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.515505855Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124562024-02-03T17:14:54.959Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.515552478Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124572024-02-03T17:14:54.959Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.515576784Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124582024-02-03T17:14:54.959Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.5156251Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124592024-02-03T17:14:54.959Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.515648222Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124602024-02-03T17:14:54.959Z{"msg":"110ac515-f95e-4b80-85a6-05e6711ea4ad is now active with session: 23151e91-b00e-4484-9fbd-63260959d5b3","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.515669954Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124612024-02-03T17:14:54.959Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.515691679Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591,"session_id":"23151e91-b00e-4484-9fbd-63260959d5b3"}
124622024-02-03T17:14:54.959Z{"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:54.515738378Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591}
124632024-02-03T17:14:54.959ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
124642024-02-03T17:14:54.959ZLoading write count information from file "/tmp/test_repair_verify.data"
124652024-02-03T17:14:54.959ZRun Repair workload
124662024-02-03T17:14:54.959Z01/30 Write block 268 len 40960 data: 15 12 12 11 11 10 9 11 10 10
124672024-02-03T17:14:54.962Z02/30 Read block 421 len 28672
124682024-02-03T17:14:54.963Z03/30 Write block 526 len 40960 data: 8 10 9 9 11 8 8 8 6 8
124692024-02-03T17:14:54.963Z04/30 Read block 480 len 4096
124702024-02-03T17:14:54.963Z05/30 Write block 256 len 28672 data: 8 8 10 10 13 13 12
124712024-02-03T17:14:54.966Z06/30 Write block 377 len 16384 data: 11 9 12 10
124722024-02-03T17:14:54.966Z07/30 Write block 140 len 12288 data: 7 11 11
124732024-02-03T17:14:54.966Z08/30 Write block 297 len 8192 data: 10 10
124742024-02-03T17:14:54.973Z09/30 Read block 247 len 4096
124752024-02-03T17:14:54.973Z10/30 Flush
124762024-02-03T17:14:54.976Z11/30 Write block 576 len 40960 data: 7 6 6 7 6 6 5 5 4 5
124772024-02-03T17:14:54.976Z12/30 Read block 190 len 12288
124782024-02-03T17:14:54.976Z13/30 Read block 192 len 20480
124792024-02-03T17:14:54.976Z14/30 Read block 550 len 24576
124802024-02-03T17:14:54.979Z15/30 Read block 46 len 16384
124812024-02-03T17:14:54.979Z16/30 Write block 19 len 12288 data: 16 12 15
124822024-02-03T17:14:54.979Z17/30 Read block 247 len 16384
124832024-02-03T17:14:54.982Z18/30 Flush
124842024-02-03T17:14:54.985Z19/30 Write block 437 len 12288 data: 8 8 8
124852024-02-03T17:14:54.988Z20/30 Write block 587 len 20480 data: 7 10 8 9 10
124862024-02-03T17:14:54.988Z21/30 Write block 257 len 36864 data: 9 11 11 14 14 13 12 13 14
124872024-02-03T17:14:54.991Z22/30 Flush
124882024-02-03T17:14:54.995Z23/30 Flush
124892024-02-03T17:14:54.995Z24/30 Flush
124902024-02-03T17:14:54.995Z25/30 Write block 168 len 12288 data: 14 14 13
124912024-02-03T17:14:54.995Z26/30 Read block 222 len 8192
124922024-02-03T17:14:54.998Z27/30 Read block 323 len 16384
124932024-02-03T17:14:54.998Z28/30 Read block 309 len 4096
124942024-02-03T17:14:55.001Z29/30 Read block 169 len 4096
124952024-02-03T17:14:55.001Z30/30 Write block 442 len 24576 data: 13 10 9 9 12 13
124962024-02-03T17:14:55.001Z----------------------------------------------------------------
124972024-02-03T17:14:55.002Z Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30
124982024-02-03T17:14:55.002ZStates: Active Active Active
124992024-02-03T17:14:55.002ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
125002024-02-03T17:14:55.002Z 1 Acked 1000 Write 10 Done Done Done false
125012024-02-03T17:14:55.002Z 2 Acked 1001 Read 7 Done Done Done false
125022024-02-03T17:14:55.002Z 3 Acked 1002 Write 10 Done Done Done false
125032024-02-03T17:14:55.002Z 4 Acked 1003 Read 1 Done Done Done false
125042024-02-03T17:14:55.002Z 5 Acked 1004 Write 7 Done Done Sent false
125052024-02-03T17:14:55.002Z 6 Acked 1005 Write 4 Done Done Sent false
125062024-02-03T17:14:55.002Z 7 Acked 1006 Write 3 Done Done Sent false
125072024-02-03T17:14:55.002Z 8 Acked 1007 Write 2 Done Done Sent false
125082024-02-03T17:14:55.002Z 9 Acked 1008 Read 1 Done Done Sent false
125092024-02-03T17:14:55.002Z 10 Acked 1009 Flush 0 Done Done Sent false
125102024-02-03T17:14:55.002Z 11 Acked 1010 Write 10 Done Done Sent false
125112024-02-03T17:14:55.002Z 12 Acked 1011 Read 3 Done Done Sent false
125122024-02-03T17:14:55.002Z 13 Acked 1012 Read 5 Done Done Sent false
125132024-02-03T17:14:55.002Z 14 Acked 1013 Read 6 Done Done Sent false
125142024-02-03T17:14:55.002Z 15 Acked 1014 Read 4 Done Done Sent false
125152024-02-03T17:14:55.002Z 16 Acked 1015 Write 3 Done Done Sent false
125162024-02-03T17:14:55.003Z 17 Acked 1016 Read 4 Done Done Sent false
125172024-02-03T17:14:55.003Z 18 Acked 1017 Flush 0 Done Done Sent false
125182024-02-03T17:14:55.003Z 19 Acked 1018 Write 3 Done Done Sent false
125192024-02-03T17:14:55.003Z 20 Acked 1019 Write 5 Done Done Sent false
125202024-02-03T17:14:55.003Z 21 Acked 1020 Write 9 Done Done Sent false
125212024-02-03T17:14:55.003Z 22 Acked 1021 Flush 0 Done Done Sent false
125222024-02-03T17:14:55.003Z 23 Acked 1022 Flush 0 Done Done Sent false
125232024-02-03T17:14:55.003Z 24 Acked 1023 Flush 0 Done Done Sent false
125242024-02-03T17:14:55.003Z 25 Acked 1024 Write 3 Done Done Sent false
125252024-02-03T17:14:55.003Z 26 Acked 1025 Read 2 Done Done Sent false
125262024-02-03T17:14:55.003Z 27 Acked 1026 Read 4 Done Done Sent false
125272024-02-03T17:14:55.003Z 28 Acked 1027 Read 1 Done Done Sent false
125282024-02-03T17:14:55.003Z 29 Acked 1028 Read 1 Done Done Sent false
125292024-02-03T17:14:55.003Z 30 Acked 1029 Write 6 Sent Sent Sent false
125302024-02-03T17:14:55.003Z STATES DS:0 DS:1 DS:2 TOTAL
125312024-02-03T17:14:55.003Z New 0 0 0 0
125322024-02-03T17:14:55.003Z Sent 1 1 26 28
125332024-02-03T17:14:55.003Z Done 29 29 4 62
125342024-02-03T17:14:55.003Z Skipped 0 0 0 0
125352024-02-03T17:14:55.003Z Error 0 0 0 0
125362024-02-03T17:14:55.003ZLast Flush: 1023 1023 0
125372024-02-03T17:14:55.003ZDownstairs last five completed:
125382024-02-03T17:14:55.003ZUpstairs last five completed: 30 29 28 27 26
125392024-02-03T17:14:55.004Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:14:54.564863245Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1591}
125402024-02-03T17:14:55.007ZWrote out file "/tmp/test_repair_verify.data"
125412024-02-03T17:14:55.010Z
125422024-02-03T17:14:55.020ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
125432024-02-03T17:14:55.039ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
125442024-02-03T17:14:55.039Z 0 000-019 114 114 106 196 196 183 F F F <---
125452024-02-03T17:14:55.039Z 1 020-039 114 114 107 196 196 184 F F F <---
125462024-02-03T17:14:55.039Z 2 040-059 112 112 112 193 193 193 F F F
125472024-02-03T17:14:55.039Z 3 060-079 113 113 113 194 194 194 F F F
125482024-02-03T17:14:55.039Z 4 080-099 108 108 108 187 187 187 F F F
125492024-02-03T17:14:55.039Z 5 100-119 112 112 112 192 192 192 F F F
125502024-02-03T17:14:55.039Z 6 120-139 110 110 110 190 190 190 F F F
125512024-02-03T17:14:55.039Z 7 140-159 114 114 110 195 195 190 F F F <---
125522024-02-03T17:14:55.039Z 8 160-179 113 113 113 194 194 194 T T F <---
125532024-02-03T17:14:55.039Z 9 180-199 110 110 110 190 190 190 F F F
125542024-02-03T17:14:55.042Z 10 200-219 107 107 107 184 184 184 F F F
125552024-02-03T17:14:55.042Z 11 220-239 112 112 112 193 193 193 F F F
125562024-02-03T17:14:55.042Z 12 240-259 114 114 112 197 197 193 F F F <---
125572024-02-03T17:14:55.042Z 13 260-279 114 114 112 197 197 192 F F T <---
125582024-02-03T17:14:55.042Z 14 280-299 114 114 113 195 195 194 F F F <---
125592024-02-03T17:14:55.042Z 15 300-319 110 110 110 190 190 190 F F F
125602024-02-03T17:14:55.042Z 16 320-339 110 110 110 190 190 190 F F F
125612024-02-03T17:14:55.042Z 17 340-359 109 109 109 189 189 189 F F F
125622024-02-03T17:14:55.043Z 18 360-379 114 114 112 195 195 193 F F F <---
125632024-02-03T17:14:55.043Z 19 380-399 114 114 113 195 195 194 F F F <---
125642024-02-03T17:14:55.043Z 20 400-419 110 110 110 190 190 190 F F F
125652024-02-03T17:14:55.043Z 21 420-439 114 114 108 197 197 188 F F F <---
125662024-02-03T17:14:55.043Z 22 440-459 108 108 108 185 185 185 T T F <---
125672024-02-03T17:14:55.043Z 23 460-479 112 112 112 193 193 193 F F F
125682024-02-03T17:14:55.043Z 24 480-499 100 100 100 173 173 173 F F F
125692024-02-03T17:14:55.043Z 25 500-519 112 112 112 193 193 193 F F F
125702024-02-03T17:14:55.043Z 26 520-539 114 114 112 195 195 193 F F T <---
125712024-02-03T17:14:55.043Z 27 540-559 112 112 112 191 191 191 F F F
125722024-02-03T17:14:55.043Z 28 560-579 114 114 113 196 196 194 F F F <---
125732024-02-03T17:14:55.043Z 29 580-599 114 114 99 197 197 172 F F F <---
125742024-02-03T17:14:55.043ZMax gen: 114, Max flush: 197
125752024-02-03T17:14:55.043ZError: Difference in extent metadata found!
125762024-02-03T17:14:55.046ZA Difference in extent metadata is expected here
125772024-02-03T17:14:55.046ZOn loop 57
125782024-02-03T17:14:55.046Z
125792024-02-03T17:14:55.049ZVerifying data now
125802024-02-03T17:15:00.239ZNo change after verify
125812024-02-03T17:15:00.239ZLoop: 57 Downstairs dump after verify (and repair):
125822024-02-03T17:15:00.263ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
125832024-02-03T17:15:00.263Z 0 000-019 114 114 114 196 196 196 F F F
125842024-02-03T17:15:00.263Z 1 020-039 114 114 114 196 196 196 F F F
125852024-02-03T17:15:00.263Z 2 040-059 112 112 112 193 193 193 F F F
125862024-02-03T17:15:00.263Z 3 060-079 113 113 113 194 194 194 F F F
125872024-02-03T17:15:00.263Z 4 080-099 108 108 108 187 187 187 F F F
125882024-02-03T17:15:00.263Z 5 100-119 112 112 112 192 192 192 F F F
125892024-02-03T17:15:00.263Z 6 120-139 110 110 110 190 190 190 F F F
125902024-02-03T17:15:00.263Z 7 140-159 114 114 114 195 195 195 F F F
125912024-02-03T17:15:00.263Z 8 160-179 115 115 115 198 198 198 F F F
125922024-02-03T17:15:00.263Z 9 180-199 110 110 110 190 190 190 F F F
125932024-02-03T17:15:00.263Z 10 200-219 107 107 107 184 184 184 F F F
125942024-02-03T17:15:00.263Z 11 220-239 112 112 112 193 193 193 F F F
125952024-02-03T17:15:00.267Z 12 240-259 114 114 114 197 197 197 F F F
125962024-02-03T17:15:00.267Z 13 260-279 114 114 114 197 197 197 F F F
125972024-02-03T17:15:00.267Z 14 280-299 114 114 114 195 195 195 F F F
125982024-02-03T17:15:00.267Z 15 300-319 110 110 110 190 190 190 F F F
125992024-02-03T17:15:00.267Z 16 320-339 110 110 110 190 190 190 F F F
126002024-02-03T17:15:00.267Z 17 340-359 109 109 109 189 189 189 F F F
126012024-02-03T17:15:00.267Z 18 360-379 114 114 114 195 195 195 F F F
126022024-02-03T17:15:00.268Z 19 380-399 114 114 114 195 195 195 F F F
126032024-02-03T17:15:00.268Z 20 400-419 110 110 110 190 190 190 F F F
126042024-02-03T17:15:00.268Z 21 420-439 114 114 114 197 197 197 F F F
126052024-02-03T17:15:00.268Z 22 440-459 115 115 115 198 198 198 F F F
126062024-02-03T17:15:00.268Z 23 460-479 112 112 112 193 193 193 F F F
126072024-02-03T17:15:00.268Z 24 480-499 100 100 100 173 173 173 F F F
126082024-02-03T17:15:00.268Z 25 500-519 112 112 112 193 193 193 F F F
126092024-02-03T17:15:00.268Z 26 520-539 114 114 114 195 195 195 F F F
126102024-02-03T17:15:00.268Z 27 540-559 112 112 112 191 191 191 F F F
126112024-02-03T17:15:00.268Z 28 560-579 114 114 114 196 196 196 F F F
126122024-02-03T17:15:00.268Z 29 580-599 114 114 114 197 197 197 F F F
126132024-02-03T17:15:00.268ZMax gen: 115, Max flush: 198
126142024-02-03T17:15:00.268Z
126152024-02-03T17:15:00.268ZBegin loop 58
126162024-02-03T17:15:00.268ZDownstairs to restart: 1
126172024-02-03T17:15:00.298Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.862708585Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599}
126182024-02-03T17:15:00.299Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.863214641Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599}
126192024-02-03T17:15:00.303ZCrucible runtime is spawned
126202024-02-03T17:15:00.304Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.863429306Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599}
126212024-02-03T17:15:00.304Z{"msg":"Crucible 185beca8-de1f-4444-9b88-b8ff70f6fc8a has session id: 5e149d9e-afbd-4136-a8e8-6f9e67b6631e","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.863507265Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126222024-02-03T17:15:00.304Z{"msg":"Upstairs opts: Upstairs UUID: 185beca8-de1f-4444-9b88-b8ff70f6fc8a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.863605706Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126232024-02-03T17:15:00.304Z{"msg":"Crucible stats registered with UUID: 185beca8-de1f-4444-9b88-b8ff70f6fc8a","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.863683218Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126242024-02-03T17:15:00.304Z{"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.86371964Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599}
126252024-02-03T17:15:00.304Z{"msg":"185beca8-de1f-4444-9b88-b8ff70f6fc8a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.863750578Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126262024-02-03T17:15:00.304Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.863782404Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126272024-02-03T17:15:00.304Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.863868864Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126282024-02-03T17:15:00.304Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.863896854Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126292024-02-03T17:15:00.304Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.863936038Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126302024-02-03T17:15:00.304Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.863975557Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126312024-02-03T17:15:00.304Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.864079823Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126322024-02-03T17:15:00.304Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.864231858Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"":"io task","client":"2","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126332024-02-03T17:15:00.304Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.864316941Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"":"io task","client":"0","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126342024-02-03T17:15:00.304Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.864386057Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"":"io task","client":"1","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126352024-02-03T17:15:00.304Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.864431178Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"":"io task","client":"2","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126362024-02-03T17:15:00.304Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.864502542Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"":"io task","client":"0","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126372024-02-03T17:15:00.304Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.864547499Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"":"io task","client":"1","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126382024-02-03T17:15:00.305Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.865418173Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126392024-02-03T17:15:00.305Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.865562871Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126402024-02-03T17:15:00.305Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.86558936Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126412024-02-03T17:15:00.305Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.865612098Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126422024-02-03T17:15:00.305Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.866455468Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126432024-02-03T17:15:00.305Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.866551102Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126442024-02-03T17:15:00.305Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.866761946Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126452024-02-03T17:15:00.314Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.866945379Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126462024-02-03T17:15:00.314Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.867576099Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126472024-02-03T17:15:00.314Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.867898519Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126482024-02-03T17:15:00.314Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.867963408Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126492024-02-03T17:15:00.314Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.867989616Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126502024-02-03T17:15:00.314Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.868080704Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126512024-02-03T17:15:00.314Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.868106761Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126522024-02-03T17:15:00.314Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.86812872Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126532024-02-03T17:15:00.314Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.868425434Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126542024-02-03T17:15:00.314Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.868514085Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126552024-02-03T17:15:00.315Z{"msg":"[0]R flush_numbers[0..12]: [196, 196, 193, 194, 187, 192, 190, 195, 198, 190, 184, 193]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.868562896Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126562024-02-03T17:15:00.315Z{"msg":"[0]R generation[0..12]: [114, 114, 112, 113, 108, 112, 110, 114, 115, 110, 107, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.868642923Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126572024-02-03T17:15:00.315Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.868724344Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126582024-02-03T17:15:00.315Z{"msg":"[1]R flush_numbers[0..12]: [196, 196, 193, 194, 187, 192, 190, 195, 198, 190, 184, 193]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.868808099Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126592024-02-03T17:15:00.315Z{"msg":"[1]R generation[0..12]: [114, 114, 112, 113, 108, 112, 110, 114, 115, 110, 107, 112]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
126602024-02-03T17:15:00.315Z,"time":"2024-02-03T17:14:59.868923512Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126612024-02-03T17:15:00.315Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.869077717Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126622024-02-03T17:15:00.315Z{"msg":"[2]R flush_numbers[0..12]: [196, 196, 193, 194, 187, 192, 190, 195, 198, 190, 184, 193]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.869163519Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126632024-02-03T17:15:00.315Z{"msg":"[2]R generation[0..12]: [114, 114, 112, 113, 108, 112, 110, 114, 115, 110, 107, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.869247919Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126642024-02-03T17:15:00.315Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.869330425Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126652024-02-03T17:15:00.315Z{"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.869409525Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126662024-02-03T17:15:00.315Z{"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.869490059Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126672024-02-03T17:15:00.315Z{"msg":"Next flush: 199","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.869566606Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126682024-02-03T17:15:00.315Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.869644663Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126692024-02-03T17:15:00.315Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.869725389Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126702024-02-03T17:15:00.315Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.869767437Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126712024-02-03T17:15:00.315Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.869847039Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126722024-02-03T17:15:00.315Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
126732024-02-03T17:15:00.316Z,"time":"2024-02-03T17:14:59.86992958Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126742024-02-03T17:15:00.316Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.870043715Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126752024-02-03T17:15:00.316Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.870072063Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126762024-02-03T17:15:00.316Z{"msg":"185beca8-de1f-4444-9b88-b8ff70f6fc8a is now active with session: 5e149d9e-afbd-4136-a8e8-6f9e67b6631e","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.870119201Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126772024-02-03T17:15:00.316Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.870141673Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599,"session_id":"5e149d9e-afbd-4136-a8e8-6f9e67b6631e"}
126782024-02-03T17:15:00.316Z{"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:14:59.870186819Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599}
126792024-02-03T17:15:00.316ZLoading write count information from file "/tmp/test_repair_verify.data"
126802024-02-03T17:15:00.316ZRun Repair workload
126812024-02-03T17:15:00.316Z01/30 Write block 367 len 32768 data: 8 9 9 9 6 8 9 9
126822024-02-03T17:15:00.324Z02/30 Read block 50 len 24576
126832024-02-03T17:15:00.324Z03/30 Read block 209 len 16384
126842024-02-03T17:15:00.324Z04/30 Read block 529 len 4096
126852024-02-03T17:15:00.324Z05/30 Read block 5 len 4096
126862024-02-03T17:15:00.324Z06/30 Read block 481 len 24576
126872024-02-03T17:15:00.324Z07/30 Write block 345 len 12288 data: 9 9 9
126882024-02-03T17:15:00.324Z08/30 Flush
126892024-02-03T17:15:00.324Z09/30 Flush
126902024-02-03T17:15:00.324Z10/30 Write block 530 len 8192 data: 12 9
126912024-02-03T17:15:00.324Z11/30 Read block 355 len 32768
126922024-02-03T17:15:00.328Z12/30 Read block 122 len 4096
126932024-02-03T17:15:00.328Z13/30 Write block 435 len 24576 data: 11 9 9 9 9 9
126942024-02-03T17:15:00.328Z14/30 Read block 114 len 4096
126952024-02-03T17:15:00.328Z15/30 Read block 81 len 28672
126962024-02-03T17:15:00.332Z16/30 Read block 219 len 4096
126972024-02-03T17:15:00.332Z17/30 Read block 483 len 36864
126982024-02-03T17:15:00.335Z18/30 Write block 334 len 20480 data: 8 7 8 9 11
126992024-02-03T17:15:00.335Z19/30 Read block 98 len 32768
127002024-02-03T17:15:00.339Z20/30 Write block 134 len 32768 data: 7 6 6 7 7 7 8 12
127012024-02-03T17:15:00.339Z21/30 Read block 469 len 40960
127022024-02-03T17:15:00.340Z22/30 Write block 390 len 12288 data: 11 12 16
127032024-02-03T17:15:00.343Z23/30 Write block 255 len 40960 data: 6 9 10 12 12 15 15 14 13 14
127042024-02-03T17:15:00.343Z24/30 Read block 83 len 32768
127052024-02-03T17:15:00.348Z25/30 Read block 365 len 32768
127062024-02-03T17:15:00.348Z26/30 Write block 510 len 28672 data: 11 14 14 11 11 12 12
127072024-02-03T17:15:00.355Z27/30 Read block 458 len 8192
127082024-02-03T17:15:00.355Z28/30 Write block 285 len 12288 data: 12 12 11
127092024-02-03T17:15:00.355Z29/30 Write block 204 len 40960 data: 6 10 9 9 10 9 11 9 9 8
127102024-02-03T17:15:00.355Z30/30 Read block 462 len 8192
127112024-02-03T17:15:00.359Z----------------------------------------------------------------
127122024-02-03T17:15:00.359Z Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:30
127132024-02-03T17:15:00.359ZStates: Active Active Active
127142024-02-03T17:15:00.359ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
127152024-02-03T17:15:00.359Z 1 Acked 1000 Write 8 Done Done Done false
127162024-02-03T17:15:00.359Z 2 Acked 1001 Read 6 Done Done Done false
127172024-02-03T17:15:00.359Z 3 Acked 1002 Read 4 Done Done Done false
127182024-02-03T17:15:00.359Z 4 Acked 1003 Read 1 Done Done Done false
127192024-02-03T17:15:00.359Z 5 Acked 1004 Read 1 Done Done Done false
127202024-02-03T17:15:00.359Z 6 Acked 1005 Read 6 Done Sent Done false
127212024-02-03T17:15:00.359Z 7 Acked 1006 Write 3 Done Sent Done false
127222024-02-03T17:15:00.359Z 8 Acked 1007 Flush 0 Done Sent Done false
127232024-02-03T17:15:00.359Z 9 Acked 1008 Flush 0 Done Sent Done false
127242024-02-03T17:15:00.359Z 10 Acked 1009 Write 2 Done Sent Done false
127252024-02-03T17:15:00.359Z 11 Acked 1010 Read 8 Done Sent Done false
127262024-02-03T17:15:00.360Z 12 Acked 1011 Read 1 Done Sent Done false
127272024-02-03T17:15:00.360Z 13 Acked 1012 Write 6 Done Sent Done false
127282024-02-03T17:15:00.360Z 14 Acked 1013 Read 1 Done Sent Done false
127292024-02-03T17:15:00.360Z 15 Acked 1014 Read 7 Done Sent Done false
127302024-02-03T17:15:00.360Z 16 Acked 1015 Read 1 Done Sent Done false
127312024-02-03T17:15:00.360Z 17 Acked 1016 Read 9 Done Sent Done false
127322024-02-03T17:15:00.360Z 18 Acked 1017 Write 5 Done Sent Done false
127332024-02-03T17:15:00.360Z 19 Acked 1018 Read 8 Done Sent Done false
127342024-02-03T17:15:00.360Z 20 Acked 1019 Write 8 Done Sent Done false
127352024-02-03T17:15:00.360Z 21 Acked 1020 Read 10 Done Sent Done false
127362024-02-03T17:15:00.360Z 22 Acked 1021 Write 3 Done Sent Done false
127372024-02-03T17:15:00.360Z 23 Acked 1022 Write 10 Done Sent Done false
127382024-02-03T17:15:00.360Z 24 Acked 1023 Read 8 Done Sent Done false
127392024-02-03T17:15:00.361Z 25 Acked 1024 Read 8 Done Sent Done false
127402024-02-03T17:15:00.361Z 26 Acked 1025 Write 7 Done Sent Done false
127412024-02-03T17:15:00.361Z 27 Acked 1026 Read 2 Done Sent Done false
127422024-02-03T17:15:00.361Z 28 Acked 1027 Write 3 Done Sent Done false
127432024-02-03T17:15:00.361Z 29 Acked 1028 Write 10 Done Sent Done false
127442024-02-03T17:15:00.361Z 30 Acked 1029 Read 2 Done Sent Sent false
127452024-02-03T17:15:00.361Z STATES DS:0 DS:1 DS:2 TOTAL
127462024-02-03T17:15:00.361Z New 0 0 0 0
127472024-02-03T17:15:00.361Z Sent 0 25 1 26
127482024-02-03T17:15:00.361Z Done 30 5 29 64
127492024-02-03T17:15:00.361Z Skipped 0 0 0 0
127502024-02-03T17:15:00.361Z Error 0 0 0 0
127512024-02-03T17:15:00.361ZLast Flush: 1008 0 1008
127522024-02-03T17:15:00.361ZDownstairs last five completed:
127532024-02-03T17:15:00.362ZUpstairs last five completed: 30 29 28 27 26
127542024-02-03T17:15:00.362Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:14:59.916831045Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1599}
127552024-02-03T17:15:00.362ZWrote out file "/tmp/test_repair_verify.data"
127562024-02-03T17:15:00.368Z
127572024-02-03T17:15:00.368ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
127582024-02-03T17:15:00.407ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
127592024-02-03T17:15:00.408Z 0 000-019 114 114 114 196 196 196 F F F
127602024-02-03T17:15:00.408Z 1 020-039 114 114 114 196 196 196 F F F
127612024-02-03T17:15:00.408Z 2 040-059 112 112 112 193 193 193 F F F
127622024-02-03T17:15:00.408Z 3 060-079 113 113 113 194 194 194 F F F
127632024-02-03T17:15:00.408Z 4 080-099 108 108 108 187 187 187 F F F
127642024-02-03T17:15:00.408Z 5 100-119 112 112 112 192 192 192 F F F
127652024-02-03T17:15:00.408Z 6 120-139 110 110 110 190 190 190 T F T <---
127662024-02-03T17:15:00.412Z 7 140-159 114 114 114 195 195 195 T F T <---
127672024-02-03T17:15:00.412Z 8 160-179 115 115 115 198 198 198 F F F
127682024-02-03T17:15:00.412Z 9 180-199 110 110 110 190 190 190 F F F
127692024-02-03T17:15:00.412Z 10 200-219 107 107 107 184 184 184 T F T <---
127702024-02-03T17:15:00.412Z 11 220-239 112 112 112 193 193 193 F F F
127712024-02-03T17:15:00.412Z 12 240-259 114 114 114 197 197 197 T F T <---
127722024-02-03T17:15:00.413Z 13 260-279 114 114 114 197 197 197 T F T <---
127732024-02-03T17:15:00.413Z 14 280-299 114 114 114 195 195 195 T F T <---
127742024-02-03T17:15:00.413Z 15 300-319 110 110 110 190 190 190 F F F
127752024-02-03T17:15:00.413Z 16 320-339 110 110 110 190 190 190 T F T <---
127762024-02-03T17:15:00.413Z 17 340-359 116 109 116 199 189 199 F F F <---
127772024-02-03T17:15:00.413Z 18 360-379 116 114 116 199 195 199 F T F <---
127782024-02-03T17:15:00.413Z 19 380-399 114 114 114 195 195 195 T F T <---
127792024-02-03T17:15:00.413Z 20 400-419 110 110 110 190 190 190 F F F
127802024-02-03T17:15:00.413Z 21 420-439 114 114 114 197 197 197 T F T <---
127812024-02-03T17:15:00.413Z 22 440-459 115 115 115 198 198 198 T F T <---
127822024-02-03T17:15:00.413Z 23 460-479 112 112 112 193 193 193 F F F
127832024-02-03T17:15:00.413Z 24 480-499 100 100 100 173 173 173 F F F
127842024-02-03T17:15:00.413Z 25 500-519 112 112 112 193 193 193 T F T <---
127852024-02-03T17:15:00.414Z 26 520-539 114 114 114 195 195 195 T F T <---
127862024-02-03T17:15:00.414Z 27 540-559 112 112 112 191 191 191 F F F
127872024-02-03T17:15:00.414Z 28 560-579 114 114 114 196 196 196 F F F
127882024-02-03T17:15:00.414Z 29 580-599 114 114 114 197 197 197 F F F
127892024-02-03T17:15:00.414ZMax gen: 116, Max flush: 199
127902024-02-03T17:15:00.414ZError: Difference in extent metadata found!
127912024-02-03T17:15:00.414ZA Difference in extent metadata is expected here
127922024-02-03T17:15:00.414ZOn loop 58
127932024-02-03T17:15:00.414Z
127942024-02-03T17:15:00.414ZVerifying data now
127952024-02-03T17:15:05.653ZNo change after verify
127962024-02-03T17:15:05.653ZLoop: 58 Downstairs dump after verify (and repair):
127972024-02-03T17:15:05.676ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
127982024-02-03T17:15:05.676Z 0 000-019 114 114 114 196 196 196 F F F
127992024-02-03T17:15:05.676Z 1 020-039 114 114 114 196 196 196 F F F
128002024-02-03T17:15:05.676Z 2 040-059 112 112 112 193 193 193 F F F
128012024-02-03T17:15:05.677Z 3 060-079 113 113 113 194 194 194 F F F
128022024-02-03T17:15:05.677Z 4 080-099 108 108 108 187 187 187 F F F
128032024-02-03T17:15:05.680Z 5 100-119 112 112 112 192 192 192 F F F
128042024-02-03T17:15:05.680Z 6 120-139 117 117 117 200 200 200 F F F
128052024-02-03T17:15:05.680Z 7 140-159 117 117 117 200 200 200 F F F
128062024-02-03T17:15:05.680Z 8 160-179 115 115 115 198 198 198 F F F
128072024-02-03T17:15:05.680Z 9 180-199 110 110 110 190 190 190 F F F
128082024-02-03T17:15:05.681Z 10 200-219 117 117 117 200 200 200 F F F
128092024-02-03T17:15:05.681Z 11 220-239 112 112 112 193 193 193 F F F
128102024-02-03T17:15:05.681Z 12 240-259 117 117 117 200 200 200 F F F
128112024-02-03T17:15:05.681Z 13 260-279 117 117 117 200 200 200 F F F
128122024-02-03T17:15:05.681Z 14 280-299 117 117 117 200 200 200 F F F
128132024-02-03T17:15:05.681Z 15 300-319 110 110 110 190 190 190 F F F
128142024-02-03T17:15:05.681Z 16 320-339 117 117 117 200 200 200 F F F
128152024-02-03T17:15:05.681Z 17 340-359 116 116 116 199 199 199 F F F
128162024-02-03T17:15:05.681Z 18 360-379 116 116 116 199 199 199 F F F
128172024-02-03T17:15:05.681Z 19 380-399 117 117 117 200 200 200 F F F
128182024-02-03T17:15:05.681Z 20 400-419 110 110 110 190 190 190 F F F
128192024-02-03T17:15:05.681Z 21 420-439 117 117 117 200 200 200 F F F
128202024-02-03T17:15:05.681Z 22 440-459 117 117 117 200 200 200 F F F
128212024-02-03T17:15:05.681Z 23 460-479 112 112 112 193 193 193 F F F
128222024-02-03T17:15:05.681Z 24 480-499 100 100 100 173 173 173 F F F
128232024-02-03T17:15:05.681Z 25 500-519 117 117 117 200 200 200 F F F
128242024-02-03T17:15:05.681Z 26 520-539 117 117 117 200 200 200 F F F
128252024-02-03T17:15:05.681Z 27 540-559 112 112 112 191 191 191 F F F
128262024-02-03T17:15:05.681Z 28 560-579 114 114 114 196 196 196 F F F
128272024-02-03T17:15:05.681Z 29 580-599 114 114 114 197 197 197 F F F
128282024-02-03T17:15:05.681ZMax gen: 117, Max flush: 200
128292024-02-03T17:15:05.685Z
128302024-02-03T17:15:05.685ZBegin loop 59
128312024-02-03T17:15:05.685ZDownstairs to restart: 1
128322024-02-03T17:15:05.718Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.281158273Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607}
128332024-02-03T17:15:05.718Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.281810243Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607}
128342024-02-03T17:15:05.718Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.2818397Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607}
128352024-02-03T17:15:05.722Z{"msg":"Crucible 0f75e979-8ef0-4266-a5e8-108269dc7847 has session id: 4d0ef9ae-b636-4ace-ada5-63d73416d693","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.281861384Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128362024-02-03T17:15:05.722Z{"msg":"Upstairs opts: Upstairs UUID: 0f75e979-8ef0-4266-a5e8-108269dc7847, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.281897955Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128372024-02-03T17:15:05.722Z{"msg":"Crucible stats registered with UUID: 0f75e979-8ef0-4266-a5e8-108269dc7847","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.281936649Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128382024-02-03T17:15:05.722ZCrucible runtime is spawned
128392024-02-03T17:15:05.722Z{"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.282420354Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607}
128402024-02-03T17:15:05.722Z{"msg":"0f75e979-8ef0-4266-a5e8-108269dc7847 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.282707746Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128412024-02-03T17:15:05.723Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.282828442Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128422024-02-03T17:15:05.723Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.283000823Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128432024-02-03T17:15:05.723Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.28308702Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128442024-02-03T17:15:05.723Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.283129126Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128452024-02-03T17:15:05.723Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.283169402Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128462024-02-03T17:15:05.723Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.283201744Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128472024-02-03T17:15:05.723Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.283224959Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"":"io task","client":"2","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128482024-02-03T17:15:05.723Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.283251678Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"":"io task","client":"0","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128492024-02-03T17:15:05.723Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.283292631Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"":"io task","client":"1","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128502024-02-03T17:15:05.723Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.283372194Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"":"io task","client":"1","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128512024-02-03T17:15:05.723Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.283620019Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"":"io task","client":"2","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128522024-02-03T17:15:05.723Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.283795148Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"":"io task","client":"0","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128532024-02-03T17:15:05.723Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.285029547Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128542024-02-03T17:15:05.723Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.285072346Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128552024-02-03T17:15:05.723Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.285112775Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128562024-02-03T17:15:05.723Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.28519138Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128572024-02-03T17:15:05.723Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.285825069Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128582024-02-03T17:15:05.723Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.285952814Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128592024-02-03T17:15:05.723Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.286108176Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128602024-02-03T17:15:05.729Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.286523937Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128612024-02-03T17:15:05.729Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.286939237Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128622024-02-03T17:15:05.729Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.287004854Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128632024-02-03T17:15:05.729Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.287031233Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128642024-02-03T17:15:05.729Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.287073642Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128652024-02-03T17:15:05.729Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.287446955Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128662024-02-03T17:15:05.729Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.287523963Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128672024-02-03T17:15:05.729Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.287552985Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128682024-02-03T17:15:05.729Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.287919108Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128692024-02-03T17:15:05.729Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.287979478Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128702024-02-03T17:15:05.729Z{"msg":"[0]R flush_numbers[0..12]: [196, 196, 193, 194, 187, 192, 200, 200, 198, 190, 200, 193]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.288006429Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128712024-02-03T17:15:05.729Z{"msg":"[0]R generation[0..12]: [114, 114, 112, 113, 108, 112, 117, 117, 115, 110, 117, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.288030889Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128722024-02-03T17:15:05.729Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.28817133Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128732024-02-03T17:15:05.729Z{"msg":"[1]R flush_numbers[0..12]: [196, 196, 193, 194, 187, 192, 200, 200, 198, 190, 200, 193]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.2882311Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128742024-02-03T17:15:05.729Z{"msg":"[1]R generation[0..12]: [114, 114, 112, 113, 108, 112, 117, 117, 115, 110, 117, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.288258246Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128752024-02-03T17:15:05.729Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.28830768Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128762024-02-03T17:15:05.729Z{"msg":"[2]R flush_numbers[0..12]: [196, 196, 193, 194, 187, 192, 200, 200, 198, 190, 200, 193]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.288333737Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128772024-02-03T17:15:05.729Z{"msg":"[2]R generation[0..12]: [114, 114, 112, 113, 108, 112, 117, 117, 115, 110, 117, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.288357364Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693Wait for a query_work_queue command to finish before sending IO
128782024-02-03T17:15:05.729Z"}
128792024-02-03T17:15:05.729Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.288467976Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128802024-02-03T17:15:05.730Z{"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.28849565Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128812024-02-03T17:15:05.730Z{"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.28851892Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128822024-02-03T17:15:05.730Z{"msg":"Next flush: 201","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.288540277Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128832024-02-03T17:15:05.730Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.288561861Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128842024-02-03T17:15:05.730Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.288583796Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128852024-02-03T17:15:05.730Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.28860401Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128862024-02-03T17:15:05.730Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.288654592Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128872024-02-03T17:15:05.730Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.288679525Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128882024-02-03T17:15:05.730Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.288701663Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128892024-02-03T17:15:05.730Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.2887236Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128902024-02-03T17:15:05.730Z{"msg":"0f75e979-8ef0-4266-a5e8-108269dc7847 is now active with session: 4d0ef9ae-b636-4ace-ada5-63d73416d693","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.288745615Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128912024-02-03T17:15:05.730Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.288765246Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607,"session_id":"4d0ef9ae-b636-4ace-ada5-63d73416d693"}
128922024-02-03T17:15:05.730Z{"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:05.28881263Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607}
128932024-02-03T17:15:05.730ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
128942024-02-03T17:15:05.730ZLoading write count information from file "/tmp/test_repair_verify.data"
128952024-02-03T17:15:05.730ZRun Repair workload
128962024-02-03T17:15:05.730Z01/30 Write block 31 len 28672 data: 11 11 9 10 8 10 10
128972024-02-03T17:15:05.735Z02/30 Write block 221 len 16384 data: 5 4 4 3
128982024-02-03T17:15:05.735Z03/30 Write block 303 len 20480 data: 9 10 9 10 8
128992024-02-03T17:15:05.735Z04/30 Write block 361 len 28672 data: 10 7 7 7 7 7 9
129002024-02-03T17:15:05.735Z05/30 Write block 418 len 16384 data: 10 8 7 9
129012024-02-03T17:15:05.735Z06/30 Read block 358 len 20480
129022024-02-03T17:15:05.738Z07/30 Write block 457 len 32768 data: 6 7 9 11 9 7 8 7
129032024-02-03T17:15:05.738Z08/30 Read block 247 len 4096
129042024-02-03T17:15:05.738Z09/30 Read block 107 len 24576
129052024-02-03T17:15:05.742Z10/30 Read block 264 len 40960
129062024-02-03T17:15:05.745Z11/30 Write block 135 len 36864 data: 7 7 8 8 8 9 13 12 10
129072024-02-03T17:15:05.745Z12/30 Read block 560 len 16384
129082024-02-03T17:15:05.748Z13/30 Write block 502 len 20480 data: 8 10 11 11 12
129092024-02-03T17:15:05.748Z14/30 Read block 8 len 16384
129102024-02-03T17:15:05.754Z15/30 Write block 41 len 40960 data: 5 7 5 6 8 8 9 8 8 8
129112024-02-03T17:15:05.754Z16/30 Read block 44 len 8192
129122024-02-03T17:15:05.754Z17/30 Read block 324 len 32768
129132024-02-03T17:15:05.758Z18/30 Flush
129142024-02-03T17:15:05.761Z19/30 Write block 17 len 20480 data: 11 14 17 13 16
129152024-02-03T17:15:05.762Z20/30 Write block 59 len 36864 data: 8 9 7 9 7 5 5 6 6
129162024-02-03T17:15:05.765Z21/30 Flush
129172024-02-03T17:15:05.768Z22/30 Write block 327 len 20480 data: 12 11 12 10 10
129182024-02-03T17:15:05.768Z23/30 Read block 225 len 28672
129192024-02-03T17:15:05.771Z24/30 Write block 430 len 12288 data: 13 10 11
129202024-02-03T17:15:05.771Z25/30 Write block 148 len 12288 data: 8 8 6
129212024-02-03T17:15:05.774Z26/30 Read block 536 len 28672
129222024-02-03T17:15:05.774Z27/30 Write block 234 len 36864 data: 8 8 6 5 4 5 4 6 6
129232024-02-03T17:15:05.778Z28/30 Read block 505 len 40960
129242024-02-03T17:15:05.778Z29/30 Write block 314 len 4096 data: 11
129252024-02-03T17:15:05.787Z30/30 Read block 256 len 32768
129262024-02-03T17:15:05.791Z----------------------------------------------------------------
129272024-02-03T17:15:05.791Z Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:30
129282024-02-03T17:15:05.791ZStates: Active Active Active
129292024-02-03T17:15:05.791ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
129302024-02-03T17:15:05.791Z 1 Acked 1000 Write 7 Done Done Done false
129312024-02-03T17:15:05.791Z 2 Acked 1001 Write 4 Done Done Done false
129322024-02-03T17:15:05.791Z 3 Acked 1002 Write 5 Done Sent Done false
129332024-02-03T17:15:05.791Z 4 Acked 1003 Write 7 Done Sent Done false
129342024-02-03T17:15:05.791Z 5 Acked 1004 Write 4 Done Sent Done false
129352024-02-03T17:15:05.791Z 6 Acked 1005 Read 5 Done Sent Done false
129362024-02-03T17:15:05.791Z 7 Acked 1006 Write 8 Done Sent Done false
129372024-02-03T17:15:05.791Z 8 Acked 1007 Read 1 Done Sent Done false
129382024-02-03T17:15:05.791Z 9 Acked 1008 Read 6 Done Sent Done false
129392024-02-03T17:15:05.791Z 10 Acked 1009 Read 10 Done Sent Done false
129402024-02-03T17:15:05.791Z 11 Acked 1010 Write 9 Done Sent Done false
129412024-02-03T17:15:05.791Z 12 Acked 1011 Read 4 Done Sent Done false
129422024-02-03T17:15:05.791Z 13 Acked 1012 Write 5 Done Sent Done false
129432024-02-03T17:15:05.791Z 14 Acked 1013 Read 4 Done Sent Done false
129442024-02-03T17:15:05.791Z 15 Acked 1014 Write 10 Done Sent Done false
129452024-02-03T17:15:05.792Z 16 Acked 1015 Read 2 Done Sent Done false
129462024-02-03T17:15:05.792Z 17 Acked 1016 Read 8 Done Sent Done false
129472024-02-03T17:15:05.792Z 18 Acked 1017 Flush 0 Done Sent Done false
129482024-02-03T17:15:05.792Z 19 Acked 1018 Write 5 Done Sent Done false
129492024-02-03T17:15:05.792Z 20 Acked 1019 Write 9 Done Sent Done false
129502024-02-03T17:15:05.792Z 21 Acked 1020 Flush 0 Done Sent Done false
129512024-02-03T17:15:05.792Z 22 Acked 1021 Write 5 Done Sent Done false
129522024-02-03T17:15:05.792Z 23 Acked 1022 Read 7 Done Sent Done false
129532024-02-03T17:15:05.792Z 24 Acked 1023 Write 3 Done Sent Done false
129542024-02-03T17:15:05.792Z 25 Acked 1024 Write 3 Done Sent Done false
129552024-02-03T17:15:05.792Z 26 Acked 1025 Read 7 Done Sent Done false
129562024-02-03T17:15:05.792Z 27 Acked 1026 Write 9 Done Sent Done false
129572024-02-03T17:15:05.792Z 28 Acked 1027 Read 10 Done Sent Done false
129582024-02-03T17:15:05.792Z 29 Acked 1028 Write 1 Done Sent Done false
129592024-02-03T17:15:05.792Z 30 Acked 1029 Read 8 Done Sent Sent false
129602024-02-03T17:15:05.792Z STATES DS:0 DS:1 DS:2 TOTAL
129612024-02-03T17:15:05.792Z New 0 0 0 0
129622024-02-03T17:15:05.792Z Sent 0 28 1 29
129632024-02-03T17:15:05.792Z Done 30 2 29 61
129642024-02-03T17:15:05.792Z Skipped 0 0 0 0
129652024-02-03T17:15:05.792Z Error 0 0 0 0
129662024-02-03T17:15:05.792ZLast Flush: 1020 0 1020
129672024-02-03T17:15:05.792ZDownstairs last five completed:
129682024-02-03T17:15:05.792ZUpstairs last five completed: 30 29 28 27 26
129692024-02-03T17:15:05.792Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:15:05.345967979Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1607}
129702024-02-03T17:15:05.793ZWrote out file "/tmp/test_repair_verify.data"
129712024-02-03T17:15:05.793Z
129722024-02-03T17:15:05.796ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
129732024-02-03T17:15:05.824ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
129742024-02-03T17:15:05.824Z 0 000-019 118 114 118 202 196 202 F F F <---
129752024-02-03T17:15:05.824Z 1 020-039 118 114 118 202 196 202 F T F <---
129762024-02-03T17:15:05.824Z 2 040-059 118 112 118 202 193 202 F F F <---
129772024-02-03T17:15:05.824Z 3 060-079 118 113 118 202 194 202 F F F <---
129782024-02-03T17:15:05.828Z 4 080-099 108 108 108 187 187 187 F F F
129792024-02-03T17:15:05.828Z 5 100-119 112 112 112 192 192 192 F F F
129802024-02-03T17:15:05.828Z 6 120-139 118 117 118 201 200 201 F F F <---
129812024-02-03T17:15:05.828Z 7 140-159 118 117 118 201 200 201 T F T <---
129822024-02-03T17:15:05.828Z 8 160-179 115 115 115 198 198 198 F F F
129832024-02-03T17:15:05.828Z 9 180-199 110 110 110 190 190 190 F F F
129842024-02-03T17:15:05.828Z 10 200-219 117 117 117 200 200 200 F F F
129852024-02-03T17:15:05.828Z 11 220-239 118 112 118 201 193 201 T T T <---
129862024-02-03T17:15:05.828Z 12 240-259 117 117 117 200 200 200 T F T <---
129872024-02-03T17:15:05.828Z 13 260-279 117 117 117 200 200 200 F F F
129882024-02-03T17:15:05.828Z 14 280-299 117 117 117 200 200 200 F F F
129892024-02-03T17:15:05.828Z 15 300-319 118 110 118 201 190 201 T F T <---
129902024-02-03T17:15:05.828Z 16 320-339 117 117 117 200 200 200 T F T <---
129912024-02-03T17:15:05.828Z 17 340-359 116 116 116 199 199 199 F F F
129922024-02-03T17:15:05.828Z 18 360-379 118 116 118 201 199 201 F F F <---
129932024-02-03T17:15:05.828Z 19 380-399 117 117 117 200 200 200 F F F
129942024-02-03T17:15:05.828Z 20 400-419 118 110 118 201 190 201 F F F <---
129952024-02-03T17:15:05.828Z 21 420-439 118 117 118 201 200 201 T F T <---
129962024-02-03T17:15:05.828Z 22 440-459 118 117 118 201 200 201 F F F <---
129972024-02-03T17:15:05.828Z 23 460-479 118 112 118 201 193 201 F F F <---
129982024-02-03T17:15:05.828Z 24 480-499 100 100 100 173 173 173 F F F
129992024-02-03T17:15:05.829Z 25 500-519 118 117 118 201 200 201 F F F <---
130002024-02-03T17:15:05.829Z 26 520-539 117 117 117 200 200 200 F F F
130012024-02-03T17:15:05.829Z 27 540-559 112 112 112 191 191 191 F F F
130022024-02-03T17:15:05.829Z 28 560-579 114 114 114 196 196 196 F F F
130032024-02-03T17:15:05.829Z 29 580-599 114 114 114 197 197 197 F F F
130042024-02-03T17:15:05.829ZMax gen: 118, Max flush: 202
130052024-02-03T17:15:05.829ZError: Difference in extent metadata found!
130062024-02-03T17:15:05.832ZA Difference in extent metadata is expected here
130072024-02-03T17:15:05.832ZOn loop 59
130082024-02-03T17:15:05.832Z
130092024-02-03T17:15:05.837ZVerifying data now
130102024-02-03T17:15:12.199ZNo change after verify
130112024-02-03T17:15:12.199ZLoop: 59 Downstairs dump after verify (and repair):
130122024-02-03T17:15:12.222ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
130132024-02-03T17:15:12.222Z 0 000-019 118 118 118 202 202 202 F F F
130142024-02-03T17:15:12.222Z 1 020-039 118 118 118 202 202 202 F F F
130152024-02-03T17:15:12.222Z 2 040-059 118 118 118 202 202 202 F F F
130162024-02-03T17:15:12.222Z 3 060-079 118 118 118 202 202 202 F F F
130172024-02-03T17:15:12.223Z 4 080-099 108 108 108 187 187 187 F F F
130182024-02-03T17:15:12.223Z 5 100-119 112 112 112 192 192 192 F F F
130192024-02-03T17:15:12.223Z 6 120-139 118 118 118 201 201 201 F F F
130202024-02-03T17:15:12.223Z 7 140-159 119 119 119 203 203 203 F F F
130212024-02-03T17:15:12.223Z 8 160-179 115 115 115 198 198 198 F F F
130222024-02-03T17:15:12.223Z 9 180-199 110 110 110 190 190 190 F F F
130232024-02-03T17:15:12.243Z 10 200-219 117 117 117 200 200 200 F F F
130242024-02-03T17:15:12.243Z 11 220-239 119 119 119 203 203 203 F F F
130252024-02-03T17:15:12.243Z 12 240-259 119 119 119 203 203 203 F F F
130262024-02-03T17:15:12.243Z 13 260-279 117 117 117 200 200 200 F F F
130272024-02-03T17:15:12.243Z 14 280-299 117 117 117 200 200 200 F F F
130282024-02-03T17:15:12.243Z 15 300-319 119 119 119 203 203 203 F F F
130292024-02-03T17:15:12.243Z 16 320-339 119 119 119 203 203 203 F F F
130302024-02-03T17:15:12.243Z 17 340-359 116 116 116 199 199 199 F F F
130312024-02-03T17:15:12.243Z 18 360-379 118 118 118 201 201 201 F F F
130322024-02-03T17:15:12.243Z 19 380-399 117 117 117 200 200 200 F F F
130332024-02-03T17:15:12.244Z 20 400-419 118 118 118 201 201 201 F F F
130342024-02-03T17:15:12.244Z 21 420-439 119 119 119 203 203 203 F F F
130352024-02-03T17:15:12.244Z 22 440-459 118 118 118 201 201 201 F F F
130362024-02-03T17:15:12.244Z 23 460-479 118 118 118 201 201 201 F F F
130372024-02-03T17:15:12.244Z 24 480-499 100 100 100 173 173 173 F F F
130382024-02-03T17:15:12.244Z 25 500-519 118 118 118 201 201 201 F F F
130392024-02-03T17:15:12.244Z 26 520-539 117 117 117 200 200 200 F F F
130402024-02-03T17:15:12.244Z 27 540-559 112 112 112 191 191 191 F F F
130412024-02-03T17:15:12.244Z 28 560-579 114 114 114 196 196 196 F F F
130422024-02-03T17:15:12.244Z 29 580-599 114 114 114 197 197 197 F F F
130432024-02-03T17:15:12.244ZMax gen: 119, Max flush: 203
130442024-02-03T17:15:12.253Z
130452024-02-03T17:15:12.253ZBegin loop 60
130462024-02-03T17:15:12.253ZDownstairs to restart: 2
130472024-02-03T17:15:12.261Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.82463232Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615}
130482024-02-03T17:15:12.261Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.825202677Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615}
130492024-02-03T17:15:12.261Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.825236168Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615}
130502024-02-03T17:15:12.261Z{"msg":"Crucible 9fd3aae4-2858-4a4e-a3dd-511f7e00af61 has session id: 51cfdb58-62f7-40fb-9d89-4c6773413611","v":0,"name":"crucible","level":30Crucible runtime is spawned
130512024-02-03T17:15:12.262Z,"time":"2024-02-03T17:15:11.825309385Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130522024-02-03T17:15:12.268Z{"msg":"Upstairs opts: Upstairs UUID: 9fd3aae4-2858-4a4e-a3dd-511f7e00af61, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.825378965Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130532024-02-03T17:15:12.268Z{"msg":"Crucible stats registered with UUID: 9fd3aae4-2858-4a4e-a3dd-511f7e00af61","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.825458066Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130542024-02-03T17:15:12.268Z{"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.82551481Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615}
130552024-02-03T17:15:12.269Z{"msg":"9fd3aae4-2858-4a4e-a3dd-511f7e00af61 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.825812884Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130562024-02-03T17:15:12.269Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.825951236Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130572024-02-03T17:15:12.269Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.826022491Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130582024-02-03T17:15:12.269Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.826057039Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130592024-02-03T17:15:12.269Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.82613798Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130602024-02-03T17:15:12.269Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.826168856Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130612024-02-03T17:15:12.269Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.826257218Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130622024-02-03T17:15:12.269Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.826302263Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"":"io task","client":"2","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130632024-02-03T17:15:12.269Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.826395337Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"":"io task","client":"0","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130642024-02-03T17:15:12.269Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.826444114Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"":"io task","client":"1","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130652024-02-03T17:15:12.269Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.826646934Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"":"io task","client":"1","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130662024-02-03T17:15:12.269Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.826702706Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"":"io task","client":"0","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130672024-02-03T17:15:12.269Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.826778954Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"":"io task","client":"2","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130682024-02-03T17:15:12.269Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.82789539Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130692024-02-03T17:15:12.269Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.827965189Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130702024-02-03T17:15:12.269Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.828006155Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130712024-02-03T17:15:12.270Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.828159345Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130722024-02-03T17:15:12.270Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.828385514Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130732024-02-03T17:15:12.270Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.828446209Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130742024-02-03T17:15:12.270Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.829185725Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130752024-02-03T17:15:12.270Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.829289524Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130762024-02-03T17:15:12.270Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.829527502Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130772024-02-03T17:15:12.270Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.830056335Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130782024-02-03T17:15:12.270Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.830126063Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130792024-02-03T17:15:12.270Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.830162974Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130802024-02-03T17:15:12.270Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.830245821Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130812024-02-03T17:15:12.270Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.830295107Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130822024-02-03T17:15:12.270Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.830411036Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130832024-02-03T17:15:12.270Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.830436729Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130842024-02-03T17:15:12.270Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.830485398Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130852024-02-03T17:15:12.270Z{"msg":"[0]R flush_numbers[0..12]: [202, 202, 202, 202, 187, 192, 201, 203, 198, 190, 200, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.830511715Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130862024-02-03T17:15:12.270Z{"msg":"[0]R generation[0..12]: [118, 118, 118, 118, 108, 112, 118, 119, 115, 110, 117, 119]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.830560061Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130872024-02-03T17:15:12.271Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.830586531Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130882024-02-03T17:15:12.271Z{"msg":"[1]R flush_numbers[0..12]: [202, 202, 202, 202, 187, 192, 201, 203, 198, 190, 200, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.830634699Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130892024-02-03T17:15:12.271Z{"msg":"[1]R generation[0..12]: [118, 118, 118, 118, 108, 112, 118, 119, 115, 110, 117, 119]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.830660561Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130902024-02-03T17:15:12.271Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
130912024-02-03T17:15:12.271Z,"time":"2024-02-03T17:15:11.830726861Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130922024-02-03T17:15:12.271Z{"msg":"[2]R flush_numbers[0..12]: [202, 202, 202, 202, 187, 192, 201, 203, 198, 190, 200, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.830810562Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130932024-02-03T17:15:12.271Z{"msg":"[2]R generation[0..12]: [118, 118, 118, 118, 108, 112, 118, 119, 115, 110, 117, 119]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.830847013Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130942024-02-03T17:15:12.271Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.830899069Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130952024-02-03T17:15:12.271Z{"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.830923733Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130962024-02-03T17:15:12.271Z{"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.830954402Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130972024-02-03T17:15:12.271Z{"msg":"Next flush: 204","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.831028211Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130982024-02-03T17:15:12.271Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.831057508Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
130992024-02-03T17:15:12.271Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.831079816Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
131002024-02-03T17:15:12.271Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.831128696Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
131012024-02-03T17:15:12.271Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.831186533Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
131022024-02-03T17:15:12.272Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.831214245Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
131032024-02-03T17:15:12.272Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.831271244Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
131042024-02-03T17:15:12.272Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.83131162Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
131052024-02-03T17:15:12.272Z{"msg":"9fd3aae4-2858-4a4e-a3dd-511f7e00af61 is now active with session: 51cfdb58-62f7-40fb-9d89-4c6773413611","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.831385952Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
131062024-02-03T17:15:12.272Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.831411629Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615,"session_id":"51cfdb58-62f7-40fb-9d89-4c6773413611"}
131072024-02-03T17:15:12.272Z{"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:11.831434244Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615}
131082024-02-03T17:15:12.272ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
131092024-02-03T17:15:12.276ZLoading write count information from file "/tmp/test_repair_verify.data"
131102024-02-03T17:15:12.276ZRun Repair workload
131112024-02-03T17:15:12.276Z01/30 Write block 460 len 40960 data: 12 10 8 9 8 8 10 9 8 8
131122024-02-03T17:15:12.276Z02/30 Write block 9 len 12288 data: 14 13 8
131132024-02-03T17:15:12.276Z03/30 Write block 323 len 20480 data: 13 12 10 8 13
131142024-02-03T17:15:12.276Z04/30 Read block 134 len 24576
131152024-02-03T17:15:12.276Z05/30 Read block 478 len 40960
131162024-02-03T17:15:12.280Z06/30 Write block 594 len 8192 data: 9 7
131172024-02-03T17:15:12.280Z07/30 Write block 218 len 16384 data: 5 5 4 6
131182024-02-03T17:15:12.280Z08/30 Write block 157 len 32768 data: 10 10 9 12 12 14 16 16
131192024-02-03T17:15:12.280Z09/30 Flush
131202024-02-03T17:15:12.285Z10/30 Write block 48 len 40960 data: 9 9 9 9 10 7 7 7 8 7
131212024-02-03T17:15:12.300Z11/30 Read block 312 len 24576
131222024-02-03T17:15:12.303Z12/30 Write block 328 len 4096 data: 12
131232024-02-03T17:15:12.304Z13/30 Write block 138 len 12288 data: 9 9 10
131242024-02-03T17:15:12.304Z14/30 Read block 273 len 20480
131252024-02-03T17:15:12.304Z15/30 Write block 334 len 36864 data: 9 8 9 10 12 9 11 9 9
131262024-02-03T17:15:12.304Z16/30 Write block 170 len 24576 data: 14 12 9 10 10 8
131272024-02-03T17:15:12.304Z17/30 Write block 197 len 4096 data: 7
131282024-02-03T17:15:12.304Z18/30 Read block 85 len 12288
131292024-02-03T17:15:12.304Z19/30 Read block 252 len 24576
131302024-02-03T17:15:12.304Z20/30 Read block 145 len 40960
131312024-02-03T17:15:12.304Z21/30 Write block 242 len 8192 data: 7 6
131322024-02-03T17:15:12.304Z22/30 Write block 550 len 40960 data: 13 12 12 14 13 10 9 9 7 8
131332024-02-03T17:15:12.304Z23/30 Read block 133 len 16384
131342024-02-03T17:15:12.308Z24/30 Read block 432 len 28672
131352024-02-03T17:15:12.308Z25/30 Read block 253 len 40960
131362024-02-03T17:15:12.312Z26/30 Read block 117 len 40960
131372024-02-03T17:15:12.312Z27/30 Read block 409 len 4096
131382024-02-03T17:15:12.315Z28/30 Read block 410 len 40960
131392024-02-03T17:15:12.315Z29/30 Write block 78 len 16384 data: 8 9 7 10
131402024-02-03T17:15:12.316Z30/30 Read block 182 len 40960
131412024-02-03T17:15:12.320Z----------------------------------------------------------------
131422024-02-03T17:15:12.320Z Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:21
131432024-02-03T17:15:12.320ZStates: Active Active Active
131442024-02-03T17:15:12.320ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
131452024-02-03T17:15:12.320Z 10 Acked 1009 Write 10 Done Done Sent false
131462024-02-03T17:15:12.320Z 11 Acked 1010 Read 6 Done Done Sent false
131472024-02-03T17:15:12.320Z 12 Acked 1011 Write 1 Done Done Sent false
131482024-02-03T17:15:12.321Z 13 Acked 1012 Write 3 Done Done Sent false
131492024-02-03T17:15:12.321Z 14 Acked 1013 Read 5 Done Done Sent false
131502024-02-03T17:15:12.321Z 15 Acked 1014 Write 9 Done Done Sent false
131512024-02-03T17:15:12.321Z 16 Acked 1015 Write 6 Done Done Sent false
131522024-02-03T17:15:12.321Z 17 Acked 1016 Write 1 Done Done Sent false
131532024-02-03T17:15:12.321Z 18 Acked 1017 Read 3 Done Done Sent false
131542024-02-03T17:15:12.321Z 19 Acked 1018 Read 6 Done Done Sent false
131552024-02-03T17:15:12.321Z 20 Acked 1019 Read 10 Done Done Sent false
131562024-02-03T17:15:12.321Z 21 Acked 1020 Write 2 Done Done Sent false
131572024-02-03T17:15:12.321Z 22 Acked 1021 Write 10 Done Done Sent false
131582024-02-03T17:15:12.321Z 23 Acked 1022 Read 4 Done Done Sent false
131592024-02-03T17:15:12.321Z 24 Acked 1023 Read 7 Done Done Sent false
131602024-02-03T17:15:12.321Z 25 Acked 1024 Read 10 Done Done Sent false
131612024-02-03T17:15:12.321Z 26 Acked 1025 Read 10 Done Done Sent false
131622024-02-03T17:15:12.321Z 27 Acked 1026 Read 1 Done Done Sent false
131632024-02-03T17:15:12.322Z 28 Acked 1027 Read 10 Done Done Sent false
131642024-02-03T17:15:12.322Z 29 Acked 1028 Write 4 Done Done Sent false
131652024-02-03T17:15:12.322Z 30 Acked 1029 Read 10 Done Done Sent false
131662024-02-03T17:15:12.322Z STATES DS:0 DS:1 DS:2 TOTAL
131672024-02-03T17:15:12.322Z New 0 0 0 0
131682024-02-03T17:15:12.322Z Sent 0 0 21 21
131692024-02-03T17:15:12.322Z Done 21 21 0 42
131702024-02-03T17:15:12.322Z Skipped 0 0 0 0
131712024-02-03T17:15:12.322Z Error 0 0 0 0
131722024-02-03T17:15:12.322ZLast Flush: 1008 1008 1008
131732024-02-03T17:15:12.322ZDownstairs last five completed: 1008 1007 1006 1005 1004
131742024-02-03T17:15:12.322ZUpstairs last five completed: 30 29 28 27 26
131752024-02-03T17:15:12.322Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:15:11.883298165Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1615}
131762024-02-03T17:15:12.322ZWrote out file "/tmp/test_repair_verify.data"
131772024-02-03T17:15:12.328Z
131782024-02-03T17:15:12.332ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
131792024-02-03T17:15:12.362ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
131802024-02-03T17:15:12.362Z 0 000-019 120 120 120 204 204 204 F F F
131812024-02-03T17:15:12.362Z 1 020-039 118 118 118 202 202 202 F F F
131822024-02-03T17:15:12.362Z 2 040-059 118 118 118 202 202 202 T T F <---
131832024-02-03T17:15:12.366Z 3 060-079 118 118 118 202 202 202 T T F <---
131842024-02-03T17:15:12.366Z 4 080-099 108 108 108 187 187 187 T T F <---
131852024-02-03T17:15:12.366Z 5 100-119 112 112 112 192 192 192 F F F
131862024-02-03T17:15:12.367Z 6 120-139 118 118 118 201 201 201 T T F <---
131872024-02-03T17:15:12.367Z 7 140-159 120 120 120 204 204 204 T T F <---
131882024-02-03T17:15:12.367Z 8 160-179 120 120 120 204 204 204 T T F <---
131892024-02-03T17:15:12.367Z 9 180-199 110 110 110 190 190 190 T T F <---
131902024-02-03T17:15:12.367Z 10 200-219 120 120 120 204 204 204 F F F
131912024-02-03T17:15:12.367Z 11 220-239 120 120 120 204 204 204 F F F
131922024-02-03T17:15:12.367Z 12 240-259 119 119 119 203 203 203 T T F <---
131932024-02-03T17:15:12.367Z 13 260-279 117 117 117 200 200 200 F F F
131942024-02-03T17:15:12.367Z 14 280-299 117 117 117 200 200 200 F F F
131952024-02-03T17:15:12.367Z 15 300-319 119 119 119 203 203 203 F F F
131962024-02-03T17:15:12.367Z 16 320-339 120 120 120 204 204 204 T T F <---
131972024-02-03T17:15:12.367Z 17 340-359 116 116 116 199 199 199 T T F <---
131982024-02-03T17:15:12.367Z 18 360-379 118 118 118 201 201 201 F F F
131992024-02-03T17:15:12.368Z 19 380-399 117 117 117 200 200 200 F F F
132002024-02-03T17:15:12.368Z 20 400-419 118 118 118 201 201 201 F F F
132012024-02-03T17:15:12.368Z 21 420-439 119 119 119 203 203 203 F F F
132022024-02-03T17:15:12.368Z 22 440-459 118 118 118 201 201 201 F F F
132032024-02-03T17:15:12.368Z 23 460-479 120 120 120 204 204 204 F F F
132042024-02-03T17:15:12.368Z 24 480-499 100 100 100 173 173 173 F F F
132052024-02-03T17:15:12.368Z 25 500-519 118 118 118 201 201 201 F F F
132062024-02-03T17:15:12.368Z 26 520-539 117 117 117 200 200 200 F F F
132072024-02-03T17:15:12.368Z 27 540-559 112 112 112 191 191 191 T T F <---
132082024-02-03T17:15:12.368Z 28 560-579 114 114 114 196 196 196 F F F
132092024-02-03T17:15:12.368Z 29 580-599 120 120 120 204 204 204 F F F
132102024-02-03T17:15:12.368ZMax gen: 120, Max flush: 204
132112024-02-03T17:15:12.368ZError: Difference in extent metadata found!
132122024-02-03T17:15:12.369ZA Difference in extent metadata is expected here
132132024-02-03T17:15:12.369ZOn loop 60
132142024-02-03T17:15:12.369Z
132152024-02-03T17:15:12.373ZVerifying data now
132162024-02-03T17:15:17.393ZNo change after verify
132172024-02-03T17:15:17.393ZLoop: 60 Downstairs dump after verify (and repair):
132182024-02-03T17:15:17.410ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
132192024-02-03T17:15:17.410Z 0 000-019 120 120 120 204 204 204 F F F
132202024-02-03T17:15:17.410Z 1 020-039 118 118 118 202 202 202 F F F
132212024-02-03T17:15:17.410Z 2 040-059 121 121 121 205 205 205 F F F
132222024-02-03T17:15:17.410Z 3 060-079 121 121 121 205 205 205 F F F
132232024-02-03T17:15:17.428Z 4 080-099 121 121 121 205 205 205 F F F
132242024-02-03T17:15:17.429Z 5 100-119 112 112 112 192 192 192 F F F
132252024-02-03T17:15:17.429Z 6 120-139 121 121 121 205 205 205 F F F
132262024-02-03T17:15:17.429Z 7 140-159 121 121 121 205 205 205 F F F
132272024-02-03T17:15:17.430Z 8 160-179 121 121 121 205 205 205 F F F
132282024-02-03T17:15:17.430Z 9 180-199 121 121 121 205 205 205 F F F
132292024-02-03T17:15:17.430Z 10 200-219 120 120 120 204 204 204 F F F
132302024-02-03T17:15:17.430Z 11 220-239 120 120 120 204 204 204 F F F
132312024-02-03T17:15:17.431Z 12 240-259 121 121 121 205 205 205 F F F
132322024-02-03T17:15:17.431Z 13 260-279 117 117 117 200 200 200 F F F
132332024-02-03T17:15:17.431Z 14 280-299 117 117 117 200 200 200 F F F
132342024-02-03T17:15:17.431Z 15 300-319 119 119 119 203 203 203 F F F
132352024-02-03T17:15:17.432Z 16 320-339 121 121 121 205 205 205 F F F
132362024-02-03T17:15:17.432Z 17 340-359 121 121 121 205 205 205 F F F
132372024-02-03T17:15:17.432Z 18 360-379 118 118 118 201 201 201 F F F
132382024-02-03T17:15:17.432Z 19 380-399 117 117 117 200 200 200 F F F
132392024-02-03T17:15:17.433Z 20 400-419 118 118 118 201 201 201 F F F
132402024-02-03T17:15:17.433Z 21 420-439 119 119 119 203 203 203 F F F
132412024-02-03T17:15:17.433Z 22 440-459 118 118 118 201 201 201 F F F
132422024-02-03T17:15:17.433Z 23 460-479 120 120 120 204 204 204 F F F
132432024-02-03T17:15:17.434Z 24 480-499 100 100 100 173 173 173 F F F
132442024-02-03T17:15:17.434Z 25 500-519 118 118 118 201 201 201 F F F
132452024-02-03T17:15:17.434Z 26 520-539 117 117 117 200 200 200 F F F
132462024-02-03T17:15:17.434Z 27 540-559 121 121 121 205 205 205 F F F
132472024-02-03T17:15:17.434Z 28 560-579 114 114 114 196 196 196 F F F
132482024-02-03T17:15:17.434Z 29 580-599 120 120 120 204 204 204 F F F
132492024-02-03T17:15:17.435ZMax gen: 121, Max flush: 205
132502024-02-03T17:15:17.435Z
132512024-02-03T17:15:17.435ZBegin loop 61
132522024-02-03T17:15:17.435ZDownstairs to restart: 2
132532024-02-03T17:15:17.451Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.008102131Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623}
132542024-02-03T17:15:17.490Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":Crucible runtime is spawned
132552024-02-03T17:15:17.490Z"crucible","level":30,"time":"2024-02-03T17:15:17.008844129Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623}
132562024-02-03T17:15:17.490Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.008927092Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623}
132572024-02-03T17:15:17.490Z{"msg":"Crucible 7219655f-291d-4fbc-9422-d9637c695c3e has session id: 0411d38d-ad60-404e-bfd4-ad3035b0d7f7","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.008969918Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132582024-02-03T17:15:17.491Z{"msg":"Upstairs opts: Upstairs UUID: 7219655f-291d-4fbc-9422-d9637c695c3e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.009060407Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132592024-02-03T17:15:17.491Z{"msg":"Crucible stats registered with UUID: 7219655f-291d-4fbc-9422-d9637c695c3e","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.009105726Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132602024-02-03T17:15:17.491Z{"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.009245419Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623}
132612024-02-03T17:15:17.491Z{"msg":"7219655f-291d-4fbc-9422-d9637c695c3e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.009285412Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132622024-02-03T17:15:17.491Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.009334241Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132632024-02-03T17:15:17.491Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.009369857Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132642024-02-03T17:15:17.492Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.009496466Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132652024-02-03T17:15:17.492Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.009542571Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132662024-02-03T17:15:17.492Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.009583753Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132672024-02-03T17:15:17.492Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.00966439Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132682024-02-03T17:15:17.492Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.009809671Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"":"io task","client":"2","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132692024-02-03T17:15:17.492Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.00989332Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"":"io task","client":"1","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132702024-02-03T17:15:17.492Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.009984167Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"":"io task","client":"0","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132712024-02-03T17:15:17.492Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.010100522Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"":"io task","client":"2","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132722024-02-03T17:15:17.493Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.010172348Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"":"io task","client":"1","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132732024-02-03T17:15:17.493Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.010199878Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"":"io task","client":"0","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132742024-02-03T17:15:17.493Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.010951529Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132752024-02-03T17:15:17.493Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.01100388Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132762024-02-03T17:15:17.494Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.011127367Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132772024-02-03T17:15:17.494Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.011248371Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132782024-02-03T17:15:17.494Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.012251728Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132792024-02-03T17:15:17.494Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.012536359Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132802024-02-03T17:15:17.494Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.012612308Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132812024-02-03T17:15:17.495Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.012639983Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132822024-02-03T17:15:17.495Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.013438032Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132832024-02-03T17:15:17.495Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.013513096Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132842024-02-03T17:15:17.495Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.013541297Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132852024-02-03T17:15:17.496Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.013562412Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132862024-02-03T17:15:17.496Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.013611612Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132872024-02-03T17:15:17.496Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.013637151Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132882024-02-03T17:15:17.496Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.013725804Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132892024-02-03T17:15:17.496Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.014480864Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132902024-02-03T17:15:17.497Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.014543322Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132912024-02-03T17:15:17.497Z{"msg":"[0]R flush_numbers[0..12]: [204, 202, 205, 205, 205, 192, 205, 205, 205, 205, 204, 204]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.01457053Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132922024-02-03T17:15:17.497Z{"msg":"[0]R generation[0..12]: [120, 118, 121, 121, 121, 112, 121, 121, 121, 121, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.014639098Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132932024-02-03T17:15:17.497Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.014675766Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132942024-02-03T17:15:17.497Z{"msg":"[1]R flush_numbers[0..12]: [204, 202, 205, 205, 205, 192, 205, 205, 205, 205, 204, 204]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.014741719Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132952024-02-03T17:15:17.497Z{"msg":"[1]R generation[0..12]: [120, 118, 121, 121, 121, 112, 121, 121, 121, 121, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.014770849Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132962024-02-03T17:15:17.498ZWait for a query_work_queue command to finish before sending IO
132972024-02-03T17:15:17.498Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.014842237Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132982024-02-03T17:15:17.498Z{"msg":"[2]R flush_numbers[0..12]: [204, 202, 205, 205, 205, 192, 205, 205, 205, 205, 204, 204]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.014876975Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
132992024-02-03T17:15:17.498Z{"msg":"[2]R generation[0..12]: [120, 118, 121, 121, 121, 112, 121, 121, 121, 121, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.014941Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
133002024-02-03T17:15:17.499Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.014970923Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
133012024-02-03T17:15:17.499Z{"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.015040787Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
133022024-02-03T17:15:17.499Z{"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.01506867Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
133032024-02-03T17:15:17.499Z{"msg":"Next flush: 206","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.015096702Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
133042024-02-03T17:15:17.499Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.015158996Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
133052024-02-03T17:15:17.500Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.015185903Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
133062024-02-03T17:15:17.500Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.015249703Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
133072024-02-03T17:15:17.500Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.015277484Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
133082024-02-03T17:15:17.500Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.015303151Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
133092024-02-03T17:15:17.501Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.01536688Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
133102024-02-03T17:15:17.501Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.015393397Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
133112024-02-03T17:15:17.501Z{"msg":"7219655f-291d-4fbc-9422-d9637c695c3e is now active with session: 0411d38d-ad60-404e-bfd4-ad3035b0d7f7","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.015460242Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
133122024-02-03T17:15:17.501Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.01548566Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623,"session_id":"0411d38d-ad60-404e-bfd4-ad3035b0d7f7"}
133132024-02-03T17:15:17.502Z{"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:17.015548765Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623}
133142024-02-03T17:15:17.502ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
133152024-02-03T17:15:17.502ZLoading write count information from file "/tmp/test_repair_verify.data"
133162024-02-03T17:15:17.502ZRun Repair workload
133172024-02-03T17:15:17.502Z01/30 Write block 195 len 16384 data: 11 9 8 6
133182024-02-03T17:15:17.502Z02/30 Write block 399 len 8192 data: 12 10
133192024-02-03T17:15:17.503Z03/30 Write block 94 len 8192 data: 12 10
133202024-02-03T17:15:17.503Z04/30 Write block 244 len 12288 data: 7 7 8
133212024-02-03T17:15:17.503Z05/30 Write block 529 len 4096 data: 10
133222024-02-03T17:15:17.503Z06/30 Write block 265 len 36864 data: 15 13 13 16 13 13 12 12 11
133232024-02-03T17:15:17.503Z07/30 Read block 287 len 28672
133242024-02-03T17:15:17.516Z08/30 Write block 475 len 4096 data: 7
133252024-02-03T17:15:17.516Z09/30 Write block 470 len 8192 data: 9 7
133262024-02-03T17:15:17.516Z10/30 Read block 544 len 32768
133272024-02-03T17:15:17.516Z11/30 Write block 420 len 32768 data: 8 10 8 6 6 7 8 11
133282024-02-03T17:15:17.516Z12/30 Read block 575 len 32768
133292024-02-03T17:15:17.516Z13/30 Write block 438 len 36864 data: 10 10 10 11 14 11 10 10 13
133302024-02-03T17:15:17.517Z14/30 Read block 348 len 20480
133312024-02-03T17:15:17.517Z15/30 Flush
133322024-02-03T17:15:17.517Z16/30 Write block 563 len 28672 data: 12 13 12 11 9 12 10
133332024-02-03T17:15:17.517Z17/30 Write block 548 len 4096 data: 10
133342024-02-03T17:15:17.517Z18/30 Write block 49 len 36864 data: 10 10 10 11 8 8 8 9 8
133352024-02-03T17:15:17.517Z19/30 Flush
133362024-02-03T17:15:17.517Z20/30 Write block 502 len 8192 data: 9 11
133372024-02-03T17:15:17.517Z21/30 Write block 148 len 36864 data: 9 9 7 10 9 9 8 8 11
133382024-02-03T17:15:17.517Z22/30 Write block 556 len 36864 data: 10 10 8 9 9 10 12 13 14
133392024-02-03T17:15:17.517Z23/30 Read block 82 len 32768
133402024-02-03T17:15:17.517Z24/30 Read block 378 len 20480
133412024-02-03T17:15:17.517Z25/30 Write block 366 len 16384 data: 8 10 10 10
133422024-02-03T17:15:17.517Z26/30 Read block 505 len 16384
133432024-02-03T17:15:17.518Z27/30 Write block 498 len 4096 data: 6
133442024-02-03T17:15:17.518Z28/30 Read block 585 len 36864
133452024-02-03T17:15:17.518Z29/30 Write block 289 len 16384 data: 10 9 8 7
133462024-02-03T17:15:17.518Z30/30 Write block 255 len 8192 data: 7 10
133472024-02-03T17:15:17.518Z----------------------------------------------------------------
133482024-02-03T17:15:17.518Z Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:30
133492024-02-03T17:15:17.518ZStates: Active Active Active
133502024-02-03T17:15:17.518ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
133512024-02-03T17:15:17.518Z 1 Acked 1000 Write 4 Done Done Done false
133522024-02-03T17:15:17.518Z 2 Acked 1001 Write 2 Done Done Done false
133532024-02-03T17:15:17.518Z 3 Acked 1002 Write 2 Done Done Done false
133542024-02-03T17:15:17.518Z 4 Acked 1003 Write 3 Done Done Sent false
133552024-02-03T17:15:17.518Z 5 Acked 1004 Write 1 Done Done Sent false
133562024-02-03T17:15:17.519Z 6 Acked 1005 Write 9 Done Done Sent false
133572024-02-03T17:15:17.519Z 7 Acked 1006 Read 7 Done Done Sent false
133582024-02-03T17:15:17.519Z 8 Acked 1007 Write 1 Done Done Sent false
133592024-02-03T17:15:17.519Z 9 Acked 1008 Write 2 Done Done Sent false
133602024-02-03T17:15:17.519Z 10 Acked 1009 Read 8 Done Done Sent false
133612024-02-03T17:15:17.519Z 11 Acked 1010 Write 8 Done Done Sent false
133622024-02-03T17:15:17.519Z 12 Acked 1011 Read 8 Done Done Sent false
133632024-02-03T17:15:17.519Z 13 Acked 1012 Write 9 Done Done Sent false
133642024-02-03T17:15:17.519Z 14 Acked 1013 Read 5 Done Done Sent false
133652024-02-03T17:15:17.519Z 15 Acked 1014 Flush 0 Done Done Sent false
133662024-02-03T17:15:17.519Z 16 Acked 1015 Write 7 Done Done Sent false
133672024-02-03T17:15:17.519Z 17 Acked 1016 Write 1 Done Done Sent false
133682024-02-03T17:15:17.519Z 18 Acked 1017 Write 9 Done Done Sent false
133692024-02-03T17:15:17.519Z 19 Acked 1018 Flush 0 Done Done Sent false
133702024-02-03T17:15:17.520Z 20 Acked 1019 Write 2 Done Done Sent false
133712024-02-03T17:15:17.520Z 21 Acked 1020 Write 9 Done Done Sent false
133722024-02-03T17:15:17.520Z 22 Acked 1021 Write 9 Done Done Sent false
133732024-02-03T17:15:17.520Z 23 Acked 1022 Read 8 Done Done Sent false
133742024-02-03T17:15:17.520Z 24 Acked 1023 Read 5 Done Done Sent false
133752024-02-03T17:15:17.520Z 25 Acked 1024 Write 4 Done Done Sent false
133762024-02-03T17:15:17.520Z 26 Acked 1025 Read 4 Done Done Sent false
133772024-02-03T17:15:17.520Z 27 Acked 1026 Write 1 Done Done Sent false
133782024-02-03T17:15:17.520Z 28 Acked 1027 Read 9 Done Done Sent false
133792024-02-03T17:15:17.520Z 29 Acked 1028 Write 4 Done Done Sent false
133802024-02-03T17:15:17.520Z 30 Acked 1029 Write 2 Sent Sent Sent false
133812024-02-03T17:15:17.521Z STATES DS:0 DS:1 DS:2 TOTAL
133822024-02-03T17:15:17.521Z New 0 0 0 0
133832024-02-03T17:15:17.521Z Sent 1 1 27 29
133842024-02-03T17:15:17.521Z Done 29 29 3 61
133852024-02-03T17:15:17.521Z Skipped 0 0 0 0
133862024-02-03T17:15:17.521Z Error 0 0 0 0
133872024-02-03T17:15:17.521ZLast Flush: 1018 1018 0
133882024-02-03T17:15:17.521ZDownstairs last five completed:
133892024-02-03T17:15:17.521ZUpstairs last five completed: 30 29 28 27 26
133902024-02-03T17:15:17.521Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:15:17.064226477Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1623}
133912024-02-03T17:15:17.521ZWrote out file "/tmp/test_repair_verify.data"
133922024-02-03T17:15:17.521Z
133932024-02-03T17:15:17.530ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
133942024-02-03T17:15:17.547ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
133952024-02-03T17:15:17.547Z 0 000-019 120 120 120 204 204 204 F F F
133962024-02-03T17:15:17.547Z 1 020-039 118 118 118 202 202 202 F F F
133972024-02-03T17:15:17.547Z 2 040-059 122 122 121 207 207 205 F F F <---
133982024-02-03T17:15:17.564Z 3 060-079 121 121 121 205 205 205 F F F
133992024-02-03T17:15:17.564Z 4 080-099 122 122 121 206 206 205 F F T <---
134002024-02-03T17:15:17.564Z 5 100-119 112 112 112 192 192 192 F F F
134012024-02-03T17:15:17.564Z 6 120-139 121 121 121 205 205 205 F F F
134022024-02-03T17:15:17.564Z 7 140-159 121 121 121 205 205 205 T T F <---
134032024-02-03T17:15:17.565Z 8 160-179 121 121 121 205 205 205 F F F
134042024-02-03T17:15:17.565Z 9 180-199 122 122 121 206 206 205 F F T <---
134052024-02-03T17:15:17.565Z 10 200-219 120 120 120 204 204 204 F F F
134062024-02-03T17:15:17.565Z 11 220-239 120 120 120 204 204 204 F F F
134072024-02-03T17:15:17.565Z 12 240-259 122 122 121 206 206 205 T T F <---
134082024-02-03T17:15:17.566Z 13 260-279 122 122 117 206 206 200 F F F <---
134092024-02-03T17:15:17.566Z 14 280-299 117 117 117 200 200 200 T T F <---
134102024-02-03T17:15:17.566Z 15 300-319 119 119 119 203 203 203 F F F
134112024-02-03T17:15:17.566Z 16 320-339 121 121 121 205 205 205 F F F
134122024-02-03T17:15:17.567Z 17 340-359 121 121 121 205 205 205 F F F
134132024-02-03T17:15:17.567Z 18 360-379 118 118 118 201 201 201 T T F <---
134142024-02-03T17:15:17.567Z 19 380-399 122 122 117 206 206 200 F F T <---
134152024-02-03T17:15:17.567Z 20 400-419 122 122 118 206 206 201 F F T <---
134162024-02-03T17:15:17.568Z 21 420-439 122 122 119 206 206 203 F F F <---
134172024-02-03T17:15:17.568Z 22 440-459 122 122 118 206 206 201 F F F <---
134182024-02-03T17:15:17.568Z 23 460-479 122 122 120 206 206 204 F F F <---
134192024-02-03T17:15:17.568Z 24 480-499 100 100 100 173 173 173 T T F <---
134202024-02-03T17:15:17.569Z 25 500-519 118 118 118 201 201 201 T T F <---
134212024-02-03T17:15:17.569Z 26 520-539 122 122 117 206 206 200 F F F <---
134222024-02-03T17:15:17.569Z 27 540-559 122 122 121 207 207 205 T T F <---
134232024-02-03T17:15:17.569Z 28 560-579 122 122 114 207 207 196 T T F <---
134242024-02-03T17:15:17.569Z 29 580-599 120 120 120 204 204 204 F F F
134252024-02-03T17:15:17.570ZMax gen: 122, Max flush: 207
134262024-02-03T17:15:17.570ZError: Difference in extent metadata found!
134272024-02-03T17:15:17.570ZA Difference in extent metadata is expected here
134282024-02-03T17:15:17.570ZOn loop 61
134292024-02-03T17:15:17.571Z
134302024-02-03T17:15:17.571ZVerifying data now
134312024-02-03T17:15:23.060ZNo change after verify
134322024-02-03T17:15:23.061ZLoop: 61 Downstairs dump after verify (and repair):
134332024-02-03T17:15:23.089ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
134342024-02-03T17:15:23.089Z 0 000-019 120 120 120 204 204 204 F F F
134352024-02-03T17:15:23.089Z 1 020-039 118 118 118 202 202 202 F F F
134362024-02-03T17:15:23.089Z 2 040-059 122 122 122 207 207 207 F F F
134372024-02-03T17:15:23.089Z 3 060-079 121 121 121 205 205 205 F F F
134382024-02-03T17:15:23.090Z 4 080-099 122 122 122 206 206 206 F F F
134392024-02-03T17:15:23.090Z 5 100-119 112 112 112 192 192 192 F F F
134402024-02-03T17:15:23.090Z 6 120-139 121 121 121 205 205 205 F F F
134412024-02-03T17:15:23.508Z 7 140-159 123 123 123 208 208 208 F F F
134422024-02-03T17:15:23.508Z 8 160-179 121 121 121 205 205 205 F F F
134432024-02-03T17:15:23.508Z 9 180-199 122 122 122 206 206 206 F F F
134442024-02-03T17:15:23.508Z 10 200-219 120 120 120 204 204 204 F F F
134452024-02-03T17:15:23.508Z 11 220-239 120 120 120 204 204 204 F F F
134462024-02-03T17:15:23.508Z 12 240-259 123 123 123 208 208 208 F F F
134472024-02-03T17:15:23.508Z 13 260-279 122 122 122 206 206 206 F F F
134482024-02-03T17:15:23.509Z 14 280-299 123 123 123 208 208 208 F F F
134492024-02-03T17:15:23.509Z 15 300-319 119 119 119 203 203 203 F F F
134502024-02-03T17:15:23.509Z 16 320-339 121 121 121 205 205 205 F F F
134512024-02-03T17:15:23.509Z 17 340-359 121 121 121 205 205 205 F F F
134522024-02-03T17:15:23.509Z 18 360-379 123 123 123 208 208 208 F F F
134532024-02-03T17:15:23.509Z 19 380-399 122 122 122 206 206 206 F F F
134542024-02-03T17:15:23.509Z 20 400-419 122 122 122 206 206 206 F F F
134552024-02-03T17:15:23.509Z 21 420-439 122 122 122 206 206 206 F F F
134562024-02-03T17:15:23.509Z 22 440-459 122 122 122 206 206 206 F F F
134572024-02-03T17:15:23.510Z 23 460-479 122 122 122 206 206 206 F F F
134582024-02-03T17:15:23.510Z 24 480-499 123 123 123 208 208 208 F F F
134592024-02-03T17:15:23.510Z 25 500-519 123 123 123 208 208 208 F F F
134602024-02-03T17:15:23.510Z 26 520-539 122 122 122 206 206 206 F F F
134612024-02-03T17:15:23.510Z 27 540-559 123 123 123 208 208 208 F F F
134622024-02-03T17:15:23.510Z 28 560-579 123 123 123 208 208 208 F F F
134632024-02-03T17:15:23.511Z 29 580-599 120 120 120 204 204 204 F F F
134642024-02-03T17:15:23.511ZMax gen: 123, Max flush: 208
134652024-02-03T17:15:23.511Z
134662024-02-03T17:15:23.511ZBegin loop 62
134672024-02-03T17:15:23.511ZDownstairs to restart: 1
134682024-02-03T17:15:23.531Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.689306269Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631}
134692024-02-03T17:15:23.531Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.689750267Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631}
134702024-02-03T17:15:23.531Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
134712024-02-03T17:15:23.531Z,"time":"2024-02-03T17:15:22.689835062Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631}
134722024-02-03T17:15:23.531Z{"msg":"Crucible 282d16e3-edc6-453f-8a82-ba31429cc2d9 has session id: 168f02db-e7fc-49f3-918d-7fd3f36f7323","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.689940128Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
134732024-02-03T17:15:23.531Z{"msg":"Upstairs opts: Upstairs UUID: 282d16e3-edc6-453f-8a82-ba31429cc2d9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.689987011Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
134742024-02-03T17:15:23.531Z{"msg":"Crucible stats registered with UUID: 282d16e3-edc6-453f-8a82-ba31429cc2d9","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.690059855Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
134752024-02-03T17:15:23.531Z{"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.690087398Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631}
134762024-02-03T17:15:23.531Z{"msg":"282d16e3-edc6-453f-8a82-ba31429cc2d9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.690315609Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
134772024-02-03T17:15:23.531Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.69037736Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
134782024-02-03T17:15:23.531Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.69040456Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
134792024-02-03T17:15:23.532Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.690432026Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
134802024-02-03T17:15:23.532Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.690474828Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
134812024-02-03T17:15:23.532Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.690512651Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
134822024-02-03T17:15:23.532Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.69055093Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
134832024-02-03T17:15:23.532Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.690624241Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"":"io task","client":"0","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
134842024-02-03T17:15:23.532Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.690656344Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"":"io task","client":"2","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
134852024-02-03T17:15:23.532Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.690687885Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"":"io task","client":"1","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
134862024-02-03T17:15:23.532Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.690975992Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"":"io task","client":"0","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
134872024-02-03T17:15:23.532Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.691086974Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"":"io task","client":"1","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
134882024-02-03T17:15:23.533Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.691134183Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"":"io task","client":"2","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
134892024-02-03T17:15:23.533Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.692860301Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
134902024-02-03T17:15:23.533Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.692936165Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
134912024-02-03T17:15:23.533Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.692963812Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
134922024-02-03T17:15:23.533Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.693123924Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
134932024-02-03T17:15:23.533Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.693493381Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
134942024-02-03T17:15:23.533Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.693592194Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
134952024-02-03T17:15:23.533Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.694380962Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
134962024-02-03T17:15:23.533Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.694565608Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
134972024-02-03T17:15:23.533Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.694960769Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
134982024-02-03T17:15:23.533Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.695427207Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
134992024-02-03T17:15:23.533Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.695520613Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
135002024-02-03T17:15:23.534Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.695568042Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
135012024-02-03T17:15:23.534Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.69569731Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
135022024-02-03T17:15:23.534Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.695729109Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
135032024-02-03T17:15:23.534Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.695769062Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
135042024-02-03T17:15:23.534Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.695845394Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
135052024-02-03T17:15:23.534Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.69588587Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
135062024-02-03T17:15:23.534Z{"msg":"[0]R flush_numbers[0..12]: [204, 202, 207, 205, 206, 192, 205, 208, 205, 206, 204, 204]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.695940955Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
135072024-02-03T17:15:23.534Z{"msg":"[0]R generation[0..12]: [120, 118, 122, 121, 122, 112, 121, 123, 121, 122, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.695967548Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
135082024-02-03T17:15:23.534Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.695991792Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
135092024-02-03T17:15:23.534Z{"msg":"[1]R flush_numbers[0..12]: [204, 202, 207, 205, 206, 192, 205, 208, 205, 206, 204, 204]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.696040781Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
135102024-02-03T17:15:23.534Z{"msg":"[1]R generation[0..12]: [120, 118, 122, 121, 122, 112, 121, 123, 121, 122, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.696066556Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
135112024-02-03T17:15:23.534Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.696090384Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
135122024-02-03T17:15:23.534Z{"msg":"[2]R flush_numbers[0..12]: [204, 202, 207, 205, 206, 192, 205, 208, 205, 206, 204, 204]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.696140491Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
135132024-02-03T17:15:23.534Z{"msg":"[2]R generation[0..12]: [120, 118, 122, 121, 122, 112, 121, 123, 121, 122, 120, 120]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
135142024-02-03T17:15:23.534Z,"time":"2024-02-03T17:15:22.696166002Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
135152024-02-03T17:15:23.535Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.696192093Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
135162024-02-03T17:15:23.535Z{"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.696238623Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
135172024-02-03T17:15:23.535Z{"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.696263145Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
135182024-02-03T17:15:23.535Z{"msg":"Next flush: 209","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.696284584Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
135192024-02-03T17:15:23.535Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.696330968Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
135202024-02-03T17:15:23.535Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.696355465Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
135212024-02-03T17:15:23.535Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.696375422Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
135222024-02-03T17:15:23.535Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.696397005Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
135232024-02-03T17:15:23.535Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.696463617Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
135242024-02-03T17:15:23.535Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.696488306Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
135252024-02-03T17:15:23.535Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.696508877Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
135262024-02-03T17:15:23.535Z{"msg":"282d16e3-edc6-453f-8a82-ba31429cc2d9 is now active with session: 168f02db-e7fc-49f3-918d-7fd3f36f7323","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.696530414Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
135272024-02-03T17:15:23.535Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.696558894Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631,"session_id":"168f02db-e7fc-49f3-918d-7fd3f36f7323"}
135282024-02-03T17:15:23.535Z{"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:22.696607598Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631}
135292024-02-03T17:15:23.535ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
135302024-02-03T17:15:23.536ZLoading write count information from file "/tmp/test_repair_verify.data"
135312024-02-03T17:15:23.536ZRun Repair workload
135322024-02-03T17:15:23.536Z01/30 Write block 397 len 28672 data: 8 11 13 11 8 7 7
135332024-02-03T17:15:23.536Z02/30 Read block 568 len 28672
135342024-02-03T17:15:23.536Z03/30 Flush
135352024-02-03T17:15:23.536Z04/30 Write block 8 len 24576 data: 13 15 14 9 9 11
135362024-02-03T17:15:23.536Z05/30 Write block 531 len 16384 data: 10 9 9 7
135372024-02-03T17:15:23.536Z06/30 Write block 143 len 32768 data: 11 7 6 6 5 10 10 8
135382024-02-03T17:15:23.536Z07/30 Read block 302 len 16384
135392024-02-03T17:15:23.536Z08/30 Read block 186 len 36864
135402024-02-03T17:15:23.536Z09/30 Write block 94 len 16384 data: 13 11 12 11
135412024-02-03T17:15:23.536Z10/30 Write block 149 len 16384 data: 11 9 11 10
135422024-02-03T17:15:23.536Z11/30 Read block 128 len 40960
135432024-02-03T17:15:23.537Z12/30 Write block 192 len 20480 data: 14 11 11 12 10
135442024-02-03T17:15:23.537Z13/30 Read block 450 len 24576
135452024-02-03T17:15:23.537Z14/30 Read block 82 len 4096
135462024-02-03T17:15:23.537Z15/30 Read block 83 len 24576
135472024-02-03T17:15:23.537Z16/30 Write block 333 len 16384 data: 8 10 9 10
135482024-02-03T17:15:23.537Z17/30 Read block 443 len 12288
135492024-02-03T17:15:23.537Z18/30 Flush
135502024-02-03T17:15:23.537Z19/30 Write block 40 len 28672 data: 7 6 8 6 7 9 9
135512024-02-03T17:15:23.537Z20/30 Read block 182 len 32768
135522024-02-03T17:15:23.537Z21/30 Write block 458 len 28672 data: 8 10 13 11 9 10 9
135532024-02-03T17:15:23.537Z22/30 Read block 223 len 16384
135542024-02-03T17:15:23.537Z23/30 Read block 191 len 28672
135552024-02-03T17:15:23.538Z24/30 Write block 556 len 28672 data: 11 11 9 10 10 11 13
135562024-02-03T17:15:23.538Z25/30 Write block 76 len 8192 data: 11 10
135572024-02-03T17:15:23.538Z26/30 Flush
135582024-02-03T17:15:23.538Z27/30 Write block 437 len 28672 data: 10 11 11 11 12 15 12
135592024-02-03T17:15:23.538Z28/30 Write block 101 len 40960 data: 10 7 8 7 8 10 11 6 6 6
135602024-02-03T17:15:23.538Z29/30 Read block 402 len 40960
135612024-02-03T17:15:23.538Z30/30 Read block 71 len 36864
135622024-02-03T17:15:23.538Z----------------------------------------------------------------
135632024-02-03T17:15:23.538Z Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:27
135642024-02-03T17:15:23.538ZStates: Active Active Active
135652024-02-03T17:15:23.538ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
135662024-02-03T17:15:23.538Z 4 Acked 1003 Write 6 Done Done Done false
135672024-02-03T17:15:23.538Z 5 Acked 1004 Write 4 Done Sent Done false
135682024-02-03T17:15:23.545Z 6 Acked 1005 Write 8 Done Sent Done false
135692024-02-03T17:15:23.545Z 7 Acked 1006 Read 4 Done Sent Done false
135702024-02-03T17:15:23.545Z 8 Acked 1007 Read 9 Done Sent Done false
135712024-02-03T17:15:23.545Z 9 Acked 1008 Write 4 Done Sent Done false
135722024-02-03T17:15:23.545Z 10 Acked 1009 Write 4 Done Sent Done false
135732024-02-03T17:15:23.546Z 11 Acked 1010 Read 10 Done Sent Done false
135742024-02-03T17:15:23.546Z 12 Acked 1011 Write 5 Done Sent Done false
135752024-02-03T17:15:23.546Z 13 Acked 1012 Read 6 Done Sent Done false
135762024-02-03T17:15:23.546Z 14 Acked 1013 Read 1 Done Sent Done false
135772024-02-03T17:15:23.546Z 15 Acked 1014 Read 6 Done Sent Done false
135782024-02-03T17:15:23.546Z 16 Acked 1015 Write 4 Done Sent Done false
135792024-02-03T17:15:23.546Z 17 Acked 1016 Read 3 Done Sent Done false
135802024-02-03T17:15:23.546Z 18 Acked 1017 Flush 0 Done Sent Done false
135812024-02-03T17:15:23.546Z 19 Acked 1018 Write 7 Done Sent Done false
135822024-02-03T17:15:23.546Z 20 Acked 1019 Read 8 Done Sent Done false
135832024-02-03T17:15:23.546Z 21 Acked 1020 Write 7 Done Sent Done false
135842024-02-03T17:15:23.546Z 22 Acked 1021 Read 4 Done Sent Done false
135852024-02-03T17:15:23.546Z 23 Acked 1022 Read 7 Done Sent Done false
135862024-02-03T17:15:23.547Z 24 Acked 1023 Write 7 Done Sent Done false
135872024-02-03T17:15:23.547Z 25 Acked 1024 Write 2 Done Sent Done false
135882024-02-03T17:15:23.547Z 26 Acked 1025 Flush 0 Done Sent Done false
135892024-02-03T17:15:23.547Z 27 Acked 1026 Write 7 Done Sent Done false
135902024-02-03T17:15:23.547Z 28 Acked 1027 Write 10 Done Sent Done false
135912024-02-03T17:15:23.547Z 29 Acked 1028 Read 10 Done Sent Done false
135922024-02-03T17:15:23.547Z 30 Acked 1029 Read 9 Done Sent Done false
135932024-02-03T17:15:23.547Z STATES DS:0 DS:1 DS:2 TOTAL
135942024-02-03T17:15:23.547Z New 0 0 0 0
135952024-02-03T17:15:23.547Z Sent 0 26 0 26
135962024-02-03T17:15:23.547Z Done 27 1 27 55
135972024-02-03T17:15:23.547Z Skipped 0 0 0 0
135982024-02-03T17:15:23.547Z Error 0 0 0 0
135992024-02-03T17:15:23.548ZLast Flush: 1025 1002 1025
136002024-02-03T17:15:23.548ZDownstairs last five completed: 1002 1001 1000
136012024-02-03T17:15:23.548ZUpstairs last five completed: 30 29 28 27 26
136022024-02-03T17:15:23.548Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:15:22.754641364Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1631}
136032024-02-03T17:15:23.548ZWrote out file "/tmp/test_repair_verify.data"
136042024-02-03T17:15:23.548Z
136052024-02-03T17:15:23.548ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
136062024-02-03T17:15:23.548ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
136072024-02-03T17:15:23.548Z 0 000-019 124 120 124 210 204 210 F T F <---
136082024-02-03T17:15:23.548Z 1 020-039 118 118 118 202 202 202 F F F
136092024-02-03T17:15:23.548Z 2 040-059 124 122 124 211 207 211 F F F <---
136102024-02-03T17:15:23.548Z 3 060-079 124 121 124 211 205 211 F F F <---
136112024-02-03T17:15:23.548Z 4 080-099 124 122 124 210 206 210 F F F <---
136122024-02-03T17:15:23.549Z 5 100-119 112 112 112 192 192 192 T F T <---
136132024-02-03T17:15:23.549Z 6 120-139 121 121 121 205 205 205 F F F
136142024-02-03T17:15:23.549Z 7 140-159 124 123 124 210 208 210 F F F <---
136152024-02-03T17:15:23.549Z 8 160-179 121 121 121 205 205 205 F F F
136162024-02-03T17:15:23.549Z 9 180-199 124 122 124 210 206 210 F F F <---
136172024-02-03T17:15:23.549Z 10 200-219 120 120 120 204 204 204 F F F
136182024-02-03T17:15:23.549Z 11 220-239 120 120 120 204 204 204 F F F
136192024-02-03T17:15:23.549Z 12 240-259 123 123 123 208 208 208 F F F
136202024-02-03T17:15:23.549Z 13 260-279 122 122 122 206 206 206 F F F
136212024-02-03T17:15:23.549Z 14 280-299 123 123 123 208 208 208 F F F
136222024-02-03T17:15:23.549Z 15 300-319 119 119 119 203 203 203 F F F
136232024-02-03T17:15:23.549Z 16 320-339 124 121 124 210 205 210 F F F <---
136242024-02-03T17:15:23.549Z 17 340-359 121 121 121 205 205 205 F F F
136252024-02-03T17:15:23.549Z 18 360-379 123 123 123 208 208 208 F F F
136262024-02-03T17:15:23.550Z 19 380-399 124 124 124 209 209 209 F F F
136272024-02-03T17:15:23.550Z 20 400-419 124 124 124 209 209 209 F F F
136282024-02-03T17:15:23.550Z 21 420-439 122 122 122 206 206 206 T F T <---
136292024-02-03T17:15:23.550Z 22 440-459 124 122 124 211 206 211 T F T <---
136302024-02-03T17:15:23.550Z 23 460-479 124 122 124 211 206 211 F F F <---
136312024-02-03T17:15:23.550Z 24 480-499 123 123 123 208 208 208 F F F
136322024-02-03T17:15:23.550Z 25 500-519 123 123 123 208 208 208 F F F
136332024-02-03T17:15:23.550Z 26 520-539 124 122 124 210 206 210 F F F <---
136342024-02-03T17:15:23.550Z 27 540-559 124 123 124 211 208 211 F F F <---
136352024-02-03T17:15:23.550Z 28 560-579 124 123 124 211 208 211 F F F <---
136362024-02-03T17:15:23.550Z 29 580-599 120 120 120 204 204 204 F F F
136372024-02-03T17:15:23.550ZMax gen: 124, Max flush: 211
136382024-02-03T17:15:23.550ZError: Difference in extent metadata found!
136392024-02-03T17:15:23.551ZA Difference in extent metadata is expected here
136402024-02-03T17:15:23.551ZOn loop 62
136412024-02-03T17:15:23.551Z
136422024-02-03T17:15:23.551ZVerifying data now
136432024-02-03T17:15:28.429ZNo change after verify
136442024-02-03T17:15:28.429ZLoop: 62 Downstairs dump after verify (and repair):
136452024-02-03T17:15:28.452ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
136462024-02-03T17:15:28.452Z 0 000-019 124 124 124 210 210 210 F F F
136472024-02-03T17:15:28.456Z 1 020-039 118 118 118 202 202 202 F F F
136482024-02-03T17:15:28.456Z 2 040-059 124 124 124 211 211 211 F F F
136492024-02-03T17:15:28.456Z 3 060-079 124 124 124 211 211 211 F F F
136502024-02-03T17:15:28.456Z 4 080-099 124 124 124 210 210 210 F F F
136512024-02-03T17:15:28.456Z 5 100-119 125 125 125 212 212 212 F F F
136522024-02-03T17:15:28.456Z 6 120-139 121 121 121 205 205 205 F F F
136532024-02-03T17:15:28.456Z 7 140-159 124 124 124 210 210 210 F F F
136542024-02-03T17:15:28.456Z 8 160-179 121 121 121 205 205 205 F F F
136552024-02-03T17:15:28.456Z 9 180-199 124 124 124 210 210 210 F F F
136562024-02-03T17:15:28.456Z 10 200-219 120 120 120 204 204 204 F F F
136572024-02-03T17:15:28.456Z 11 220-239 120 120 120 204 204 204 F F F
136582024-02-03T17:15:28.456Z 12 240-259 123 123 123 208 208 208 F F F
136592024-02-03T17:15:28.457Z 13 260-279 122 122 122 206 206 206 F F F
136602024-02-03T17:15:28.457Z 14 280-299 123 123 123 208 208 208 F F F
136612024-02-03T17:15:28.457Z 15 300-319 119 119 119 203 203 203 F F F
136622024-02-03T17:15:28.457Z 16 320-339 124 124 124 210 210 210 F F F
136632024-02-03T17:15:28.457Z 17 340-359 121 121 121 205 205 205 F F F
136642024-02-03T17:15:28.457Z 18 360-379 123 123 123 208 208 208 F F F
136652024-02-03T17:15:28.457Z 19 380-399 124 124 124 209 209 209 F F F
136662024-02-03T17:15:28.457Z 20 400-419 124 124 124 209 209 209 F F F
136672024-02-03T17:15:28.457Z 21 420-439 125 125 125 212 212 212 F F F
136682024-02-03T17:15:28.457Z 22 440-459 125 125 125 212 212 212 F F F
136692024-02-03T17:15:28.457Z 23 460-479 124 124 124 211 211 211 F F F
136702024-02-03T17:15:28.457Z 24 480-499 123 123 123 208 208 208 F F F
136712024-02-03T17:15:28.457Z 25 500-519 123 123 123 208 208 208 F F F
136722024-02-03T17:15:28.457Z 26 520-539 124 124 124 210 210 210 F F F
136732024-02-03T17:15:28.457Z 27 540-559 124 124 124 211 211 211 F F F
136742024-02-03T17:15:28.457Z 28 560-579 124 124 124 211 211 211 F F F
136752024-02-03T17:15:28.457Z 29 580-599 120 120 120 204 204 204 F F F
136762024-02-03T17:15:28.457ZMax gen: 125, Max flush: 212
136772024-02-03T17:15:28.460Z
136782024-02-03T17:15:28.460ZBegin loop 63
136792024-02-03T17:15:28.460ZDownstairs to restart: 1
136802024-02-03T17:15:28.488Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.054931744Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639}
136812024-02-03T17:15:28.488Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.055440855Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639}
136822024-02-03T17:15:28.492Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.055591981Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639}
136832024-02-03T17:15:28.492Z{"msg":"Crucible ce51d849-b8b1-450c-ac24-562ae5961ac5 has session id: 5b5564d3-692b-4ad9-a45c-14334d449a76","v":0,"name":"crucible","level":30Crucible runtime is spawned
136842024-02-03T17:15:28.492Z,"time":"2024-02-03T17:15:28.055687163Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
136852024-02-03T17:15:28.492Z{"msg":"Upstairs opts: Upstairs UUID: ce51d849-b8b1-450c-ac24-562ae5961ac5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.055914793Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
136862024-02-03T17:15:28.492Z{"msg":"Crucible stats registered with UUID: ce51d849-b8b1-450c-ac24-562ae5961ac5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.055987004Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
136872024-02-03T17:15:28.492Z{"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.056014171Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639}
136882024-02-03T17:15:28.492Z{"msg":"ce51d849-b8b1-450c-ac24-562ae5961ac5 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.056058088Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
136892024-02-03T17:15:28.492Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.056080692Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
136902024-02-03T17:15:28.492Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.056105781Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
136912024-02-03T17:15:28.492Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.05615355Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
136922024-02-03T17:15:28.492Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.056179474Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
136932024-02-03T17:15:28.492Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.056256731Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
136942024-02-03T17:15:28.492Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.056302856Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
136952024-02-03T17:15:28.492Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.056388718Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"":"io task","client":"0","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
136962024-02-03T17:15:28.492Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.056490528Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"":"io task","client":"1","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
136972024-02-03T17:15:28.493Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.056539502Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"":"io task","client":"2","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
136982024-02-03T17:15:28.493Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.056754803Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"":"io task","client":"2","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
136992024-02-03T17:15:28.493Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.056802032Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"":"io task","client":"1","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137002024-02-03T17:15:28.493Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.056846447Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"":"io task","client":"0","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137012024-02-03T17:15:28.493Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.058333757Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137022024-02-03T17:15:28.493Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.058407571Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137032024-02-03T17:15:28.493Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.058460118Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137042024-02-03T17:15:28.493Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.05851541Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137052024-02-03T17:15:28.493Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.058541374Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137062024-02-03T17:15:28.497Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.058569851Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137072024-02-03T17:15:28.497Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.059448541Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137082024-02-03T17:15:28.497Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.059610085Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137092024-02-03T17:15:28.497Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.06003628Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137102024-02-03T17:15:28.497Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.060685138Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137112024-02-03T17:15:28.497Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.06075653Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137122024-02-03T17:15:28.497Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.060884932Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137132024-02-03T17:15:28.497Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.060976614Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137142024-02-03T17:15:28.497Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.061025857Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137152024-02-03T17:15:28.497Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.061067997Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137162024-02-03T17:15:28.497Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.061106522Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137172024-02-03T17:15:28.497Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.061186472Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137182024-02-03T17:15:28.497Z{"msg":"[0]R flush_numbers[0..12]: [210, 202, 211, 211, 210, 212, 205, 210, 205, 210, 204, 204]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.061270107Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137192024-02-03T17:15:28.498Z{"msg":"[0]R generation[0..12]: [124, 118, 124, 124, 124, 125, 121, 124, 121, 124, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.06135489Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137202024-02-03T17:15:28.498Z{"msg":"Wait for a query_work_queue command to finish before sending IO
137212024-02-03T17:15:28.498Z[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.061542777Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137222024-02-03T17:15:28.498Z{"msg":"[1]R flush_numbers[0..12]: [210, 202, 211, 211, 210, 212, 205, 210, 205, 210, 204, 204]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.061629981Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137232024-02-03T17:15:28.498Z{"msg":"[1]R generation[0..12]: [124, 118, 124, 124, 124, 125, 121, 124, 121, 124, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.061716942Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137242024-02-03T17:15:28.498Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.061802659Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137252024-02-03T17:15:28.498Z{"msg":"[2]R flush_numbers[0..12]: [210, 202, 211, 211, 210, 212, 205, 210, 205, 210, 204, 204]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.061830793Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137262024-02-03T17:15:28.498Z{"msg":"[2]R generation[0..12]: [124, 118, 124, 124, 124, 125, 121, 124, 121, 124, 120, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.06185413Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137272024-02-03T17:15:28.498Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.061877902Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137282024-02-03T17:15:28.498Z{"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.061899752Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137292024-02-03T17:15:28.498Z{"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.061955505Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137302024-02-03T17:15:28.498Z{"msg":"Next flush: 213","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.061981084Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137312024-02-03T17:15:28.498Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.062027941Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137322024-02-03T17:15:28.498Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.062052805Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137332024-02-03T17:15:28.498Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.062072748Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137342024-02-03T17:15:28.498Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.06212076Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137352024-02-03T17:15:28.498Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.062145207Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137362024-02-03T17:15:28.498Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.062167561Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137372024-02-03T17:15:28.498Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.062214954Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137382024-02-03T17:15:28.498Z{"msg":"ce51d849-b8b1-450c-ac24-562ae5961ac5 is now active with session: 5b5564d3-692b-4ad9-a45c-14334d449a76","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.062238763Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137392024-02-03T17:15:28.498Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.062260145Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639,"session_id":"5b5564d3-692b-4ad9-a45c-14334d449a76"}
137402024-02-03T17:15:28.498Z{"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:28.062280985Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639}
137412024-02-03T17:15:28.498ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
137422024-02-03T17:15:28.499ZLoading write count information from file "/tmp/test_repair_verify.data"
137432024-02-03T17:15:28.499ZRun Repair workload
137442024-02-03T17:15:28.502Z01/30 Write block 216 len 36864 data: 7 6 6 6 5 7 5 5 4
137452024-02-03T17:15:28.502Z02/30 Read block 358 len 8192
137462024-02-03T17:15:28.502Z03/30 Write block 581 len 28672 data: 7 6 6 5 6 9 8
137472024-02-03T17:15:28.505Z04/30 Read block 216 len 36864
137482024-02-03T17:15:28.508Z05/30 Read block 191 len 28672
137492024-02-03T17:15:28.508Z06/30 Read block 484 len 36864
137502024-02-03T17:15:28.511Z07/30 Read block 489 len 28672
137512024-02-03T17:15:28.511Z08/30 Write block 181 len 8192 data: 11 10
137522024-02-03T17:15:28.514Z09/30 Read block 551 len 40960
137532024-02-03T17:15:28.514Z10/30 Flush
137542024-02-03T17:15:28.517Z11/30 Flush
137552024-02-03T17:15:28.522Z12/30 Write block 456 len 32768 data: 5 7 9 11 14 12 10 11
137562024-02-03T17:15:28.522Z13/30 Read block 374 len 4096
137572024-02-03T17:15:28.526Z14/30 Write block 31 len 4096 data: 12
137582024-02-03T17:15:28.526Z15/30 Read block 588 len 36864
137592024-02-03T17:15:28.526Z16/30 Read block 377 len 36864
137602024-02-03T17:15:28.529Z17/30 Write block 496 len 16384 data: 7 7 7 6
137612024-02-03T17:15:28.529Z18/30 Write block 587 len 28672 data: 9 11 9 10 11 11 11
137622024-02-03T17:15:28.529Z19/30 Read block 508 len 40960
137632024-02-03T17:15:28.540Z20/30 Read block 468 len 12288
137642024-02-03T17:15:28.543Z21/30 Read block 273 len 24576
137652024-02-03T17:15:28.543Z22/30 Write block 183 len 20480 data: 13 13 17 14 18
137662024-02-03T17:15:28.543Z23/30 Write block 372 len 28672 data: 9 10 10 8 10 12 10
137672024-02-03T17:15:28.544Z24/30 Write block 19 len 28672 data: 18 14 17 14 14 13 12
137682024-02-03T17:15:28.544Z25/30 Flush
137692024-02-03T17:15:28.544Z26/30 Write block 42 len 8192 data: 9 7
137702024-02-03T17:15:28.544Z27/30 Read block 334 len 24576
137712024-02-03T17:15:28.547Z28/30 Read block 170 len 20480
137722024-02-03T17:15:28.547Z29/30 Write block 198 len 32768 data: 7 4 3 4 8 8 7 11
137732024-02-03T17:15:28.550Z30/30 Write block 428 len 8192 data: 11 13
137742024-02-03T17:15:28.550Z----------------------------------------------------------------
137752024-02-03T17:15:28.550Z Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30
137762024-02-03T17:15:28.550ZStates: Active Active Active
137772024-02-03T17:15:28.550ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
137782024-02-03T17:15:28.550Z 1 Acked 1000 Write 9 Done Done Done false
137792024-02-03T17:15:28.550Z 2 Acked 1001 Read 2 Done Done Done false
137802024-02-03T17:15:28.550Z 3 Acked 1002 Write 7 Done Sent Done false
137812024-02-03T17:15:28.550Z 4 Acked 1003 Read 9 Done Sent Done false
137822024-02-03T17:15:28.550Z 5 Acked 1004 Read 7 Done Sent Done false
137832024-02-03T17:15:28.550Z 6 Acked 1005 Read 9 Done Sent Done false
137842024-02-03T17:15:28.551Z 7 Acked 1006 Read 7 Done Sent Done false
137852024-02-03T17:15:28.551Z 8 Acked 1007 Write 2 Done Sent Done false
137862024-02-03T17:15:28.551Z 9 Acked 1008 Read 10 Done Sent Done false
137872024-02-03T17:15:28.551Z 10 Acked 1009 Flush 0 Done Sent Done false
137882024-02-03T17:15:28.551Z 11 Acked 1010 Flush 0 Done Sent Done false
137892024-02-03T17:15:28.551Z 12 Acked 1011 Write 8 Done Sent Done false
137902024-02-03T17:15:28.551Z 13 Acked 1012 Read 1 Done Sent Done false
137912024-02-03T17:15:28.551Z 14 Acked 1013 Write 1 Done Sent Done false
137922024-02-03T17:15:28.551Z 15 Acked 1014 Read 9 Done Sent Done false
137932024-02-03T17:15:28.551Z 16 Acked 1015 Read 9 Done Sent Done false
137942024-02-03T17:15:28.551Z 17 Acked 1016 Write 4 Done Sent Done false
137952024-02-03T17:15:28.551Z 18 Acked 1017 Write 7 Done Sent Done false
137962024-02-03T17:15:28.551Z 19 Acked 1018 Read 10 Done Sent Done false
137972024-02-03T17:15:28.551Z 20 Acked 1019 Read 3 Done Sent Done false
137982024-02-03T17:15:28.551Z 21 Acked 1020 Read 6 Done Sent Done false
137992024-02-03T17:15:28.551Z 22 Acked 1021 Write 5 Done Sent Done false
138002024-02-03T17:15:28.551Z 23 Acked 1022 Write 7 Done Sent Done false
138012024-02-03T17:15:28.551Z 24 Acked 1023 Write 7 Done Sent Done false
138022024-02-03T17:15:28.551Z 25 Acked 1024 Flush 0 Done Sent Done false
138032024-02-03T17:15:28.551Z 26 Acked 1025 Write 2 Done Sent Done false
138042024-02-03T17:15:28.551Z 27 Acked 1026 Read 6 Done Sent Done false
138052024-02-03T17:15:28.551Z 28 Acked 1027 Read 5 Done Sent Done false
138062024-02-03T17:15:28.551Z 29 Acked 1028 Write 8 Sent Sent Sent false
138072024-02-03T17:15:28.551Z 30 Acked 1029 Write 2 Sent Sent Sent false
138082024-02-03T17:15:28.551Z STATES DS:0 DS:1 DS:2 TOTAL
138092024-02-03T17:15:28.552Z New 0 0 0 0
138102024-02-03T17:15:28.552Z Sent 2 28 2 32
138112024-02-03T17:15:28.552Z Done 28 2 28 58
138122024-02-03T17:15:28.552Z Skipped 0 0 0 0
138132024-02-03T17:15:28.552Z Error 0 0 0 0
138142024-02-03T17:15:28.552ZLast Flush: 1024 0 1024
138152024-02-03T17:15:28.552ZDownstairs last five completed:
138162024-02-03T17:15:28.552ZUpstairs last five completed: 30 29 28 27 26
138172024-02-03T17:15:28.552Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:15:28.115613509Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1639}
138182024-02-03T17:15:28.552ZWrote out file "/tmp/test_repair_verify.data"
138192024-02-03T17:15:28.560Z
138202024-02-03T17:15:28.563ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
138212024-02-03T17:15:28.586ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
138222024-02-03T17:15:28.587Z 0 000-019 126 124 126 215 210 215 F F F <---
138232024-02-03T17:15:28.587Z 1 020-039 126 118 126 215 202 215 F F F <---
138242024-02-03T17:15:28.587Z 2 040-059 124 124 124 211 211 211 T F T <---
138252024-02-03T17:15:28.587Z 3 060-079 124 124 124 211 211 211 F F F
138262024-02-03T17:15:28.590Z 4 080-099 124 124 124 210 210 210 F F F
138272024-02-03T17:15:28.590Z 5 100-119 125 125 125 212 212 212 F F F
138282024-02-03T17:15:28.590Z 6 120-139 121 121 121 205 205 205 F F F
138292024-02-03T17:15:28.590Z 7 140-159 124 124 124 210 210 210 F F F
138302024-02-03T17:15:28.590Z 8 160-179 121 121 121 205 205 205 F F F
138312024-02-03T17:15:28.590Z 9 180-199 126 124 126 215 210 215 T F T <---
138322024-02-03T17:15:28.590Z 10 200-219 126 120 126 213 204 213 T T T <---
138332024-02-03T17:15:28.591Z 11 220-239 126 120 126 213 204 213 F T F <---
138342024-02-03T17:15:28.591Z 12 240-259 123 123 123 208 208 208 F F F
138352024-02-03T17:15:28.591Z 13 260-279 122 122 122 206 206 206 F F F
138362024-02-03T17:15:28.591Z 14 280-299 123 123 123 208 208 208 F F F
138372024-02-03T17:15:28.591Z 15 300-319 119 119 119 203 203 203 F F F
138382024-02-03T17:15:28.591Z 16 320-339 124 124 124 210 210 210 F F F
138392024-02-03T17:15:28.591Z 17 340-359 121 121 121 205 205 205 F F F
138402024-02-03T17:15:28.591Z 18 360-379 126 123 126 215 208 215 F F F <---
138412024-02-03T17:15:28.591Z 19 380-399 124 124 124 209 209 209 F F F
138422024-02-03T17:15:28.591Z 20 400-419 124 124 124 209 209 209 F F F
138432024-02-03T17:15:28.591Z 21 420-439 125 125 125 212 212 212 T F T <---
138442024-02-03T17:15:28.591Z 22 440-459 126 125 126 215 212 215 F F F <---
138452024-02-03T17:15:28.591Z 23 460-479 126 124 126 215 211 215 F F F <---
138462024-02-03T17:15:28.591Z 24 480-499 126 123 126 215 208 215 F F F <---
138472024-02-03T17:15:28.591Z 25 500-519 123 123 123 208 208 208 F F F
138482024-02-03T17:15:28.591Z 26 520-539 124 124 124 210 210 210 F F F
138492024-02-03T17:15:28.591Z 27 540-559 124 124 124 211 211 211 F F F
138502024-02-03T17:15:28.591Z 28 560-579 124 124 124 211 211 211 F F F
138512024-02-03T17:15:28.591Z 29 580-599 126 120 126 215 204 215 F F F <---
138522024-02-03T17:15:28.591ZMax gen: 126, Max flush: 215
138532024-02-03T17:15:28.591ZError: Difference in extent metadata found!
138542024-02-03T17:15:28.594ZA Difference in extent metadata is expected here
138552024-02-03T17:15:28.595ZOn loop 63
138562024-02-03T17:15:28.595Z
138572024-02-03T17:15:28.595ZVerifying data now
138582024-02-03T17:15:33.658ZNo change after verify
138592024-02-03T17:15:33.658ZLoop: 63 Downstairs dump after verify (and repair):
138602024-02-03T17:15:33.681ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
138612024-02-03T17:15:33.681Z 0 000-019 126 126 126 215 215 215 F F F
138622024-02-03T17:15:33.681Z 1 020-039 126 126 126 215 215 215 F F F
138632024-02-03T17:15:33.681Z 2 040-059 127 127 127 216 216 216 F F F
138642024-02-03T17:15:33.681Z 3 060-079 124 124 124 211 211 211 F F F
138652024-02-03T17:15:33.684Z 4 080-099 124 124 124 210 210 210 F F F
138662024-02-03T17:15:33.684Z 5 100-119 125 125 125 212 212 212 F F F
138672024-02-03T17:15:33.685Z 6 120-139 121 121 121 205 205 205 F F F
138682024-02-03T17:15:33.685Z 7 140-159 124 124 124 210 210 210 F F F
138692024-02-03T17:15:33.685Z 8 160-179 121 121 121 205 205 205 F F F
138702024-02-03T17:15:33.685Z 9 180-199 127 127 127 216 216 216 F F F
138712024-02-03T17:15:33.685Z 10 200-219 127 127 127 216 216 216 F F F
138722024-02-03T17:15:33.685Z 11 220-239 126 126 126 213 213 213 F F F
138732024-02-03T17:15:33.685Z 12 240-259 123 123 123 208 208 208 F F F
138742024-02-03T17:15:33.685Z 13 260-279 122 122 122 206 206 206 F F F
138752024-02-03T17:15:33.685Z 14 280-299 123 123 123 208 208 208 F F F
138762024-02-03T17:15:33.685Z 15 300-319 119 119 119 203 203 203 F F F
138772024-02-03T17:15:33.685Z 16 320-339 124 124 124 210 210 210 F F F
138782024-02-03T17:15:33.685Z 17 340-359 121 121 121 205 205 205 F F F
138792024-02-03T17:15:33.685Z 18 360-379 126 126 126 215 215 215 F F F
138802024-02-03T17:15:33.685Z 19 380-399 124 124 124 209 209 209 F F F
138812024-02-03T17:15:33.685Z 20 400-419 124 124 124 209 209 209 F F F
138822024-02-03T17:15:33.685Z 21 420-439 127 127 127 216 216 216 F F F
138832024-02-03T17:15:33.685Z 22 440-459 126 126 126 215 215 215 F F F
138842024-02-03T17:15:33.685Z 23 460-479 126 126 126 215 215 215 F F F
138852024-02-03T17:15:33.685Z 24 480-499 126 126 126 215 215 215 F F F
138862024-02-03T17:15:33.685Z 25 500-519 123 123 123 208 208 208 F F F
138872024-02-03T17:15:33.685Z 26 520-539 124 124 124 210 210 210 F F F
138882024-02-03T17:15:33.685Z 27 540-559 124 124 124 211 211 211 F F F
138892024-02-03T17:15:33.685Z 28 560-579 124 124 124 211 211 211 F F F
138902024-02-03T17:15:33.685Z 29 580-599 126 126 126 215 215 215 F F F
138912024-02-03T17:15:33.686ZMax gen: 127, Max flush: 216
138922024-02-03T17:15:33.689Z
138932024-02-03T17:15:33.689ZBegin loop 64
138942024-02-03T17:15:33.689ZDownstairs to restart: 1
138952024-02-03T17:15:33.724Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.283405314Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647}
138962024-02-03T17:15:33.724Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.28400102Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647}
138972024-02-03T17:15:33.724Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.284029637Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647}
138982024-02-03T17:15:33.741Z{"msg":"Crucible a5f41411-c56a-46db-bb4f-e24aaf7c7529 has session id: 6aeb7aac-610a-40c4-aff3-67c6dd0ba999","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.284056477Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
138992024-02-03T17:15:33.741Z{"msg":"Crucible runtime is spawned
139002024-02-03T17:15:33.741ZUpstairs opts: Upstairs UUID: a5f41411-c56a-46db-bb4f-e24aaf7c7529, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.28415923Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139012024-02-03T17:15:33.741Z{"msg":"Crucible stats registered with UUID: a5f41411-c56a-46db-bb4f-e24aaf7c7529","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.28420554Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139022024-02-03T17:15:33.741Z{"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.284280835Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647}
139032024-02-03T17:15:33.741Z{"msg":"a5f41411-c56a-46db-bb4f-e24aaf7c7529 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.284449427Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139042024-02-03T17:15:33.741Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.284518886Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139052024-02-03T17:15:33.741Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.284583314Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139062024-02-03T17:15:33.741Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.284626429Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139072024-02-03T17:15:33.741Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.284651717Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139082024-02-03T17:15:33.741Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.284742936Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139092024-02-03T17:15:33.741Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.284779054Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139102024-02-03T17:15:33.741Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.28481897Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"":"io task","client":"0","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139112024-02-03T17:15:33.741Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.284860211Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"":"io task","client":"1","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139122024-02-03T17:15:33.741Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.284902178Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"":"io task","client":"2","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139132024-02-03T17:15:33.741Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.284992259Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"":"io task","client":"2","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139142024-02-03T17:15:33.741Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.285110633Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"":"io task","client":"1","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139152024-02-03T17:15:33.741Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.285156999Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"":"io task","client":"0","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139162024-02-03T17:15:33.742Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.286244462Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139172024-02-03T17:15:33.742Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.286321869Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139182024-02-03T17:15:33.742Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.286432949Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139192024-02-03T17:15:33.742Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.286564548Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139202024-02-03T17:15:33.742Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.287471715Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139212024-02-03T17:15:33.742Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.287544669Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139222024-02-03T17:15:33.742Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.287578594Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139232024-02-03T17:15:33.742Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.287678532Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139242024-02-03T17:15:33.742Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.288481343Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139252024-02-03T17:15:33.742Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.288570361Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139262024-02-03T17:15:33.742Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.288597742Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139272024-02-03T17:15:33.742Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.288620094Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139282024-02-03T17:15:33.742Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.288685883Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139292024-02-03T17:15:33.742Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.288711553Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139302024-02-03T17:15:33.742Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.288794585Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139312024-02-03T17:15:33.742Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.289515546Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139322024-02-03T17:15:33.742Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.289615363Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139332024-02-03T17:15:33.742Z{"msg":"[0]R flush_numbers[0..12]: [215, 215, 216, 211, 210, 212, 205, 210, 205, 216, 216, 213]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.289749611Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139342024-02-03T17:15:33.742Z{"msg":"[0]R generation[0..12]: [126, 126, 127, 124, 124, 125, 121, 124, 121, 127, 127, 126]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.289777221Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139352024-02-03T17:15:33.742Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.289801713Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139362024-02-03T17:15:33.742Z{"msg":"[1]R flush_numbers[0..12]: [215, 215, 216, 211, 210, 212, 205, 210, 205, 216, 216, 213]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.289825437Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139372024-02-03T17:15:33.742Z{"msg":"[1]R generation[0..12]: [126, 126, 127, 124, 124, 125, 121, 124, 121, 127, 127, 126]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.289848395Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"":"Wait for a query_work_queue command to finish before sending IO
139382024-02-03T17:15:33.742Zdownstairs","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139392024-02-03T17:15:33.743Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.289876519Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139402024-02-03T17:15:33.743Z{"msg":"[2]R flush_numbers[0..12]: [215, 215, 216, 211, 210, 212, 205, 210, 205, 216, 216, 213]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.289899922Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139412024-02-03T17:15:33.743Z{"msg":"[2]R generation[0..12]: [126, 126, 127, 124, 124, 125, 121, 124, 121, 127, 127, 126]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.289923075Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139422024-02-03T17:15:33.743Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.289972485Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139432024-02-03T17:15:33.743Z{"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.289997288Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139442024-02-03T17:15:33.743Z{"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.290020981Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139452024-02-03T17:15:33.743Z{"msg":"Next flush: 217","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.290068276Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139462024-02-03T17:15:33.743Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.290092608Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139472024-02-03T17:15:33.743Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.290114956Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139482024-02-03T17:15:33.743Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.29016145Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139492024-02-03T17:15:33.743Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.290185987Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139502024-02-03T17:15:33.743Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.290233244Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139512024-02-03T17:15:33.743Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.290258616Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139522024-02-03T17:15:33.743Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.29027938Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139532024-02-03T17:15:33.743Z{"msg":"a5f41411-c56a-46db-bb4f-e24aaf7c7529 is now active with session: 6aeb7aac-610a-40c4-aff3-67c6dd0ba999","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.290301314Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139542024-02-03T17:15:33.743Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.290347341Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647,"session_id":"6aeb7aac-610a-40c4-aff3-67c6dd0ba999"}
139552024-02-03T17:15:33.743Z{"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:33.290370155Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647}
139562024-02-03T17:15:33.743ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
139572024-02-03T17:15:33.743ZLoading write count information from file "/tmp/test_repair_verify.data"
139582024-02-03T17:15:33.743ZRun Repair workload
139592024-02-03T17:15:33.743Z01/30 Write block 139 len 16384 data: 10 11 14 13
139602024-02-03T17:15:33.743Z02/30 Read block 103 len 4096
139612024-02-03T17:15:33.743Z03/30 Read block 563 len 40960
139622024-02-03T17:15:33.747Z04/30 Read block 512 len 32768
139632024-02-03T17:15:33.747Z05/30 Read block 414 len 4096
139642024-02-03T17:15:33.747Z06/30 Read block 451 len 24576
139652024-02-03T17:15:33.747Z07/30 Write block 249 len 16384 data: 9 8 8 8
139662024-02-03T17:15:33.747Z08/30 Write block 257 len 28672 data: 11 13 13 16 16 15 14
139672024-02-03T17:15:33.747Z09/30 Write block 330 len 8192 data: 11 11
139682024-02-03T17:15:33.747Z10/30 Write block 494 len 8192 data: 7 7
139692024-02-03T17:15:33.748Z11/30 Read block 258 len 24576
139702024-02-03T17:15:33.748Z12/30 Read block 246 len 4096
139712024-02-03T17:15:33.748Z13/30 Flush
139722024-02-03T17:15:33.748Z14/30 Write block 256 len 36864 data: 11 12 14 14 17 17 16 15 15
139732024-02-03T17:15:33.748Z15/30 Read block 543 len 20480
139742024-02-03T17:15:33.751Z16/30 Read block 367 len 16384
139752024-02-03T17:15:33.751Z17/30 Write block 445 len 16384 data: 11 14 14 13
139762024-02-03T17:15:33.751Z18/30 Flush
139772024-02-03T17:15:33.757Z19/30 Flush
139782024-02-03T17:15:33.760Z20/30 Write block 452 len 20480 data: 6 7 7 6 6
139792024-02-03T17:15:33.760Z21/30 Read block 356 len 12288
139802024-02-03T17:15:33.760Z22/30 Write block 2 len 36864 data: 6 7 8 8 10 9 14 16 15
139812024-02-03T17:15:33.760Z23/30 Flush
139822024-02-03T17:15:33.763Z24/30 Flush
139832024-02-03T17:15:33.766Z25/30 Write block 112 len 4096 data: 9
139842024-02-03T17:15:33.766Z26/30 Read block 70 len 28672
139852024-02-03T17:15:33.766Z27/30 Write block 10 len 12288 data: 16 10 10
139862024-02-03T17:15:33.769Z28/30 Read block 318 len 36864
139872024-02-03T17:15:33.769Z29/30 Write block 78 len 32768 data: 9 10 8 11 9 8 8 8
139882024-02-03T17:15:33.772Z30/30 Write block 419 len 28672 data: 9 9 11 9 7 7 8
139892024-02-03T17:15:33.772Z----------------------------------------------------------------
139902024-02-03T17:15:33.772Z Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:30
139912024-02-03T17:15:33.772ZStates: Active Active Active
139922024-02-03T17:15:33.772ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
139932024-02-03T17:15:33.773Z 1 Acked 1000 Write 4 Done Sent Done false
139942024-02-03T17:15:33.773Z 2 Acked 1001 Read 1 Done Sent Done false
139952024-02-03T17:15:33.773Z 3 Acked 1002 Read 10 Done Sent Done false
139962024-02-03T17:15:33.773Z 4 Acked 1003 Read 8 Done Sent Done false
139972024-02-03T17:15:33.773Z 5 Acked 1004 Read 1 Done Sent Done false
139982024-02-03T17:15:33.773Z 6 Acked 1005 Read 6 Done Sent Done false
139992024-02-03T17:15:33.773Z 7 Acked 1006 Write 4 Done Sent Done false
140002024-02-03T17:15:33.773Z 8 Acked 1007 Write 7 Done Sent Done false
140012024-02-03T17:15:33.773Z 9 Acked 1008 Write 2 Done Sent Done false
140022024-02-03T17:15:33.773Z 10 Acked 1009 Write 2 Done Sent Done false
140032024-02-03T17:15:33.773Z 11 Acked 1010 Read 6 Done Sent Done false
140042024-02-03T17:15:33.773Z 12 Acked 1011 Read 1 Done Sent Done false
140052024-02-03T17:15:33.773Z 13 Acked 1012 Flush 0 Done Sent Done false
140062024-02-03T17:15:33.773Z 14 Acked 1013 Write 9 Done Sent Done false
140072024-02-03T17:15:33.773Z 15 Acked 1014 Read 5 Done Sent Done false
140082024-02-03T17:15:33.773Z 16 Acked 1015 Read 4 Done Sent Done false
140092024-02-03T17:15:33.773Z 17 Acked 1016 Write 4 Done Sent Done false
140102024-02-03T17:15:33.773Z 18 Acked 1017 Flush 0 Done Sent Done false
140112024-02-03T17:15:33.773Z 19 Acked 1018 Flush 0 Done Sent Done false
140122024-02-03T17:15:33.773Z 20 Acked 1019 Write 5 Done Sent Done false
140132024-02-03T17:15:33.773Z 21 Acked 1020 Read 3 Done Sent Done false
140142024-02-03T17:15:33.773Z 22 Acked 1021 Write 9 Done Sent Done false
140152024-02-03T17:15:33.773Z 23 Acked 1022 Flush 0 Done Sent Done false
140162024-02-03T17:15:33.773Z 24 Acked 1023 Flush 0 Done Sent Done false
140172024-02-03T17:15:33.773Z 25 Acked 1024 Write 1 Done Sent Done false
140182024-02-03T17:15:33.774Z 26 Acked 1025 Read 7 Done Sent Done false
140192024-02-03T17:15:33.774Z 27 Acked 1026 Write 3 Done Sent Done false
140202024-02-03T17:15:33.774Z 28 Acked 1027 Read 9 Done Sent Done false
140212024-02-03T17:15:33.774Z 29 Acked 1028 Write 8 Done Sent Done false
140222024-02-03T17:15:33.774Z 30 Acked 1029 Write 7 Sent Sent Sent false
140232024-02-03T17:15:33.774Z STATES DS:0 DS:1 DS:2 TOTAL
140242024-02-03T17:15:33.774Z New 0 0 0 0
140252024-02-03T17:15:33.774Z Sent 1 30 1 32
140262024-02-03T17:15:33.774Z Done 29 0 29 58
140272024-02-03T17:15:33.774Z Skipped 0 0 0 0
140282024-02-03T17:15:33.774Z Error 0 0 0 0
140292024-02-03T17:15:33.774ZLast Flush: 1023 0 1023
140302024-02-03T17:15:33.774ZDownstairs last five completed:
140312024-02-03T17:15:33.774ZUpstairs last five completed: 30 29 28 27 26
140322024-02-03T17:15:33.774Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:15:33.338940587Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1647}
140332024-02-03T17:15:33.774ZWrote out file "/tmp/test_repair_verify.data"
140342024-02-03T17:15:33.779Z
140352024-02-03T17:15:33.786ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
140362024-02-03T17:15:33.812ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
140372024-02-03T17:15:33.812Z 0 000-019 128 126 128 220 215 220 T F T <---
140382024-02-03T17:15:33.812Z 1 020-039 126 126 126 215 215 215 F F F
140392024-02-03T17:15:33.812Z 2 040-059 127 127 127 216 216 216 F F F
140402024-02-03T17:15:33.812Z 3 060-079 124 124 124 211 211 211 T F T <---
140412024-02-03T17:15:33.815Z 4 080-099 124 124 124 210 210 210 T F T <---
140422024-02-03T17:15:33.815Z 5 100-119 125 125 125 212 212 212 T F T <---
140432024-02-03T17:15:33.815Z 6 120-139 128 121 128 217 205 217 F F F <---
140442024-02-03T17:15:33.816Z 7 140-159 128 124 128 217 210 217 F F F <---
140452024-02-03T17:15:33.816Z 8 160-179 121 121 121 205 205 205 F F F
140462024-02-03T17:15:33.816Z 9 180-199 127 127 127 216 216 216 F F F
140472024-02-03T17:15:33.816Z 10 200-219 127 127 127 216 216 216 F F F
140482024-02-03T17:15:33.816Z 11 220-239 126 126 126 213 213 213 F F F
140492024-02-03T17:15:33.816Z 12 240-259 128 123 128 218 208 218 F F F <---
140502024-02-03T17:15:33.816Z 13 260-279 128 122 128 218 206 218 F F F <---
140512024-02-03T17:15:33.816Z 14 280-299 123 123 123 208 208 208 F F F
140522024-02-03T17:15:33.816Z 15 300-319 119 119 119 203 203 203 F F F
140532024-02-03T17:15:33.816Z 16 320-339 128 124 128 217 210 217 F F F <---
140542024-02-03T17:15:33.816Z 17 340-359 121 121 121 205 205 205 F F F
140552024-02-03T17:15:33.816Z 18 360-379 126 126 126 215 215 215 F F F
140562024-02-03T17:15:33.816Z 19 380-399 124 124 124 209 209 209 F F F
140572024-02-03T17:15:33.816Z 20 400-419 124 124 124 209 209 209 T F T <---
140582024-02-03T17:15:33.816Z 21 420-439 127 127 127 216 216 216 T F T <---
140592024-02-03T17:15:33.816Z 22 440-459 128 126 128 220 215 220 F F F <---
140602024-02-03T17:15:33.816Z 23 460-479 126 126 126 215 215 215 F F F
140612024-02-03T17:15:33.816Z 24 480-499 128 126 128 217 215 217 F F F <---
140622024-02-03T17:15:33.816Z 25 500-519 123 123 123 208 208 208 F F F
140632024-02-03T17:15:33.816Z 26 520-539 124 124 124 210 210 210 F F F
140642024-02-03T17:15:33.816Z 27 540-559 124 124 124 211 211 211 F F F
140652024-02-03T17:15:33.816Z 28 560-579 124 124 124 211 211 211 F F F
140662024-02-03T17:15:33.816Z 29 580-599 126 126 126 215 215 215 F F F
140672024-02-03T17:15:33.816ZMax gen: 128, Max flush: 220
140682024-02-03T17:15:33.817ZError: Difference in extent metadata found!
140692024-02-03T17:15:33.820ZA Difference in extent metadata is expected here
140702024-02-03T17:15:33.820ZOn loop 64
140712024-02-03T17:15:33.820Z
140722024-02-03T17:15:33.820ZVerifying data now
140732024-02-03T17:15:38.962ZNo change after verify
140742024-02-03T17:15:38.962ZLoop: 64 Downstairs dump after verify (and repair):
140752024-02-03T17:15:38.985ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
140762024-02-03T17:15:38.985Z 0 000-019 129 129 129 221 221 221 F F F
140772024-02-03T17:15:38.985Z 1 020-039 126 126 126 215 215 215 F F F
140782024-02-03T17:15:38.985Z 2 040-059 127 127 127 216 216 216 F F F
140792024-02-03T17:15:38.985Z 3 060-079 129 129 129 221 221 221 F F F
140802024-02-03T17:15:38.985Z 4 080-099 129 129 129 221 221 221 F F F
140812024-02-03T17:15:38.985Z 5 100-119 129 129 129 221 221 221 F F F
140822024-02-03T17:15:38.985Z 6 120-139 128 128 128 217 217 217 F F F
140832024-02-03T17:15:38.985Z 7 140-159 128 128 128 217 217 217 F F F
140842024-02-03T17:15:38.986Z 8 160-179 121 121 121 205 205 205 F F F
140852024-02-03T17:15:38.989Z 9 180-199 127 127 127 216 216 216 F F F
140862024-02-03T17:15:38.989Z 10 200-219 127 127 127 216 216 216 F F F
140872024-02-03T17:15:38.989Z 11 220-239 126 126 126 213 213 213 F F F
140882024-02-03T17:15:38.989Z 12 240-259 128 128 128 218 218 218 F F F
140892024-02-03T17:15:38.990Z 13 260-279 128 128 128 218 218 218 F F F
140902024-02-03T17:15:38.990Z 14 280-299 123 123 123 208 208 208 F F F
140912024-02-03T17:15:38.990Z 15 300-319 119 119 119 203 203 203 F F F
140922024-02-03T17:15:38.990Z 16 320-339 128 128 128 217 217 217 F F F
140932024-02-03T17:15:38.990Z 17 340-359 121 121 121 205 205 205 F F F
140942024-02-03T17:15:38.990Z 18 360-379 126 126 126 215 215 215 F F F
140952024-02-03T17:15:38.990Z 19 380-399 124 124 124 209 209 209 F F F
140962024-02-03T17:15:38.990Z 20 400-419 129 129 129 221 221 221 F F F
140972024-02-03T17:15:38.990Z 21 420-439 129 129 129 221 221 221 F F F
140982024-02-03T17:15:38.990Z 22 440-459 128 128 128 220 220 220 F F F
140992024-02-03T17:15:38.990Z 23 460-479 126 126 126 215 215 215 F F F
141002024-02-03T17:15:38.990Z 24 480-499 128 128 128 217 217 217 F F F
141012024-02-03T17:15:38.990Z 25 500-519 123 123 123 208 208 208 F F F
141022024-02-03T17:15:38.991Z 26 520-539 124 124 124 210 210 210 F F F
141032024-02-03T17:15:38.991Z 27 540-559 124 124 124 211 211 211 F F F
141042024-02-03T17:15:38.991Z 28 560-579 124 124 124 211 211 211 F F F
141052024-02-03T17:15:38.991Z 29 580-599 126 126 126 215 215 215 F F F
141062024-02-03T17:15:38.991ZMax gen: 129, Max flush: 221
141072024-02-03T17:15:38.991Z
141082024-02-03T17:15:38.991ZBegin loop 65
141092024-02-03T17:15:38.991ZDownstairs to restart: 1
141102024-02-03T17:15:39.020Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.586832273Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655}
141112024-02-03T17:15:39.020Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.587305007Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655}
141122024-02-03T17:15:39.024Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.587382024Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655}
141132024-02-03T17:15:39.024Z{"msg":"Crucible d880d946-0e47-49ef-b55e-1f18d3f586f5 has session id: aadb42eb-ac27-44fa-8d99-21f5f9dd27ca","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.587424373Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141142024-02-03T17:15:39.025Z{"msg":"Upstairs opts: Upstairs UUID: d880d946-0e47-49ef-b55e-1f18d3f586f5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.587481219Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141152024-02-03T17:15:39.025Z{"msg":"Crucible stats registered with UUID: d880d946-0e47-49ef-b55e-1f18d3f586f5","v":0,"name":"crucible","level":30Crucible runtime is spawned
141162024-02-03T17:15:39.025Z,"time":"2024-02-03T17:15:38.587594027Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141172024-02-03T17:15:39.025Z{"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.58781528Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655}
141182024-02-03T17:15:39.025Z{"msg":"d880d946-0e47-49ef-b55e-1f18d3f586f5 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.587896723Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141192024-02-03T17:15:39.025Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.587922816Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141202024-02-03T17:15:39.025Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.588100325Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141212024-02-03T17:15:39.025Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.588149319Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141222024-02-03T17:15:39.025Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.588172893Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141232024-02-03T17:15:39.025Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.588194926Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141242024-02-03T17:15:39.025Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.588272601Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141252024-02-03T17:15:39.025Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.588313724Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"":"io task","client":"0","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141262024-02-03T17:15:39.025Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.588357341Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"":"io task","client":"2","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141272024-02-03T17:15:39.025Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.588437137Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"":"io task","client":"1","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141282024-02-03T17:15:39.026Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.588525297Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"":"io task","client":"0","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141292024-02-03T17:15:39.026Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.5886101Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"":"io task","client":"1","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141302024-02-03T17:15:39.026Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.588720364Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"":"io task","client":"2","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141312024-02-03T17:15:39.026Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.589920205Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141322024-02-03T17:15:39.026Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.5900243Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141332024-02-03T17:15:39.026Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.590117297Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141342024-02-03T17:15:39.026Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.590164978Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141352024-02-03T17:15:39.026Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.590575388Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141362024-02-03T17:15:39.026Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.590613942Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141372024-02-03T17:15:39.026Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.591306221Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141382024-02-03T17:15:39.031Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.591443867Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141392024-02-03T17:15:39.031Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.5920977Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141402024-02-03T17:15:39.031Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.592450364Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141412024-02-03T17:15:39.031Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.592518486Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141422024-02-03T17:15:39.032Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.59254491Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141432024-02-03T17:15:39.032Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.592565562Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141442024-02-03T17:15:39.032Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.592616206Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141452024-02-03T17:15:39.032Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.592640758Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141462024-02-03T17:15:39.032Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.592984708Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141472024-02-03T17:15:39.032Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.593056467Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141482024-02-03T17:15:39.032Z{"msg":"[0]R flush_numbers[0..12]: [221, 215, 216, 221, 221, 221, 217, 217, 205, 216, 216, 213]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.593085027Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141492024-02-03T17:15:39.032Z{"msg":"[0]R generation[0..12]: [129, 126, 127, 129, 129, 129, 128, 128, 121, 127, 127, 126]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.593109769Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141502024-02-03T17:15:39.032Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.593159671Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141512024-02-03T17:15:39.032Z{"msg":"[1]R flush_numbers[0..12]: [221, 215, 216, 221, 221, 221, 217, 217, 205, 216, 216, 213]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.593186506Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141522024-02-03T17:15:39.032Z{"msg":"[1]R generation[0..12]: [129, 126, 127, 129, 129, 129, 128, 128, 121, 127, 127, 126]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.593234624Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141532024-02-03T17:15:39.032Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.593261022Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141542024-02-03T17:15:39.032Z{"msg":"[2]R flush_numbers[0..12]: [221, 215, 216, 221, 221, 221, 217, 217, 205, 216, 216, 213]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.593286108Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141552024-02-03T17:15:39.033Z{"msg":"[2]R generation[0..12]: [129, 126, 127, 129, 129, 129, 128, 128, 121, 127, 127, 126]"Wait for a query_work_queue command to finish before sending IO
141562024-02-03T17:15:39.033Z,"v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.593313356Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141572024-02-03T17:15:39.033Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.593337462Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141582024-02-03T17:15:39.033Z{"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.593385289Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141592024-02-03T17:15:39.033Z{"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.593409964Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141602024-02-03T17:15:39.033Z{"msg":"Next flush: 222","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.593431285Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141612024-02-03T17:15:39.033Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.593452632Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141622024-02-03T17:15:39.033Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.593474726Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141632024-02-03T17:15:39.033Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.59353088Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141642024-02-03T17:15:39.033Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.593558171Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141652024-02-03T17:15:39.033Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.593617395Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141662024-02-03T17:15:39.033Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.59364317Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141672024-02-03T17:15:39.033Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.593663987Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141682024-02-03T17:15:39.034Z{"msg":"d880d946-0e47-49ef-b55e-1f18d3f586f5 is now active with session: aadb42eb-ac27-44fa-8d99-21f5f9dd27ca","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.593710922Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141692024-02-03T17:15:39.034Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.59373337Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655,"session_id":"aadb42eb-ac27-44fa-8d99-21f5f9dd27ca"}
141702024-02-03T17:15:39.034Z{"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:38.593753806Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655}
141712024-02-03T17:15:39.034ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
141722024-02-03T17:15:39.034ZLoading write count information from file "/tmp/test_repair_verify.data"
141732024-02-03T17:15:39.034ZRun Repair workload
141742024-02-03T17:15:39.034Z01/30 Flush
141752024-02-03T17:15:39.034Z02/30 Write block 476 len 20480 data: 6 7 9 11 11
141762024-02-03T17:15:39.038Z03/30 Write block 263 len 8192 data: 16 16
141772024-02-03T17:15:39.038Z04/30 Write block 338 len 8192 data: 13 10
141782024-02-03T17:15:39.038Z05/30 Flush
141792024-02-03T17:15:39.038Z06/30 Write block 362 len 36864 data: 8 8 8 8 9 11 11 11 10
141802024-02-03T17:15:39.038Z07/30 Write block 413 len 16384 data: 9 10 11 10
141812024-02-03T17:15:39.042Z08/30 Write block 339 len 16384 data: 11 12 10 10
141822024-02-03T17:15:39.042Z09/30 Write block 194 len 36864 data: 12 13 11 9 8 5 4 5 9
141832024-02-03T17:15:39.042Z10/30 Write block 364 len 8192 data: 9 9
141842024-02-03T17:15:39.046Z11/30 Write block 92 len 16384 data: 10 13 14 12
141852024-02-03T17:15:39.046Z12/30 Flush
141862024-02-03T17:15:39.049Z13/30 Write block 233 len 36864 data: 6 9 9 7 6 5 6 5 7
141872024-02-03T17:15:39.053Z14/30 Write block 43 len 28672 data: 8 8 10 10 10 10 11
141882024-02-03T17:15:39.053Z15/30 Read block 112 len 12288
141892024-02-03T17:15:39.056Z16/30 Write block 152 len 4096 data: 11
141902024-02-03T17:15:39.056Z17/30 Read block 523 len 32768
141912024-02-03T17:15:39.056Z18/30 Flush
141922024-02-03T17:15:39.060Z19/30 Write block 503 len 4096 data: 12
141932024-02-03T17:15:39.061Z20/30 Write block 99 len 12288 data: 10 10 11
141942024-02-03T17:15:39.061Z21/30 Flush
141952024-02-03T17:15:39.065Z22/30 Write block 61 len 36864 data: 8 10 8 6 6 7 7 7 5
141962024-02-03T17:15:39.068Z23/30 Write block 287 len 40960 data: 12 9 11 10 9 8 9 10 10 11
141972024-02-03T17:15:39.068Z24/30 Write block 200 len 20480 data: 5 6 10 9 8
141982024-02-03T17:15:39.068Z25/30 Read block 230 len 28672
141992024-02-03T17:15:39.071Z26/30 Write block 429 len 16384 data: 14 14 11 12
142002024-02-03T17:15:39.075Z27/30 Write block 127 len 40960 data: 8 9 10 10 12 11 9 8 8 8
142012024-02-03T17:15:39.075Z28/30 Write block 549 len 36864 data: 12 14 13 13 15 14 11 12 12
142022024-02-03T17:15:39.075Z29/30 Write block 502 len 36864 data: 10 13 12 12 13 13 13 13 12
142032024-02-03T17:15:39.079Z30/30 Read block 111 len 28672
142042024-02-03T17:15:39.079Z----------------------------------------------------------------
142052024-02-03T17:15:39.079Z Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:29
142062024-02-03T17:15:39.079ZStates: Active Active Active
142072024-02-03T17:15:39.080ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
142082024-02-03T17:15:39.080Z 2 Acked 1001 Write 5 Done Done Done false
142092024-02-03T17:15:39.080Z 3 Acked 1002 Write 2 Done Done Done false
142102024-02-03T17:15:39.080Z 4 Acked 1003 Write 2 Done Done Done false
142112024-02-03T17:15:39.080Z 5 Acked 1004 Flush 0 Done Sent Done false
142122024-02-03T17:15:39.080Z 6 Acked 1005 Write 9 Done Sent Done false
142132024-02-03T17:15:39.080Z 7 Acked 1006 Write 4 Done Sent Done false
142142024-02-03T17:15:39.080Z 8 Acked 1007 Write 4 Done Sent Done false
142152024-02-03T17:15:39.080Z 9 Acked 1008 Write 9 Done Sent Done false
142162024-02-03T17:15:39.080Z 10 Acked 1009 Write 2 Done Sent Done false
142172024-02-03T17:15:39.080Z 11 Acked 1010 Write 4 Done Sent Done false
142182024-02-03T17:15:39.080Z 12 Acked 1011 Flush 0 Done Sent Done false
142192024-02-03T17:15:39.080Z 13 Acked 1012 Write 9 Done Sent Done false
142202024-02-03T17:15:39.081Z 14 Acked 1013 Write 7 Done Sent Done false
142212024-02-03T17:15:39.081Z 15 Acked 1014 Read 3 Done Sent Done false
142222024-02-03T17:15:39.081Z 16 Acked 1015 Write 1 Done Sent Done false
142232024-02-03T17:15:39.081Z 17 Acked 1016 Read 8 Done Sent Done false
142242024-02-03T17:15:39.081Z 18 Acked 1017 Flush 0 Done Sent Done false
142252024-02-03T17:15:39.081Z 19 Acked 1018 Write 1 Done Sent Done false
142262024-02-03T17:15:39.081Z 20 Acked 1019 Write 3 Done Sent Done false
142272024-02-03T17:15:39.081Z 21 Acked 1020 Flush 0 Done Sent Done false
142282024-02-03T17:15:39.081Z 22 Acked 1021 Write 9 Done Sent Done false
142292024-02-03T17:15:39.081Z 23 Acked 1022 Write 10 Done Sent Done false
142302024-02-03T17:15:39.081Z 24 Acked 1023 Write 5 Done Sent Done false
142312024-02-03T17:15:39.081Z 25 Acked 1024 Read 7 Done Sent Done false
142322024-02-03T17:15:39.081Z 26 Acked 1025 Write 4 Done Sent Done false
142332024-02-03T17:15:39.082Z 27 Acked 1026 Write 10 Done Sent Done false
142342024-02-03T17:15:39.082Z 28 Acked 1027 Write 9 Done Sent Done false
142352024-02-03T17:15:39.082Z 29 Acked 1028 Write 9 Done Sent Done false
142362024-02-03T17:15:39.082Z 30 Acked 1029 Read 7 Sent Sent Done false
142372024-02-03T17:15:39.082Z STATES DS:0 DS:1 DS:2 TOTAL
142382024-02-03T17:15:39.082Z New 0 0 0 0
142392024-02-03T17:15:39.082Z Sent 1 26 0 27
142402024-02-03T17:15:39.082Z Done 28 3 29 60
142412024-02-03T17:15:39.082Z Skipped 0 0 0 0
142422024-02-03T17:15:39.082Z Error 0 0 0 0
142432024-02-03T17:15:39.082ZLast Flush: 1020 1000 1020
142442024-02-03T17:15:39.082ZDownstairs last five completed: 1000
142452024-02-03T17:15:39.082ZUpstairs last five completed: 30 29 28 27 26
142462024-02-03T17:15:39.083Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:15:38.646366117Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1655}
142472024-02-03T17:15:39.086ZWrote out file "/tmp/test_repair_verify.data"
142482024-02-03T17:15:39.086Z
142492024-02-03T17:15:39.092ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
142502024-02-03T17:15:39.137ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
142512024-02-03T17:15:39.137Z 0 000-019 129 129 129 221 221 221 F F F
142522024-02-03T17:15:39.145Z 1 020-039 126 126 126 215 215 215 F F F
142532024-02-03T17:15:39.145Z 2 040-059 130 127 130 225 216 225 F F F <---
142542024-02-03T17:15:39.145Z 3 060-079 129 129 129 221 221 221 T F T <---
142552024-02-03T17:15:39.145Z 4 080-099 130 129 130 226 221 226 F F F <---
142562024-02-03T17:15:39.146Z 5 100-119 130 129 130 226 221 226 F F F <---
142572024-02-03T17:15:39.146Z 6 120-139 128 128 128 217 217 217 T F T <---
142582024-02-03T17:15:39.146Z 7 140-159 130 128 130 225 217 225 F F F <---
142592024-02-03T17:15:39.146Z 8 160-179 121 121 121 205 205 205 F F F
142602024-02-03T17:15:39.146Z 9 180-199 130 127 130 224 216 224 F F F <---
142612024-02-03T17:15:39.146Z 10 200-219 130 127 130 224 216 224 T F T <---
142622024-02-03T17:15:39.146Z 11 220-239 130 126 130 225 213 225 F F F <---
142632024-02-03T17:15:39.146Z 12 240-259 130 128 130 225 218 225 F F F <---
142642024-02-03T17:15:39.146Z 13 260-279 130 128 130 223 218 223 F T F <---
142652024-02-03T17:15:39.146Z 14 280-299 123 123 123 208 208 208 T F T <---
142662024-02-03T17:15:39.146Z 15 300-319 119 119 119 203 203 203 F F F
142672024-02-03T17:15:39.146Z 16 320-339 130 128 130 224 217 224 F T F <---
142682024-02-03T17:15:39.146Z 17 340-359 130 121 130 224 205 224 F F F <---
142692024-02-03T17:15:39.146Z 18 360-379 130 126 130 224 215 224 F F F <---
142702024-02-03T17:15:39.146Z 19 380-399 124 124 124 209 209 209 F F F
142712024-02-03T17:15:39.146Z 20 400-419 130 129 130 224 221 224 F F F <---
142722024-02-03T17:15:39.146Z 21 420-439 129 129 129 221 221 221 T F T <---
142732024-02-03T17:15:39.146Z 22 440-459 128 128 128 220 220 220 F F F
142742024-02-03T17:15:39.146Z 23 460-479 130 126 130 223 215 223 F T F <---
142752024-02-03T17:15:39.146Z 24 480-499 130 128 130 223 217 223 F T F <---
142762024-02-03T17:15:39.146Z 25 500-519 130 123 130 226 208 226 T F T <---
142772024-02-03T17:15:39.146Z 26 520-539 124 124 124 210 210 210 F F F
142782024-02-03T17:15:39.146Z 27 540-559 124 124 124 211 211 211 T F T <---
142792024-02-03T17:15:39.147Z 28 560-579 124 124 124 211 211 211 F F F
142802024-02-03T17:15:39.147Z 29 580-599 126 126 126 215 215 215 F F F
142812024-02-03T17:15:39.147ZMax gen: 130, Max flush: 226
142822024-02-03T17:15:39.147ZError: Difference in extent metadata found!
142832024-02-03T17:15:39.147ZA Difference in extent metadata is expected here
142842024-02-03T17:15:39.147ZOn loop 65
142852024-02-03T17:15:39.147Z
142862024-02-03T17:15:39.147ZVerifying data now
142872024-02-03T17:15:45.846ZNo change after verify
142882024-02-03T17:15:45.847ZLoop: 65 Downstairs dump after verify (and repair):
142892024-02-03T17:15:45.864ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
142902024-02-03T17:15:45.864Z 0 000-019 129 129 129 221 221 221 F F F
142912024-02-03T17:15:45.864Z 1 020-039 126 126 126 215 215 215 F F F
142922024-02-03T17:15:45.864Z 2 040-059 130 130 130 225 225 225 F F F
142932024-02-03T17:15:45.875Z 3 060-079 131 131 131 227 227 227 F F F
142942024-02-03T17:15:45.875Z 4 080-099 130 130 130 226 226 226 F F F
142952024-02-03T17:15:45.875Z 5 100-119 130 130 130 226 226 226 F F F
142962024-02-03T17:15:45.875Z 6 120-139 131 131 131 227 227 227 F F F
142972024-02-03T17:15:45.876Z 7 140-159 130 130 130 225 225 225 F F F
142982024-02-03T17:15:45.876Z 8 160-179 121 121 121 205 205 205 F F F
142992024-02-03T17:15:45.876Z 9 180-199 130 130 130 224 224 224 F F F
143002024-02-03T17:15:45.877Z 10 200-219 131 131 131 227 227 227 F F F
143012024-02-03T17:15:45.877Z 11 220-239 130 130 130 225 225 225 F F F
143022024-02-03T17:15:45.877Z 12 240-259 130 130 130 225 225 225 F F F
143032024-02-03T17:15:45.877Z 13 260-279 130 130 130 223 223 223 F F F
143042024-02-03T17:15:45.877Z 14 280-299 131 131 131 227 227 227 F F F
143052024-02-03T17:15:45.878Z 15 300-319 119 119 119 203 203 203 F F F
143062024-02-03T17:15:45.878Z 16 320-339 130 130 130 224 224 224 F F F
143072024-02-03T17:15:45.878Z 17 340-359 130 130 130 224 224 224 F F F
143082024-02-03T17:15:45.878Z 18 360-379 130 130 130 224 224 224 F F F
143092024-02-03T17:15:45.878Z 19 380-399 124 124 124 209 209 209 F F F
143102024-02-03T17:15:45.878Z 20 400-419 130 130 130 224 224 224 F F F
143112024-02-03T17:15:45.878Z 21 420-439 131 131 131 227 227 227 F F F
143122024-02-03T17:15:45.879Z 22 440-459 128 128 128 220 220 220 F F F
143132024-02-03T17:15:45.879Z 23 460-479 130 130 130 223 223 223 F F F
143142024-02-03T17:15:45.879Z 24 480-499 130 130 130 223 223 223 F F F
143152024-02-03T17:15:45.879Z 25 500-519 131 131 131 227 227 227 F F F
143162024-02-03T17:15:45.879Z 26 520-539 124 124 124 210 210 210 F F F
143172024-02-03T17:15:45.880Z 27 540-559 131 131 131 227 227 227 F F F
143182024-02-03T17:15:45.880Z 28 560-579 124 124 124 211 211 211 F F F
143192024-02-03T17:15:45.880Z 29 580-599 126 126 126 215 215 215 F F F
143202024-02-03T17:15:45.880ZMax gen: 131, Max flush: 227
143212024-02-03T17:15:45.880Z
143222024-02-03T17:15:45.880ZBegin loop 66
143232024-02-03T17:15:45.880ZDownstairs to restart: 0
143242024-02-03T17:15:45.904Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.463166036Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663}
143252024-02-03T17:15:45.904Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.463633595Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663}
143262024-02-03T17:15:45.945Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.463721168Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663}
143272024-02-03T17:15:45.946Z{"msg":"Crucible 4267ff42-11bd-4fb9-9af6-69f671caa4cb has session id: 4ed8d3a1-b5d5-446c-a364-f426a67291aa","v":0,"name":"crucible","level":30Crucible runtime is spawned
143282024-02-03T17:15:45.946Z,"time":"2024-02-03T17:15:45.463762018Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143292024-02-03T17:15:45.946Z{"msg":"Upstairs opts: Upstairs UUID: 4267ff42-11bd-4fb9-9af6-69f671caa4cb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.463979109Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143302024-02-03T17:15:45.946Z{"msg":"Crucible stats registered with UUID: 4267ff42-11bd-4fb9-9af6-69f671caa4cb","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.464009343Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143312024-02-03T17:15:45.946Z{"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.464068844Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663}
143322024-02-03T17:15:45.946Z{"msg":"4267ff42-11bd-4fb9-9af6-69f671caa4cb active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.464109382Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143332024-02-03T17:15:45.946Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.464160409Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143342024-02-03T17:15:45.946Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.464186377Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143352024-02-03T17:15:45.947Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.464243873Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143362024-02-03T17:15:45.947Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.46427072Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143372024-02-03T17:15:45.947Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.464303001Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143382024-02-03T17:15:45.947Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.464443652Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143392024-02-03T17:15:45.947Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.464492484Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"":"io task","client":"0","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143402024-02-03T17:15:45.947Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.464630782Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"":"io task","client":"2","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143412024-02-03T17:15:45.947Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.464719674Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"":"io task","client":"1","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143422024-02-03T17:15:45.948Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.464765451Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"":"io task","client":"0","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143432024-02-03T17:15:45.948Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.464850981Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"":"io task","client":"2","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143442024-02-03T17:15:45.948Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.464896189Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"":"io task","client":"1","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143452024-02-03T17:15:45.948Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.466072787Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143462024-02-03T17:15:45.948Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.466202852Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143472024-02-03T17:15:45.948Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.466331421Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143482024-02-03T17:15:45.948Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.466379815Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143492024-02-03T17:15:45.949Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.466976729Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143502024-02-03T17:15:45.949Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.467240858Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143512024-02-03T17:15:45.949Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.46749003Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143522024-02-03T17:15:45.949Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.467554043Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143532024-02-03T17:15:45.949Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.467744881Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143542024-02-03T17:15:45.949Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.467811354Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143552024-02-03T17:15:45.949Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.467837546Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143562024-02-03T17:15:45.949Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.468069661Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143572024-02-03T17:15:45.949Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.468151212Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143582024-02-03T17:15:45.949Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.468179824Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143592024-02-03T17:15:45.950Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.468462696Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143602024-02-03T17:15:45.950Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.469176856Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143612024-02-03T17:15:45.950Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.469241131Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143622024-02-03T17:15:45.950Z{"msg":"[0]R flush_numbers[0..12]: [221, 215, 225, 227, 226, 226, 227, 225, 205, 224, 227, 225]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.469268283Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143632024-02-03T17:15:45.950Z{"msg":"[0]R generation[0..12]: [129, 126, 130, 131, 130, 130, 131, 130, 121, 130, 131, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.469291998Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143642024-02-03T17:15:45.950Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.469362699Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143652024-02-03T17:15:45.950Z{"msg":"[1]R flush_numbers[0..12]: [221, 215, 225, 227, 226, 226, 227, 225, 205, 224, 227, 225]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.469390966Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143662024-02-03T17:15:45.951Z{"msg":"[1]R generation[0..12]: [129, 126, 130, 131, 130, 130, 131, 130, 121, 130, 131, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.469440806Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143672024-02-03T17:15:45.951Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.469467308Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143682024-02-03T17:15:45.951Z{"msg":"[2]R flush_numbers[0..12]: [221, 215, 225, 227, 226, 226, 227, 225, 205, 224, 227, 225]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.469535651Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143692024-02-03T17:15:45.951Z{"msg":"[2]R generation[0..12]: [129, 126, 130, 131, 130, 130, 131, 130, 121, 130, 131, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.469561656Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143702024-02-03T17:15:45.951Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
143712024-02-03T17:15:45.951Z,"time":"2024-02-03T17:15:45.469585559Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143722024-02-03T17:15:45.952Z{"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.469755354Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143732024-02-03T17:15:45.952Z{"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.469784451Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143742024-02-03T17:15:45.952Z{"msg":"Next flush: 228","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.469841256Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143752024-02-03T17:15:45.952Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.469866459Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143762024-02-03T17:15:45.952Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.469925381Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143772024-02-03T17:15:45.952Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.469957065Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143782024-02-03T17:15:45.953Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.469980355Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143792024-02-03T17:15:45.953Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.470041806Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143802024-02-03T17:15:45.953Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.470070528Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143812024-02-03T17:15:45.953Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.470091821Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143822024-02-03T17:15:45.953Z{"msg":"4267ff42-11bd-4fb9-9af6-69f671caa4cb is now active with session: 4ed8d3a1-b5d5-446c-a364-f426a67291aa","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.470139611Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143832024-02-03T17:15:45.953Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.470162014Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663,"session_id":"4ed8d3a1-b5d5-446c-a364-f426a67291aa"}
143842024-02-03T17:15:45.954Z{"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:45.470228107Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663}
143852024-02-03T17:15:45.954ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
143862024-02-03T17:15:45.954ZLoading write count information from file "/tmp/test_repair_verify.data"
143872024-02-03T17:15:45.954ZRun Repair workload
143882024-02-03T17:15:45.955Z01/30 Write block 175 len 32768 data: 9 8 11 9 9 9 12 11
143892024-02-03T17:15:45.955Z02/30 Write block 588 len 36864 data: 12 10 11 12 12 12 10 8 8
143902024-02-03T17:15:45.955Z03/30 Write block 445 len 4096 data: 12
143912024-02-03T17:15:45.955Z04/30 Read block 134 len 28672
143922024-02-03T17:15:45.955Z05/30 Write block 196 len 24576 data: 12 10 9 6 6 7
143932024-02-03T17:15:45.955Z06/30 Read block 184 len 24576
143942024-02-03T17:15:45.956Z07/30 Flush
143952024-02-03T17:15:45.978Z08/30 Write block 370 len 32768 data: 11 7 10 11 11 9 11 13
143962024-02-03T17:15:45.979Z09/30 Read block 500 len 16384
143972024-02-03T17:15:45.979Z10/30 Read block 177 len 20480
143982024-02-03T17:15:45.979Z11/30 Read block 291 len 24576
143992024-02-03T17:15:45.979Z12/30 Read block 294 len 8192
144002024-02-03T17:15:45.979Z13/30 Read block 234 len 12288
144012024-02-03T17:15:45.979Z14/30 Read block 235 len 36864
144022024-02-03T17:15:45.979Z15/30 Write block 35 len 8192 data: 9 11
144032024-02-03T17:15:45.980Z16/30 Write block 152 len 4096 data: 12
144042024-02-03T17:15:45.980Z17/30 Write block 159 len 28672 data: 10 13 13 15 17 17 16
144052024-02-03T17:15:45.980Z18/30 Flush
144062024-02-03T17:15:45.980Z19/30 Write block 82 len 12288 data: 10 9 9
144072024-02-03T17:15:45.980Z20/30 Read block 387 len 32768
144082024-02-03T17:15:45.980Z21/30 Read block 475 len 40960
144092024-02-03T17:15:45.981Z22/30 Flush
144102024-02-03T17:15:45.981Z23/30 Write block 28 len 16384 data: 11 9 9 13
144112024-02-03T17:15:45.981Z24/30 Read block 51 len 28672
144122024-02-03T17:15:45.981Z25/30 Write block 167 len 16384 data: 18 15 15 15
144132024-02-03T17:15:45.981Z26/30 Read block 161 len 16384
144142024-02-03T17:15:45.982Z27/30 Write block 225 len 40960 data: 4 5 6 9 6 6 7 8 7 10
144152024-02-03T17:15:45.982Z28/30 Read block 258 len 12288
144162024-02-03T17:15:45.982Z29/30 Write block 435 len 28672 data: 12 10 11 12 12 12 13
144172024-02-03T17:15:45.982Z30/30 Write block 177 len 24576 data: 12 10 10 10 13 12
144182024-02-03T17:15:45.982Z----------------------------------------------------------------
144192024-02-03T17:15:45.983Z Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:30
144202024-02-03T17:15:45.983ZStates: Active Active Active
144212024-02-03T17:15:45.983ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
144222024-02-03T17:15:45.984Z 1 Acked 1000 Write 8 Done Done Done false
144232024-02-03T17:15:45.984Z 2 Acked 1001 Write 9 Done Done Done false
144242024-02-03T17:15:45.984Z 3 Acked 1002 Write 1 Done Done Done false
144252024-02-03T17:15:45.985Z 4 Acked 1003 Read 7 Done Done Done false
144262024-02-03T17:15:45.985Z 5 Acked 1004 Write 6 Sent Done Done false
144272024-02-03T17:15:45.985Z 6 Acked 1005 Read 6 Sent Done Done false
144282024-02-03T17:15:45.985Z 7 Acked 1006 Flush 0 Sent Done Done false
144292024-02-03T17:15:45.986Z 8 Acked 1007 Write 8 Sent Done Done false
144302024-02-03T17:15:45.986Z 9 Acked 1008 Read 4 Sent Done Done false
144312024-02-03T17:15:45.986Z 10 Acked 1009 Read 5 Sent Done Done false
144322024-02-03T17:15:45.987Z 11 Acked 1010 Read 6 Sent Done Done false
144332024-02-03T17:15:45.987Z 12 Acked 1011 Read 2 Sent Done Done false
144342024-02-03T17:15:45.987Z 13 Acked 1012 Read 3 Sent Done Done false
144352024-02-03T17:15:45.988Z 14 Acked 1013 Read 9 Sent Done Done false
144362024-02-03T17:15:45.988Z 15 Acked 1014 Write 2 Sent Done Done false
144372024-02-03T17:15:45.988Z 16 Acked 1015 Write 1 Sent Done Done false
144382024-02-03T17:15:45.989Z 17 Acked 1016 Write 7 Sent Done Done false
144392024-02-03T17:15:45.989Z 18 Acked 1017 Flush 0 Sent Done Done false
144402024-02-03T17:15:45.989Z 19 Acked 1018 Write 3 Sent Done Done false
144412024-02-03T17:15:45.989Z 20 Acked 1019 Read 8 Sent Done Done false
144422024-02-03T17:15:45.990Z 21 Acked 1020 Read 10 Sent Done Done false
144432024-02-03T17:15:45.990Z 22 Acked 1021 Flush 0 Sent Done Done false
144442024-02-03T17:15:45.991Z 23 Acked 1022 Write 4 Sent Done Done false
144452024-02-03T17:15:45.991Z 24 Acked 1023 Read 7 Sent Done Done false
144462024-02-03T17:15:45.991Z 25 Acked 1024 Write 4 Sent Done Done false
144472024-02-03T17:15:45.991Z 26 Acked 1025 Read 4 Sent Done Done false
144482024-02-03T17:15:45.992Z 27 Acked 1026 Write 10 Sent Done Done false
144492024-02-03T17:15:45.992Z 28 Acked 1027 Read 3 Sent Done Done false
144502024-02-03T17:15:45.992Z 29 Acked 1028 Write 7 Sent Sent Done false
144512024-02-03T17:15:45.992Z 30 Acked 1029 Write 6 Sent Sent Sent false
144522024-02-03T17:15:45.992Z STATES DS:0 DS:1 DS:2 TOTAL
144532024-02-03T17:15:45.993Z New 0 0 0 0
144542024-02-03T17:15:45.993Z Sent 26 2 1 29
144552024-02-03T17:15:45.993Z Done 4 28 29 61
144562024-02-03T17:15:45.993Z Skipped 0 0 0 0
144572024-02-03T17:15:45.994Z Error 0 0 0 0
144582024-02-03T17:15:45.994ZLast Flush: 0 1021 1021
144592024-02-03T17:15:45.994ZDownstairs last five completed:
144602024-02-03T17:15:45.994ZUpstairs last five completed: 30 29 28 27 26
144612024-02-03T17:15:45.994Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:15:45.52254306Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1663}
144622024-02-03T17:15:45.994ZWrote out file "/tmp/test_repair_verify.data"
144632024-02-03T17:15:45.994Z
144642024-02-03T17:15:46.007ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
144652024-02-03T17:15:46.007ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
144662024-02-03T17:15:46.007Z 0 000-019 129 129 129 221 221 221 F F F
144672024-02-03T17:15:46.007Z 1 020-039 126 132 132 215 229 229 F T T <---
144682024-02-03T17:15:46.008Z 2 040-059 130 130 130 225 225 225 F F F
144692024-02-03T17:15:46.008Z 3 060-079 131 131 131 227 227 227 F F F
144702024-02-03T17:15:46.008Z 4 080-099 130 132 132 226 230 230 F F F <---
144712024-02-03T17:15:46.008Z 5 100-119 130 130 130 226 226 226 F F F
144722024-02-03T17:15:46.009Z 6 120-139 131 131 131 227 227 227 F F F
144732024-02-03T17:15:46.009Z 7 140-159 130 132 132 225 229 229 F F F <---
144742024-02-03T17:15:46.009Z 8 160-179 121 132 132 205 229 229 T T T <---
144752024-02-03T17:15:46.009Z 9 180-199 130 132 132 224 228 228 T T T <---
144762024-02-03T17:15:46.009Z 10 200-219 131 132 132 227 228 228 F F F <---
144772024-02-03T17:15:46.010Z 11 220-239 130 130 130 225 225 225 F T T <---
144782024-02-03T17:15:46.010Z 12 240-259 130 130 130 225 225 225 F F F
144792024-02-03T17:15:46.010Z 13 260-279 130 130 130 223 223 223 F F F
144802024-02-03T17:15:46.010Z 14 280-299 131 131 131 227 227 227 F F F
144812024-02-03T17:15:46.010Z 15 300-319 119 119 119 203 203 203 F F F
144822024-02-03T17:15:46.010Z 16 320-339 130 130 130 224 224 224 F F F
144832024-02-03T17:15:46.011Z 17 340-359 130 130 130 224 224 224 F F F
144842024-02-03T17:15:46.011Z 18 360-379 130 132 132 224 229 229 F F F <---
144852024-02-03T17:15:46.011Z 19 380-399 124 124 124 209 209 209 F F F
144862024-02-03T17:15:46.011Z 20 400-419 130 130 130 224 224 224 F F F
144872024-02-03T17:15:46.011Z 21 420-439 131 131 131 227 227 227 F T T <---
144882024-02-03T17:15:46.011Z 22 440-459 128 132 132 220 228 228 T T T <---
144892024-02-03T17:15:46.011Z 23 460-479 130 130 130 223 223 223 F F F
144902024-02-03T17:15:46.011Z 24 480-499 130 130 130 223 223 223 F F F
144912024-02-03T17:15:46.011Z 25 500-519 131 131 131 227 227 227 F F F
144922024-02-03T17:15:46.011Z 26 520-539 124 124 124 210 210 210 F F F
144932024-02-03T17:15:46.011Z 27 540-559 131 131 131 227 227 227 F F F
144942024-02-03T17:15:46.011Z 28 560-579 124 124 124 211 211 211 F F F
144952024-02-03T17:15:46.012Z 29 580-599 126 132 132 215 228 228 T F F <---
144962024-02-03T17:15:46.012ZMax gen: 132, Max flush: 230
144972024-02-03T17:15:46.012ZError: Difference in extent metadata found!
144982024-02-03T17:15:46.012ZA Difference in extent metadata is expected here
144992024-02-03T17:15:46.012ZOn loop 66
145002024-02-03T17:15:46.012Z
145012024-02-03T17:15:46.027ZVerifying data now
145022024-02-03T17:15:51.990ZNo change after verify
145032024-02-03T17:15:51.991ZLoop: 66 Downstairs dump after verify (and repair):
145042024-02-03T17:15:52.013ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
145052024-02-03T17:15:52.014Z 0 000-019 129 129 129 221 221 221 F F F
145062024-02-03T17:15:52.017Z 1 020-039 133 133 133 231 231 231 F F F
145072024-02-03T17:15:52.017Z 2 040-059 130 130 130 225 225 225 F F F
145082024-02-03T17:15:52.017Z 3 060-079 131 131 131 227 227 227 F F F
145092024-02-03T17:15:52.018Z 4 080-099 132 132 132 230 230 230 F F F
145102024-02-03T17:15:52.018Z 5 100-119 130 130 130 226 226 226 F F F
145112024-02-03T17:15:52.018Z 6 120-139 131 131 131 227 227 227 F F F
145122024-02-03T17:15:52.018Z 7 140-159 132 132 132 229 229 229 F F F
145132024-02-03T17:15:52.018Z 8 160-179 133 133 133 231 231 231 F F F
145142024-02-03T17:15:52.018Z 9 180-199 133 133 133 231 231 231 F F F
145152024-02-03T17:15:52.018Z 10 200-219 132 132 132 228 228 228 F F F
145162024-02-03T17:15:52.018Z 11 220-239 133 133 133 231 231 231 F F F
145172024-02-03T17:15:52.018Z 12 240-259 130 130 130 225 225 225 F F F
145182024-02-03T17:15:52.018Z 13 260-279 130 130 130 223 223 223 F F F
145192024-02-03T17:15:52.018Z 14 280-299 131 131 131 227 227 227 F F F
145202024-02-03T17:15:52.018Z 15 300-319 119 119 119 203 203 203 F F F
145212024-02-03T17:15:52.018Z 16 320-339 130 130 130 224 224 224 F F F
145222024-02-03T17:15:52.018Z 17 340-359 130 130 130 224 224 224 F F F
145232024-02-03T17:15:52.019Z 18 360-379 132 132 132 229 229 229 F F F
145242024-02-03T17:15:52.019Z 19 380-399 124 124 124 209 209 209 F F F
145252024-02-03T17:15:52.019Z 20 400-419 130 130 130 224 224 224 F F F
145262024-02-03T17:15:52.019Z 21 420-439 133 133 133 231 231 231 F F F
145272024-02-03T17:15:52.019Z 22 440-459 133 133 133 231 231 231 F F F
145282024-02-03T17:15:52.019Z 23 460-479 130 130 130 223 223 223 F F F
145292024-02-03T17:15:52.019Z 24 480-499 130 130 130 223 223 223 F F F
145302024-02-03T17:15:52.019Z 25 500-519 131 131 131 227 227 227 F F F
145312024-02-03T17:15:52.019Z 26 520-539 124 124 124 210 210 210 F F F
145322024-02-03T17:15:52.019Z 27 540-559 131 131 131 227 227 227 F F F
145332024-02-03T17:15:52.019Z 28 560-579 124 124 124 211 211 211 F F F
145342024-02-03T17:15:52.019Z 29 580-599 132 132 132 228 228 228 F F F
145352024-02-03T17:15:52.019ZMax gen: 133, Max flush: 231
145362024-02-03T17:15:52.024Z
145372024-02-03T17:15:52.024ZBegin loop 67
145382024-02-03T17:15:52.024ZDownstairs to restart: 1
145392024-02-03T17:15:52.049Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.617503785Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671}
145402024-02-03T17:15:52.055Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.618008712Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671}
145412024-02-03T17:15:52.055Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.618091148Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671}
145422024-02-03T17:15:52.055Z{Crucible runtime is spawned
145432024-02-03T17:15:52.055Z"msg":"Crucible 599d8439-27f8-4b60-96cf-1a3b542201e8 has session id: 2600de68-ddbb-4632-b331-736728738f8a","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.618362732Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145442024-02-03T17:15:52.055Z{"msg":"Upstairs opts: Upstairs UUID: 599d8439-27f8-4b60-96cf-1a3b542201e8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.618458234Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145452024-02-03T17:15:52.055Z{"msg":"Crucible stats registered with UUID: 599d8439-27f8-4b60-96cf-1a3b542201e8","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.618506036Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145462024-02-03T17:15:52.055Z{"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.618578969Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671}
145472024-02-03T17:15:52.055Z{"msg":"599d8439-27f8-4b60-96cf-1a3b542201e8 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.618614128Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145482024-02-03T17:15:52.055Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.618672418Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145492024-02-03T17:15:52.055Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.618700835Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145502024-02-03T17:15:52.055Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.618783368Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145512024-02-03T17:15:52.055Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.618831902Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145522024-02-03T17:15:52.056Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.618870406Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145532024-02-03T17:15:52.056Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.61891387Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145542024-02-03T17:15:52.056Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.619225477Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"":"io task","client":"2","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145552024-02-03T17:15:52.056Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.619328426Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"":"io task","client":"0","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145562024-02-03T17:15:52.056Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.619392821Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"":"io task","client":"1","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145572024-02-03T17:15:52.056Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.619437309Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"":"io task","client":"1","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145582024-02-03T17:15:52.056Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.619515458Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"":"io task","client":"2","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145592024-02-03T17:15:52.056Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.61956254Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"":"io task","client":"0","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145602024-02-03T17:15:52.056Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.620811023Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145612024-02-03T17:15:52.056Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.620887649Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145622024-02-03T17:15:52.056Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.621002665Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145632024-02-03T17:15:52.056Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.621106687Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145642024-02-03T17:15:52.056Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.621318899Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145652024-02-03T17:15:52.056Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.621474296Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145662024-02-03T17:15:52.074Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.622004376Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145672024-02-03T17:15:52.075Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.622264848Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145682024-02-03T17:15:52.075Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.6224382Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145692024-02-03T17:15:52.075Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.622764257Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145702024-02-03T17:15:52.075Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.622903769Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145712024-02-03T17:15:52.075Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.622962536Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145722024-02-03T17:15:52.075Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.622991673Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145732024-02-03T17:15:52.075Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.62302035Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145742024-02-03T17:15:52.075Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.623132873Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145752024-02-03T17:15:52.075Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.623433937Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145762024-02-03T17:15:52.075Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.623519314Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145772024-02-03T17:15:52.076Z{"msg":"[0]R flush_numbers[0..12]: [221, 231, 225, 227, 230, 226, 227, 229, 231, 231, 228, 231]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.623569216Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145782024-02-03T17:15:52.076Z{"msg":"[0]R generation[0..12]: [129, 133, 130, 131, 132, 130, 131, 132, 133, 133, 132, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.623630602Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145792024-02-03T17:15:52.076Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.623658802Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145802024-02-03T17:15:52.076Z{"msg":"[1]R flush_numbers[0..12]: [221, 231, 225, 227, 230, 226, 227, 229, 231, 231, 228, 231]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.623684482Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145812024-02-03T17:15:52.076Z{"msg":"[1]R generation[0..12]: [129, 133, 130, 131, 132, 130, 131, 132, 133, 133, 132, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.623708078Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145822024-02-03T17:15:52.076Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.623733185Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145832024-02-03T17:15:52.076Z{"msg":"[2]R flush_numbers[0..12]: [221, 231, 225, 227, 230, 226, 227, 229, 231, 231, 228, 231]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.623757041Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145842024-02-03T17:15:52.076Z{Wait for a query_work_queue command to finish before sending IO
145852024-02-03T17:15:52.076Z"msg":"[2]R generation[0..12]: [129, 133, 130, 131, 132, 130, 131, 132, 133, 133, 132, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.623785757Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145862024-02-03T17:15:52.076Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.623809955Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145872024-02-03T17:15:52.076Z{"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.623857693Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145882024-02-03T17:15:52.076Z{"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.623882687Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145892024-02-03T17:15:52.077Z{"msg":"Next flush: 232","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.623904153Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145902024-02-03T17:15:52.077Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.623925935Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145912024-02-03T17:15:52.077Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.623995066Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145922024-02-03T17:15:52.077Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.624017839Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145932024-02-03T17:15:52.077Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.624039773Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145942024-02-03T17:15:52.077Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.624061805Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145952024-02-03T17:15:52.077Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.624083892Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145962024-02-03T17:15:52.077Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.624104474Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145972024-02-03T17:15:52.077Z{"msg":"599d8439-27f8-4b60-96cf-1a3b542201e8 is now active with session: 2600de68-ddbb-4632-b331-736728738f8a","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.624126193Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145982024-02-03T17:15:52.077Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.624174502Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671,"session_id":"2600de68-ddbb-4632-b331-736728738f8a"}
145992024-02-03T17:15:52.077Z{"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:51.624197558Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671}
146002024-02-03T17:15:52.078ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
146012024-02-03T17:15:52.078ZLoading write count information from file "/tmp/test_repair_verify.data"
146022024-02-03T17:15:52.078ZRun Repair workload
146032024-02-03T17:15:52.078Z01/30 Write block 0 len 4096 data: 2
146042024-02-03T17:15:52.078Z02/30 Read block 260 len 12288
146052024-02-03T17:15:52.082Z03/30 Read block 476 len 28672
146062024-02-03T17:15:52.082Z04/30 Read block 322 len 20480
146072024-02-03T17:15:52.082Z05/30 Flush
146082024-02-03T17:15:52.082Z06/30 Flush
146092024-02-03T17:15:52.082Z07/30 Write block 375 len 24576 data: 10 12 14 11 13 11
146102024-02-03T17:15:52.083Z08/30 Read block 412 len 24576
146112024-02-03T17:15:52.083Z09/30 Read block 505 len 12288
146122024-02-03T17:15:52.083Z10/30 Read block 151 len 8192
146132024-02-03T17:15:52.083Z11/30 Write block 522 len 28672 data: 8 9 8 10 9 11 10
146142024-02-03T17:15:52.083Z12/30 Read block 19 len 40960
146152024-02-03T17:15:52.083Z13/30 Write block 33 len 16384 data: 10 11 10 12
146162024-02-03T17:15:52.083Z14/30 Write block 334 len 20480 data: 11 10 11 11 14
146172024-02-03T17:15:52.083Z15/30 Write block 245 len 28672 data: 8 9 6 10 10 9 9
146182024-02-03T17:15:52.083Z16/30 Read block 353 len 12288
146192024-02-03T17:15:52.092Z17/30 Read block 589 len 4096
146202024-02-03T17:15:52.092Z18/30 Write block 276 len 4096 data: 11
146212024-02-03T17:15:52.092Z19/30 Read block 567 len 8192
146222024-02-03T17:15:52.092Z20/30 Write block 562 len 20480 data: 14 14 15 13 12
146232024-02-03T17:15:52.092Z21/30 Flush
146242024-02-03T17:15:52.096Z22/30 Write block 481 len 28672 data: 10 10 9 8 9 9 6
146252024-02-03T17:15:52.096Z23/30 Write block 433 len 40960 data: 9 9 13 11 12 13 13 13 14 16
146262024-02-03T17:15:52.096Z24/30 Write block 17 len 4096 data: 12
146272024-02-03T17:15:52.096Z25/30 Write block 63 len 16384 data: 9 7 7 8
146282024-02-03T17:15:52.096Z26/30 Read block 202 len 20480
146292024-02-03T17:15:52.102Z27/30 Write block 315 len 24576 data: 12 12 13 10 13 12
146302024-02-03T17:15:52.103Z28/30 Write block 407 len 24576 data: 10 8 8 9 9 10
146312024-02-03T17:15:52.107Z29/30 Write block 523 len 28672 data: 10 9 11 10 12 11 11
146322024-02-03T17:15:52.107Z30/30 Read block 512 len 8192
146332024-02-03T17:15:52.107Z----------------------------------------------------------------
146342024-02-03T17:15:52.108Z Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30
146352024-02-03T17:15:52.108ZStates: Active Active Active
146362024-02-03T17:15:52.108ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
146372024-02-03T17:15:52.108Z 1 Acked 1000 Write 1 Done Done Done false
146382024-02-03T17:15:52.108Z 2 Acked 1001 Read 3 Done Sent Done false
146392024-02-03T17:15:52.108Z 3 Acked 1002 Read 7 Done Sent Done false
146402024-02-03T17:15:52.108Z 4 Acked 1003 Read 5 Done Sent Done false
146412024-02-03T17:15:52.108Z 5 Acked 1004 Flush 0 Done Sent Done false
146422024-02-03T17:15:52.108Z 6 Acked 1005 Flush 0 Done Sent Done false
146432024-02-03T17:15:52.108Z 7 Acked 1006 Write 6 Done Sent Done false
146442024-02-03T17:15:52.108Z 8 Acked 1007 Read 6 Done Sent Done false
146452024-02-03T17:15:52.108Z 9 Acked 1008 Read 3 Done Sent Done false
146462024-02-03T17:15:52.108Z 10 Acked 1009 Read 2 Done Sent Done false
146472024-02-03T17:15:52.109Z 11 Acked 1010 Write 7 Done Sent Done false
146482024-02-03T17:15:52.109Z 12 Acked 1011 Read 10 Done Sent Done false
146492024-02-03T17:15:52.109Z 13 Acked 1012 Write 4 Done Sent Done false
146502024-02-03T17:15:52.109Z 14 Acked 1013 Write 5 Done Sent Done false
146512024-02-03T17:15:52.109Z 15 Acked 1014 Write 7 Done Sent Done false
146522024-02-03T17:15:52.109Z 16 Acked 1015 Read 3 Done Sent Done false
146532024-02-03T17:15:52.109Z 17 Acked 1016 Read 1 Done Sent Done false
146542024-02-03T17:15:52.110Z 18 Acked 1017 Write 1 Done Sent Done false
146552024-02-03T17:15:52.110Z 19 Acked 1018 Read 2 Done Sent Done false
146562024-02-03T17:15:52.110Z 20 Acked 1019 Write 5 Done Sent Done false
146572024-02-03T17:15:52.110Z 21 Acked 1020 Flush 0 Done Sent Done false
146582024-02-03T17:15:52.110Z 22 Acked 1021 Write 7 Done Sent Done false
146592024-02-03T17:15:52.110Z 23 Acked 1022 Write 10 Done Sent Done false
146602024-02-03T17:15:52.110Z 24 Acked 1023 Write 1 Done Sent Done false
146612024-02-03T17:15:52.110Z 25 Acked 1024 Write 4 Done Sent Done false
146622024-02-03T17:15:52.110Z 26 Acked 1025 Read 5 Done Sent Done false
146632024-02-03T17:15:52.110Z 27 Acked 1026 Write 6 Done Sent Done false
146642024-02-03T17:15:52.110Z 28 Acked 1027 Write 6 Done Sent Done false
146652024-02-03T17:15:52.110Z 29 Acked 1028 Write 7 Done Sent Done false
146662024-02-03T17:15:52.111Z 30 Acked 1029 Read 2 Done Sent Done false
146672024-02-03T17:15:52.111Z STATES DS:0 DS:1 DS:2 TOTAL
146682024-02-03T17:15:52.111Z New 0 0 0 0
146692024-02-03T17:15:52.111Z Sent 0 29 0 29
146702024-02-03T17:15:52.111Z Done 30 1 30 61
146712024-02-03T17:15:52.111Z Skipped 0 0 0 0
146722024-02-03T17:15:52.111Z Error 0 0 0 0
146732024-02-03T17:15:52.111ZLast Flush: 1020 0 1020
146742024-02-03T17:15:52.111ZDownstairs last five completed:
146752024-02-03T17:15:52.111ZUpstairs last five completed: 30 29 28 27 26
146762024-02-03T17:15:52.111Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:15:51.671616341Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1671}
146772024-02-03T17:15:52.111ZWrote out file "/tmp/test_repair_verify.data"
146782024-02-03T17:15:52.118Z
146792024-02-03T17:15:52.118ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
146802024-02-03T17:15:52.151ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
146812024-02-03T17:15:52.151Z 0 000-019 134 129 134 232 221 232 T T T <---
146822024-02-03T17:15:52.167Z 1 020-039 134 133 134 234 231 234 F F F <---
146832024-02-03T17:15:52.168Z 2 040-059 130 130 130 225 225 225 F F F
146842024-02-03T17:15:52.168Z 3 060-079 131 131 131 227 227 227 T F T <---
146852024-02-03T17:15:52.168Z 4 080-099 132 132 132 230 230 230 F F F
146862024-02-03T17:15:52.168Z 5 100-119 130 130 130 226 226 226 F F F
146872024-02-03T17:15:52.168Z 6 120-139 131 131 131 227 227 227 F F F
146882024-02-03T17:15:52.168Z 7 140-159 132 132 132 229 229 229 F F F
146892024-02-03T17:15:52.168Z 8 160-179 133 133 133 231 231 231 F F F
146902024-02-03T17:15:52.168Z 9 180-199 133 133 133 231 231 231 F F F
146912024-02-03T17:15:52.168Z 10 200-219 132 132 132 228 228 228 F F F
146922024-02-03T17:15:52.168Z 11 220-239 133 133 133 231 231 231 F F F
146932024-02-03T17:15:52.168Z 12 240-259 134 130 134 234 225 234 F F F <---
146942024-02-03T17:15:52.168Z 13 260-279 134 130 134 234 223 234 F F F <---
146952024-02-03T17:15:52.168Z 14 280-299 131 131 131 227 227 227 F F F
146962024-02-03T17:15:52.168Z 15 300-319 119 119 119 203 203 203 T F T <---
146972024-02-03T17:15:52.168Z 16 320-339 134 130 134 234 224 234 T F T <---
146982024-02-03T17:15:52.168Z 17 340-359 130 130 130 224 224 224 F F F
146992024-02-03T17:15:52.168Z 18 360-379 134 132 134 234 229 234 F F F <---
147002024-02-03T17:15:52.168Z 19 380-399 134 124 134 234 209 234 F F F <---
147012024-02-03T17:15:52.169Z 20 400-419 130 130 130 224 224 224 T F T <---
147022024-02-03T17:15:52.169Z 21 420-439 133 133 133 231 231 231 T F T <---
147032024-02-03T17:15:52.169Z 22 440-459 133 133 133 231 231 231 T F T <---
147042024-02-03T17:15:52.169Z 23 460-479 130 130 130 223 223 223 F F F
147052024-02-03T17:15:52.169Z 24 480-499 130 130 130 223 223 223 T F T <---
147062024-02-03T17:15:52.169Z 25 500-519 131 131 131 227 227 227 F F F
147072024-02-03T17:15:52.169Z 26 520-539 134 124 134 234 210 234 T F T <---
147082024-02-03T17:15:52.169Z 27 540-559 131 131 131 227 227 227 F F F
147092024-02-03T17:15:52.169Z 28 560-579 134 124 134 234 211 234 F F F <---
147102024-02-03T17:15:52.169Z 29 580-599 132 132 132 228 228 228 F F F
147112024-02-03T17:15:52.169ZMax gen: 134, Max flush: 234
147122024-02-03T17:15:52.169ZError: Difference in extent metadata found!
147132024-02-03T17:15:52.169ZA Difference in extent metadata is expected here
147142024-02-03T17:15:52.169ZOn loop 67
147152024-02-03T17:15:52.169Z
147162024-02-03T17:15:52.169ZVerifying data now
147172024-02-03T17:15:57.442ZNo change after verify
147182024-02-03T17:15:57.442ZLoop: 67 Downstairs dump after verify (and repair):
147192024-02-03T17:15:57.462ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
147202024-02-03T17:15:57.462Z 0 000-019 135 135 135 235 235 235 F F F
147212024-02-03T17:15:57.462Z 1 020-039 134 134 134 234 234 234 F F F
147222024-02-03T17:15:57.462Z 2 040-059 130 130 130 225 225 225 F F F
147232024-02-03T17:15:57.462Z 3 060-079 135 135 135 235 235 235 F F F
147242024-02-03T17:15:57.465Z 4 080-099 132 132 132 230 230 230 F F F
147252024-02-03T17:15:57.466Z 5 100-119 130 130 130 226 226 226 F F F
147262024-02-03T17:15:57.466Z 6 120-139 131 131 131 227 227 227 F F F
147272024-02-03T17:15:57.466Z 7 140-159 132 132 132 229 229 229 F F F
147282024-02-03T17:15:57.466Z 8 160-179 133 133 133 231 231 231 F F F
147292024-02-03T17:15:57.466Z 9 180-199 133 133 133 231 231 231 F F F
147302024-02-03T17:15:57.466Z 10 200-219 132 132 132 228 228 228 F F F
147312024-02-03T17:15:57.466Z 11 220-239 133 133 133 231 231 231 F F F
147322024-02-03T17:15:57.466Z 12 240-259 134 134 134 234 234 234 F F F
147332024-02-03T17:15:57.466Z 13 260-279 134 134 134 234 234 234 F F F
147342024-02-03T17:15:57.466Z 14 280-299 131 131 131 227 227 227 F F F
147352024-02-03T17:15:57.466Z 15 300-319 135 135 135 235 235 235 F F F
147362024-02-03T17:15:57.466Z 16 320-339 135 135 135 235 235 235 F F F
147372024-02-03T17:15:57.466Z 17 340-359 130 130 130 224 224 224 F F F
147382024-02-03T17:15:57.466Z 18 360-379 134 134 134 234 234 234 F F F
147392024-02-03T17:15:57.466Z 19 380-399 134 134 134 234 234 234 F F F
147402024-02-03T17:15:57.466Z 20 400-419 135 135 135 235 235 235 F F F
147412024-02-03T17:15:57.466Z 21 420-439 135 135 135 235 235 235 F F F
147422024-02-03T17:15:57.466Z 22 440-459 135 135 135 235 235 235 F F F
147432024-02-03T17:15:57.466Z 23 460-479 130 130 130 223 223 223 F F F
147442024-02-03T17:15:57.466Z 24 480-499 135 135 135 235 235 235 F F F
147452024-02-03T17:15:57.466Z 25 500-519 131 131 131 227 227 227 F F F
147462024-02-03T17:15:57.466Z 26 520-539 135 135 135 235 235 235 F F F
147472024-02-03T17:15:57.466Z 27 540-559 131 131 131 227 227 227 F F F
147482024-02-03T17:15:57.467Z 28 560-579 134 134 134 234 234 234 F F F
147492024-02-03T17:15:57.467Z 29 580-599 132 132 132 228 228 228 F F F
147502024-02-03T17:15:57.467ZMax gen: 135, Max flush: 235
147512024-02-03T17:15:57.470Z
147522024-02-03T17:15:57.470ZBegin loop 68
147532024-02-03T17:15:57.470ZDownstairs to restart: 1
147542024-02-03T17:15:57.498Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.066735387Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679}
147552024-02-03T17:15:57.502Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.067321168Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679}
147562024-02-03T17:15:57.502Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
147572024-02-03T17:15:57.502Z,"time":"2024-02-03T17:15:57.067445622Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679}
147582024-02-03T17:15:57.502Z{"msg":"Crucible 455d5098-da03-4dd0-b0f3-64c679edface has session id: eb71e698-2ae5-4843-9f0f-8994ae246dd7","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.067577359Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147592024-02-03T17:15:57.502Z{"msg":"Upstairs opts: Upstairs UUID: 455d5098-da03-4dd0-b0f3-64c679edface, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.067640352Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147602024-02-03T17:15:57.502Z{"msg":"Crucible stats registered with UUID: 455d5098-da03-4dd0-b0f3-64c679edface","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.067725435Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147612024-02-03T17:15:57.502Z{"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.067766355Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679}
147622024-02-03T17:15:57.502Z{"msg":"455d5098-da03-4dd0-b0f3-64c679edface active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.067801804Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147632024-02-03T17:15:57.502Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.067926062Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147642024-02-03T17:15:57.502Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.067972697Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147652024-02-03T17:15:57.502Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.068060128Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147662024-02-03T17:15:57.502Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.068106342Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147672024-02-03T17:15:57.502Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.068146472Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147682024-02-03T17:15:57.502Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.068186153Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147692024-02-03T17:15:57.502Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.068265592Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"":"io task","client":"2","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147702024-02-03T17:15:57.502Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.068360492Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"":"io task","client":"0","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147712024-02-03T17:15:57.502Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.068472364Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"":"io task","client":"1","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147722024-02-03T17:15:57.503Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.068529523Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"":"io task","client":"2","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147732024-02-03T17:15:57.503Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.06857656Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"":"io task","client":"1","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147742024-02-03T17:15:57.503Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.068696469Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"":"io task","client":"0","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147752024-02-03T17:15:57.503Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.069742576Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147762024-02-03T17:15:57.503Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.069853873Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147772024-02-03T17:15:57.503Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.069885899Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147782024-02-03T17:15:57.503Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.06991536Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147792024-02-03T17:15:57.508Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.070778919Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147802024-02-03T17:15:57.508Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.071055562Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147812024-02-03T17:15:57.508Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.071628727Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147822024-02-03T17:15:57.508Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.071750273Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147832024-02-03T17:15:57.508Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.071891038Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147842024-02-03T17:15:57.508Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.071961438Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147852024-02-03T17:15:57.508Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.071994987Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147862024-02-03T17:15:57.508Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.072046169Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147872024-02-03T17:15:57.508Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.072071298Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147882024-02-03T17:15:57.509Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.072093578Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147892024-02-03T17:15:57.509Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.072552351Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147902024-02-03T17:15:57.509Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.073290856Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147912024-02-03T17:15:57.509Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.073448856Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147922024-02-03T17:15:57.509Z{"msg":"[0]R flush_numbers[0..12]: [235, 234, 225, 235, 230, 226, 227, 229, 231, 231, 228, 231]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.073498348Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147932024-02-03T17:15:57.509Z{"msg":"[0]R generation[0..12]: [135, 134, 130, 135, 132, 130, 131, 132, 133, 133, 132, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.073574136Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147942024-02-03T17:15:57.509Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.073619449Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147952024-02-03T17:15:57.509Z{"msg":"[1]R flush_numbers[0..12]: [235, 234, 225, 235, 230, 226, 227, 229, 231, 231, 228, 231]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.073678772Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147962024-02-03T17:15:57.509Z{"msg":"[1]R generation[0..12]: [135, 134, 130, 135, 132, 130, 131, 132, 133, 133, 132, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.073720079Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147972024-02-03T17:15:57.509Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]",Wait for a query_work_queue command to finish before sending IO
147982024-02-03T17:15:57.509Z"v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.073827746Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
147992024-02-03T17:15:57.509Z{"msg":"[2]R flush_numbers[0..12]: [235, 234, 225, 235, 230, 226, 227, 229, 231, 231, 228, 231]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.073910465Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
148002024-02-03T17:15:57.509Z{"msg":"[2]R generation[0..12]: [135, 134, 130, 135, 132, 130, 131, 132, 133, 133, 132, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.073953443Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
148012024-02-03T17:15:57.509Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.074010791Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
148022024-02-03T17:15:57.509Z{"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.074066675Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
148032024-02-03T17:15:57.509Z{"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.074096548Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
148042024-02-03T17:15:57.509Z{"msg":"Next flush: 236","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.074167111Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
148052024-02-03T17:15:57.509Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.074195279Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
148062024-02-03T17:15:57.509Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.074231934Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
148072024-02-03T17:15:57.509Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.074290491Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
148082024-02-03T17:15:57.509Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.07433112Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
148092024-02-03T17:15:57.509Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.074397311Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
148102024-02-03T17:15:57.509Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.074440289Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
148112024-02-03T17:15:57.509ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
148122024-02-03T17:15:57.510Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.074509206Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
148132024-02-03T17:15:57.510Z{"msg":"455d5098-da03-4dd0-b0f3-64c679edface is now active with session: eb71e698-2ae5-4843-9f0f-8994ae246dd7","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.07454991Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
148142024-02-03T17:15:57.510Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.074601349Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679,"session_id":"eb71e698-2ae5-4843-9f0f-8994ae246dd7"}
148152024-02-03T17:15:57.510Z{"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:15:57.074625275Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679}
148162024-02-03T17:15:57.510ZLoading write count information from file "/tmp/test_repair_verify.data"
148172024-02-03T17:15:57.510ZRun Repair workload
148182024-02-03T17:15:57.510Z01/30 Flush
148192024-02-03T17:15:57.513Z02/30 Write block 12 len 40960 data: 11 12 12 9 9 13 15 19 15 18
148202024-02-03T17:15:57.513Z03/30 Write block 189 len 40960 data: 21 19 14 15 12 13 14 13 11 10
148212024-02-03T17:15:57.513Z04/30 Read block 205 len 28672
148222024-02-03T17:15:57.519Z05/30 Write block 537 len 12288 data: 6 6 5
148232024-02-03T17:15:57.519Z06/30 Write block 6 len 16384 data: 11 10 15 17
148242024-02-03T17:15:57.522Z07/30 Read block 410 len 28672
148252024-02-03T17:15:57.522Z08/30 Write block 27 len 36864 data: 11 12 10 10 14 12 11 12 11
148262024-02-03T17:15:57.522Z09/30 Read block 583 len 36864
148272024-02-03T17:15:57.525Z10/30 Read block 225 len 16384
148282024-02-03T17:15:57.525Z11/30 Write block 443 len 4096 data: 13
148292024-02-03T17:15:57.525Z12/30 Read block 361 len 40960
148302024-02-03T17:15:57.528Z13/30 Read block 358 len 8192
148312024-02-03T17:15:57.528Z14/30 Read block 227 len 20480
148322024-02-03T17:15:57.531Z15/30 Write block 137 len 40960 data: 9 10 11 12 15 14 12 8 7 7
148332024-02-03T17:15:57.531Z16/30 Flush
148342024-02-03T17:15:57.534Z17/30 Flush
148352024-02-03T17:15:57.537Z18/30 Write block 233 len 28672 data: 8 11 10 8 7 6 7
148362024-02-03T17:15:57.537Z19/30 Read block 305 len 8192
148372024-02-03T17:15:57.540Z20/30 Read block 421 len 8192
148382024-02-03T17:15:57.540Z21/30 Read block 144 len 16384
148392024-02-03T17:15:57.543Z22/30 Read block 3 len 8192
148402024-02-03T17:15:57.543Z23/30 Write block 160 len 12288 data: 14 14 16
148412024-02-03T17:15:57.543Z24/30 Read block 335 len 36864
148422024-02-03T17:15:57.546Z25/30 Read block 412 len 36864
148432024-02-03T17:15:57.547Z26/30 Read block 339 len 36864
148442024-02-03T17:15:57.550Z27/30 Write block 596 len 16384 data: 9 6 4 3
148452024-02-03T17:15:57.550Z28/30 Read block 540 len 24576
148462024-02-03T17:15:57.553Z29/30 Write block 160 len 12288 data: 15 15 17
148472024-02-03T17:15:57.553Z30/30 Write block 272 len 12288 data: 13 12 10
148482024-02-03T17:15:57.553Z----------------------------------------------------------------
148492024-02-03T17:15:57.553Z Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:29
148502024-02-03T17:15:57.553ZStates: Active Active Active
148512024-02-03T17:15:57.554ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
148522024-02-03T17:15:57.554Z 2 Acked 1001 Write 10 Done Done Done false
148532024-02-03T17:15:57.554Z 3 Acked 1002 Write 10 Done Done Done false
148542024-02-03T17:15:57.554Z 4 Acked 1003 Read 7 Done Done Done false
148552024-02-03T17:15:57.554Z 5 Acked 1004 Write 3 Done Done Done false
148562024-02-03T17:15:57.554Z 6 Acked 1005 Write 4 Done Sent Done false
148572024-02-03T17:15:57.554Z 7 Acked 1006 Read 7 Done Sent Done false
148582024-02-03T17:15:57.554Z 8 Acked 1007 Write 9 Done Sent Done false
148592024-02-03T17:15:57.554Z 9 Acked 1008 Read 9 Done Sent Done false
148602024-02-03T17:15:57.554Z 10 Acked 1009 Read 4 Done Sent Done false
148612024-02-03T17:15:57.554Z 11 Acked 1010 Write 1 Done Sent Done false
148622024-02-03T17:15:57.554Z 12 Acked 1011 Read 10 Done Sent Done false
148632024-02-03T17:15:57.554Z 13 Acked 1012 Read 2 Done Sent Done false
148642024-02-03T17:15:57.554Z 14 Acked 1013 Read 5 Done Sent Done false
148652024-02-03T17:15:57.554Z 15 Acked 1014 Write 10 Done Sent Done false
148662024-02-03T17:15:57.554Z 16 Acked 1015 Flush 0 Done Sent Done false
148672024-02-03T17:15:57.554Z 17 Acked 1016 Flush 0 Done Sent Done false
148682024-02-03T17:15:57.554Z 18 Acked 1017 Write 7 Done Sent Done false
148692024-02-03T17:15:57.554Z 19 Acked 1018 Read 2 Done Sent Done false
148702024-02-03T17:15:57.554Z 20 Acked 1019 Read 2 Done Sent Done false
148712024-02-03T17:15:57.554Z 21 Acked 1020 Read 4 Done Sent Done false
148722024-02-03T17:15:57.554Z 22 Acked 1021 Read 2 Done Sent Done false
148732024-02-03T17:15:57.554Z 23 Acked 1022 Write 3 Done Sent Done false
148742024-02-03T17:15:57.554Z 24 Acked 1023 Read 9 Done Sent Done false
148752024-02-03T17:15:57.555Z 25 Acked 1024 Read 9 Done Sent Done false
148762024-02-03T17:15:57.555Z 26 Acked 1025 Read 9 Done Sent Done false
148772024-02-03T17:15:57.555Z 27 Acked 1026 Write 4 Done Sent Done false
148782024-02-03T17:15:57.555Z 28 Acked 1027 Read 6 Done Sent Done false
148792024-02-03T17:15:57.555Z 29 Acked 1028 Write 3 Done Sent Done false
148802024-02-03T17:15:57.555Z 30 Acked 1029 Write 3 Sent Sent Sent false
148812024-02-03T17:15:57.555Z STATES DS:0 DS:1 DS:2 TOTAL
148822024-02-03T17:15:57.555Z New 0 0 0 0
148832024-02-03T17:15:57.555Z Sent 1 25 1 27
148842024-02-03T17:15:57.555Z Done 28 4 28 60
148852024-02-03T17:15:57.555Z Skipped 0 0 0 0
148862024-02-03T17:15:57.555Z Error 0 0 0 0
148872024-02-03T17:15:57.555ZLast Flush: 1016 1000 1016
148882024-02-03T17:15:57.555ZDownstairs last five completed: 1000
148892024-02-03T17:15:57.555ZUpstairs last five completed: 30 29 28 27 26
148902024-02-03T17:15:57.558Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:15:57.122563253Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1679}
148912024-02-03T17:15:57.558ZWrote out file "/tmp/test_repair_verify.data"
148922024-02-03T17:15:57.561Z
148932024-02-03T17:15:57.566ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
148942024-02-03T17:15:57.593ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
148952024-02-03T17:15:57.593Z 0 000-019 136 135 136 237 235 237 F T F <---
148962024-02-03T17:15:57.596Z 1 020-039 136 134 136 237 234 237 F T F <---
148972024-02-03T17:15:57.596Z 2 040-059 130 130 130 225 225 225 F F F
148982024-02-03T17:15:57.596Z 3 060-079 135 135 135 235 235 235 F F F
148992024-02-03T17:15:57.596Z 4 080-099 132 132 132 230 230 230 F F F
149002024-02-03T17:15:57.596Z 5 100-119 130 130 130 226 226 226 F F F
149012024-02-03T17:15:57.596Z 6 120-139 136 131 136 237 227 237 F F F <---
149022024-02-03T17:15:57.596Z 7 140-159 136 132 136 237 229 237 F F F <---
149032024-02-03T17:15:57.596Z 8 160-179 133 133 133 231 231 231 T F T <---
149042024-02-03T17:15:57.596Z 9 180-199 136 133 136 237 231 237 F T F <---
149052024-02-03T17:15:57.596Z 10 200-219 132 132 132 228 228 228 F F F
149062024-02-03T17:15:57.596Z 11 220-239 133 133 133 231 231 231 T F T <---
149072024-02-03T17:15:57.596Z 12 240-259 134 134 134 234 234 234 F F F
149082024-02-03T17:15:57.596Z 13 260-279 134 134 134 234 234 234 T F T <---
149092024-02-03T17:15:57.597Z 14 280-299 131 131 131 227 227 227 F F F
149102024-02-03T17:15:57.597Z 15 300-319 135 135 135 235 235 235 F F F
149112024-02-03T17:15:57.597Z 16 320-339 135 135 135 235 235 235 F F F
149122024-02-03T17:15:57.597Z 17 340-359 130 130 130 224 224 224 F F F
149132024-02-03T17:15:57.597Z 18 360-379 134 134 134 234 234 234 F F F
149142024-02-03T17:15:57.597Z 19 380-399 134 134 134 234 234 234 F F F
149152024-02-03T17:15:57.597Z 20 400-419 135 135 135 235 235 235 F F F
149162024-02-03T17:15:57.597Z 21 420-439 135 135 135 235 235 235 F F F
149172024-02-03T17:15:57.597Z 22 440-459 136 135 136 237 235 237 F F F <---
149182024-02-03T17:15:57.597Z 23 460-479 130 130 130 223 223 223 F F F
149192024-02-03T17:15:57.597Z 24 480-499 135 135 135 235 235 235 F F F
149202024-02-03T17:15:57.597Z 25 500-519 131 131 131 227 227 227 F F F
149212024-02-03T17:15:57.597Z 26 520-539 136 135 136 237 235 237 F T F <---
149222024-02-03T17:15:57.597Z 27 540-559 131 131 131 227 227 227 F F F
149232024-02-03T17:15:57.597Z 28 560-579 134 134 134 234 234 234 F F F
149242024-02-03T17:15:57.597Z 29 580-599 132 132 132 228 228 228 T F T <---
149252024-02-03T17:15:57.597ZMax gen: 136, Max flush: 237
149262024-02-03T17:15:57.597ZError: Difference in extent metadata found!
149272024-02-03T17:15:57.600ZA Difference in extent metadata is expected here
149282024-02-03T17:15:57.600ZOn loop 68
149292024-02-03T17:15:57.600Z
149302024-02-03T17:15:57.606ZVerifying data now
149312024-02-03T17:16:03.585ZNo change after verify
149322024-02-03T17:16:03.585ZLoop: 68 Downstairs dump after verify (and repair):
149332024-02-03T17:16:03.608ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
149342024-02-03T17:16:03.608Z 0 000-019 136 136 136 237 237 237 F F F
149352024-02-03T17:16:03.608Z 1 020-039 136 136 136 237 237 237 F F F
149362024-02-03T17:16:03.608Z 2 040-059 130 130 130 225 225 225 F F F
149372024-02-03T17:16:03.608Z 3 060-079 135 135 135 235 235 235 F F F
149382024-02-03T17:16:03.618Z 4 080-099 132 132 132 230 230 230 F F F
149392024-02-03T17:16:03.618Z 5 100-119 130 130 130 226 226 226 F F F
149402024-02-03T17:16:03.618Z 6 120-139 136 136 136 237 237 237 F F F
149412024-02-03T17:16:03.618Z 7 140-159 136 136 136 237 237 237 F F F
149422024-02-03T17:16:03.618Z 8 160-179 137 137 137 238 238 238 F F F
149432024-02-03T17:16:03.618Z 9 180-199 136 136 136 237 237 237 F F F
149442024-02-03T17:16:03.618Z 10 200-219 132 132 132 228 228 228 F F F
149452024-02-03T17:16:03.618Z 11 220-239 137 137 137 238 238 238 F F F
149462024-02-03T17:16:03.618Z 12 240-259 134 134 134 234 234 234 F F F
149472024-02-03T17:16:03.618Z 13 260-279 137 137 137 238 238 238 F F F
149482024-02-03T17:16:03.618Z 14 280-299 131 131 131 227 227 227 F F F
149492024-02-03T17:16:03.619Z 15 300-319 135 135 135 235 235 235 F F F
149502024-02-03T17:16:03.619Z 16 320-339 135 135 135 235 235 235 F F F
149512024-02-03T17:16:03.619Z 17 340-359 130 130 130 224 224 224 F F F
149522024-02-03T17:16:03.619Z 18 360-379 134 134 134 234 234 234 F F F
149532024-02-03T17:16:03.619Z 19 380-399 134 134 134 234 234 234 F F F
149542024-02-03T17:16:03.619Z 20 400-419 135 135 135 235 235 235 F F F
149552024-02-03T17:16:03.619Z 21 420-439 135 135 135 235 235 235 F F F
149562024-02-03T17:16:03.619Z 22 440-459 136 136 136 237 237 237 F F F
149572024-02-03T17:16:03.619Z 23 460-479 130 130 130 223 223 223 F F F
149582024-02-03T17:16:03.619Z 24 480-499 135 135 135 235 235 235 F F F
149592024-02-03T17:16:03.619Z 25 500-519 131 131 131 227 227 227 F F F
149602024-02-03T17:16:03.619Z 26 520-539 136 136 136 237 237 237 F F F
149612024-02-03T17:16:03.620Z 27 540-559 131 131 131 227 227 227 F F F
149622024-02-03T17:16:03.620Z 28 560-579 134 134 134 234 234 234 F F F
149632024-02-03T17:16:03.620Z 29 580-599 137 137 137 238 238 238 F F F
149642024-02-03T17:16:03.620ZMax gen: 137, Max flush: 238
149652024-02-03T17:16:03.620Z
149662024-02-03T17:16:03.620ZBegin loop 69
149672024-02-03T17:16:03.620ZDownstairs to restart: 2
149682024-02-03T17:16:03.643Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.210562248Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687}
149692024-02-03T17:16:03.643Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.21112292Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687}
149702024-02-03T17:16:03.649Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.211159295Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687}
149712024-02-03T17:16:03.649Z{"msg":"Crucible 03f913d2-0b9c-4888-bec6-8c9a20a672b9 has session id: f7932040-a7d5-4ff0-8a6a-9319b7b6be54","v":0,"name":"crucible","level":30Crucible runtime is spawned
149722024-02-03T17:16:03.649Z,"time":"2024-02-03T17:16:03.211239524Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
149732024-02-03T17:16:03.650Z{"msg":"Upstairs opts: Upstairs UUID: 03f913d2-0b9c-4888-bec6-8c9a20a672b9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.211311253Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
149742024-02-03T17:16:03.650Z{"msg":"Crucible stats registered with UUID: 03f913d2-0b9c-4888-bec6-8c9a20a672b9","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.211400015Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
149752024-02-03T17:16:03.650Z{"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.211429625Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687}
149762024-02-03T17:16:03.650Z{"msg":"03f913d2-0b9c-4888-bec6-8c9a20a672b9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.211653616Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
149772024-02-03T17:16:03.650Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.211715206Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
149782024-02-03T17:16:03.650Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.211840449Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
149792024-02-03T17:16:03.650Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.21192031Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
149802024-02-03T17:16:03.650Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.211996755Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
149812024-02-03T17:16:03.650Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.212066067Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
149822024-02-03T17:16:03.650Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.212197216Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
149832024-02-03T17:16:03.650Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.212247866Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"":"io task","client":"0","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
149842024-02-03T17:16:03.650Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.212291564Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"":"io task","client":"2","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
149852024-02-03T17:16:03.650Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.212391361Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"":"io task","client":"1","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
149862024-02-03T17:16:03.650Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.212505675Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"":"io task","client":"2","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
149872024-02-03T17:16:03.650Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.212551096Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"":"io task","client":"1","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
149882024-02-03T17:16:03.650Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.212636018Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"":"io task","client":"0","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
149892024-02-03T17:16:03.650Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.213800335Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
149902024-02-03T17:16:03.650Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.213966033Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
149912024-02-03T17:16:03.650Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.214017535Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
149922024-02-03T17:16:03.650Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.21406276Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
149932024-02-03T17:16:03.650Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.214710597Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
149942024-02-03T17:16:03.650Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.214804468Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
149952024-02-03T17:16:03.650Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.214855007Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
149962024-02-03T17:16:03.650Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.215250013Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
149972024-02-03T17:16:03.651Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.215763987Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
149982024-02-03T17:16:03.651Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.215826924Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
149992024-02-03T17:16:03.651Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.215852376Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
150002024-02-03T17:16:03.651Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.215940666Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
150012024-02-03T17:16:03.651Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.216189793Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
150022024-02-03T17:16:03.651Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.216253038Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
150032024-02-03T17:16:03.651Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.216278951Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
150042024-02-03T17:16:03.651Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.216719163Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
150052024-02-03T17:16:03.651Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.216785427Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
150062024-02-03T17:16:03.651Z{"msg":"[0]R flush_numbers[0..12]: [237, 237, 225, 235, 230, 226, 237, 237, 238, 237, 228, 238]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.216831879Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
150072024-02-03T17:16:03.651Z{"msg":"[0]R generation[0..12]: [136, 136, 130, 135, 132, 130, 136, 136, 137, 136, 132, 137]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.216896914Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
150082024-02-03T17:16:03.651Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.216924356Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
150092024-02-03T17:16:03.651Z{"msg":"[1]R flush_numbers[0..12]: [237, 237, 225, 235, 230, 226, 237, 237, 238, 237, 228, 238]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.216977431Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
150102024-02-03T17:16:03.651Z{"msg":"[1]R generation[0..12]: [136, 136, 130, 135, 132, 130, 136, 136, 137, 136, 132, 137]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.217051517Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
150112024-02-03T17:16:03.651Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.217076894Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
150122024-02-03T17:16:03.656Z{"msg":"[2]R flush_numbers[0..12]: [237, 237, 225, 235, 230, 226, 237, 237, 238, 237, 228, 238]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
150132024-02-03T17:16:03.656Z,"time":"2024-02-03T17:16:03.217100415Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
150142024-02-03T17:16:03.656Z{"msg":"[2]R generation[0..12]: [136, 136, 130, 135, 132, 130, 136, 136, 137, 136, 132, 137]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.217123937Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
150152024-02-03T17:16:03.656Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.217147285Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
150162024-02-03T17:16:03.656Z{"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.217168571Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
150172024-02-03T17:16:03.656Z{"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.21719261Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
150182024-02-03T17:16:03.656Z{"msg":"Next flush: 239","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.217291074Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
150192024-02-03T17:16:03.656Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.217342063Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
150202024-02-03T17:16:03.656Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.217366526Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
150212024-02-03T17:16:03.656Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.217386737Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
150222024-02-03T17:16:03.656Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.217435813Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
150232024-02-03T17:16:03.656Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.217459849Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
150242024-02-03T17:16:03.657Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.217531379Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
150252024-02-03T17:16:03.657Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.217554131Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
150262024-02-03T17:16:03.657Z{"msg":"03f913d2-0b9c-4888-bec6-8c9a20a672b9 is now active with session: f7932040-a7d5-4ff0-8a6a-9319b7b6be54","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.217575705Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
150272024-02-03T17:16:03.657Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.217595411Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687,"session_id":"f7932040-a7d5-4ff0-8a6a-9319b7b6be54"}
150282024-02-03T17:16:03.657Z{"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:03.217616129Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687}
150292024-02-03T17:16:03.657ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
150302024-02-03T17:16:03.657ZLoading write count information from file "/tmp/test_repair_verify.data"
150312024-02-03T17:16:03.657ZRun Repair workload
150322024-02-03T17:16:03.657Z01/30 Write block 440 len 32768 data: 14 15 17 14 11 13 15 15
150332024-02-03T17:16:03.657Z02/30 Read block 365 len 8192
150342024-02-03T17:16:03.657Z03/30 Write block 187 len 24576 data: 19 20 22 20 15 16
150352024-02-03T17:16:03.665Z04/30 Write block 405 len 40960 data: 9 10 11 9 9 10 10 11 10 11
150362024-02-03T17:16:03.665Z05/30 Flush
150372024-02-03T17:16:03.665Z06/30 Write block 248 len 24576 data: 11 11 10 10 9 6
150382024-02-03T17:16:03.665Z07/30 Flush
150392024-02-03T17:16:03.682Z08/30 Write block 357 len 16384 data: 9 8 11 10
150402024-02-03T17:16:03.682Z09/30 Read block 57 len 4096
150412024-02-03T17:16:03.682Z10/30 Write block 463 len 40960 data: 12 10 9 11 10 9 9 10 8 8
150422024-02-03T17:16:03.682Z11/30 Write block 113 len 36864 data: 7 8 7 9 13 11 10 9 8
150432024-02-03T17:16:03.686Z12/30 Write block 538 len 32768 data: 7 6 6 8 11 10 8 9
150442024-02-03T17:16:03.687Z13/30 Read block 130 len 20480
150452024-02-03T17:16:03.687Z14/30 Flush
150462024-02-03T17:16:03.687Z15/30 Write block 369 len 32768 data: 12 12 8 11 12 12 11 13
150472024-02-03T17:16:03.687Z16/30 Read block 448 len 28672
150482024-02-03T17:16:03.687Z17/30 Write block 567 len 16384 data: 10 13 11 11
150492024-02-03T17:16:03.687Z18/30 Read block 569 len 12288
150502024-02-03T17:16:03.687Z19/30 Write block 207 len 28672 data: 10 11 10 12 10 10 9
150512024-02-03T17:16:03.687Z20/30 Read block 67 len 8192
150522024-02-03T17:16:03.694Z21/30 Write block 335 len 40960 data: 11 12 12 15 12 13 11 11 7 8
150532024-02-03T17:16:03.694Z22/30 Read block 451 len 24576
150542024-02-03T17:16:03.694Z23/30 Read block 394 len 16384
150552024-02-03T17:16:03.694Z24/30 Read block 515 len 4096
150562024-02-03T17:16:03.698Z25/30 Write block 333 len 16384 data: 9 12 12 13
150572024-02-03T17:16:03.698Z26/30 Read block 306 len 20480
150582024-02-03T17:16:03.698Z27/30 Read block 252 len 40960
150592024-02-03T17:16:03.698Z28/30 Read block 332 len 40960
150602024-02-03T17:16:03.705Z29/30 Write block 231 len 4096 data: 8
150612024-02-03T17:16:03.706Z30/30 Write block 277 len 40960 data: 11 11 13 12 13 15 17 13 13 13
150622024-02-03T17:16:03.706Z----------------------------------------------------------------
150632024-02-03T17:16:03.706Z Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:30
150642024-02-03T17:16:03.706ZStates: Active Active Active
150652024-02-03T17:16:03.706ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
150662024-02-03T17:16:03.706Z 1 Acked 1000 Write 8 Done Done Sent false
150672024-02-03T17:16:03.706Z 2 Acked 1001 Read 2 Done Done Sent false
150682024-02-03T17:16:03.706Z 3 Acked 1002 Write 6 Done Done Sent false
150692024-02-03T17:16:03.706Z 4 Acked 1003 Write 10 Done Done Sent false
150702024-02-03T17:16:03.706Z 5 Acked 1004 Flush 0 Done Done Sent false
150712024-02-03T17:16:03.706Z 6 Acked 1005 Write 6 Done Done Sent false
150722024-02-03T17:16:03.706Z 7 Acked 1006 Flush 0 Done Done Sent false
150732024-02-03T17:16:03.707Z 8 Acked 1007 Write 4 Done Done Sent false
150742024-02-03T17:16:03.707Z 9 Acked 1008 Read 1 Done Done Sent false
150752024-02-03T17:16:03.707Z 10 Acked 1009 Write 10 Done Done Sent false
150762024-02-03T17:16:03.707Z 11 Acked 1010 Write 9 Done Done Sent false
150772024-02-03T17:16:03.707Z 12 Acked 1011 Write 8 Done Done Sent false
150782024-02-03T17:16:03.707Z 13 Acked 1012 Read 5 Done Done Sent false
150792024-02-03T17:16:03.707Z 14 Acked 1013 Flush 0 Done Done Sent false
150802024-02-03T17:16:03.707Z 15 Acked 1014 Write 8 Done Done Sent false
150812024-02-03T17:16:03.707Z 16 Acked 1015 Read 7 Done Done Sent false
150822024-02-03T17:16:03.707Z 17 Acked 1016 Write 4 Done Done Sent false
150832024-02-03T17:16:03.707Z 18 Acked 1017 Read 3 Done Done Sent false
150842024-02-03T17:16:03.707Z 19 Acked 1018 Write 7 Done Done Sent false
150852024-02-03T17:16:03.708Z 20 Acked 1019 Read 2 Done Done Sent false
150862024-02-03T17:16:03.708Z 21 Acked 1020 Write 10 Done Done Sent false
150872024-02-03T17:16:03.708Z 22 Acked 1021 Read 6 Done Done Sent false
150882024-02-03T17:16:03.708Z 23 Acked 1022 Read 4 Done Done Sent false
150892024-02-03T17:16:03.708Z 24 Acked 1023 Read 1 Done Done Sent false
150902024-02-03T17:16:03.708Z 25 Acked 1024 Write 4 Done Done Sent false
150912024-02-03T17:16:03.708Z 26 Acked 1025 Read 5 Done Done Sent false
150922024-02-03T17:16:03.708Z 27 Acked 1026 Read 10 Done Done Sent false
150932024-02-03T17:16:03.708Z 28 Acked 1027 Read 10 Done Done Sent false
150942024-02-03T17:16:03.708Z 29 Acked 1028 Write 1 Done Done Sent false
150952024-02-03T17:16:03.708Z 30 Acked 1029 Write 10 Sent Sent Sent false
150962024-02-03T17:16:03.708Z STATES DS:0 DS:1 DS:2 TOTAL
150972024-02-03T17:16:03.708Z New 0 0 0 0
150982024-02-03T17:16:03.709Z Sent 1 1 30 32
150992024-02-03T17:16:03.709Z Done 29 29 0 58
151002024-02-03T17:16:03.709Z Skipped 0 0 0 0
151012024-02-03T17:16:03.709Z Error 0 0 0 0
151022024-02-03T17:16:03.709ZLast Flush: 1013 1013 0
151032024-02-03T17:16:03.709ZDownstairs last five completed:
151042024-02-03T17:16:03.709ZUpstairs last five completed: 30 29 28 27 26
151052024-02-03T17:16:03.709Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:16:03.271602154Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1687}
151062024-02-03T17:16:03.709ZWrote out file "/tmp/test_repair_verify.data"
151072024-02-03T17:16:03.713Z
151082024-02-03T17:16:03.717ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
151092024-02-03T17:16:03.747ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
151102024-02-03T17:16:03.747Z 0 000-019 136 136 136 237 237 237 F F F
151112024-02-03T17:16:03.748Z 1 020-039 136 136 136 237 237 237 F F F
151122024-02-03T17:16:03.748Z 2 040-059 130 130 130 225 225 225 F F F
151132024-02-03T17:16:03.748Z 3 060-079 135 135 135 235 235 235 F F F
151142024-02-03T17:16:03.757Z 4 080-099 132 132 132 230 230 230 F F F
151152024-02-03T17:16:03.757Z 5 100-119 138 138 130 241 241 226 F F F <---
151162024-02-03T17:16:03.757Z 6 120-139 138 138 136 241 241 237 F F F <---
151172024-02-03T17:16:03.757Z 7 140-159 136 136 136 237 237 237 F F F
151182024-02-03T17:16:03.757Z 8 160-179 137 137 137 238 238 238 F F F
151192024-02-03T17:16:03.757Z 9 180-199 138 138 136 239 239 237 F F F <---
151202024-02-03T17:16:03.757Z 10 200-219 132 132 132 228 228 228 T T F <---
151212024-02-03T17:16:03.757Z 11 220-239 137 137 137 238 238 238 T T F <---
151222024-02-03T17:16:03.757Z 12 240-259 138 138 134 240 240 234 F F F <---
151232024-02-03T17:16:03.757Z 13 260-279 137 137 137 238 238 238 T T F <---
151242024-02-03T17:16:03.758Z 14 280-299 131 131 131 227 227 227 T T F <---
151252024-02-03T17:16:03.758Z 15 300-319 135 135 135 235 235 235 F F F
151262024-02-03T17:16:03.758Z 16 320-339 135 135 135 235 235 235 T T F <---
151272024-02-03T17:16:03.758Z 17 340-359 138 138 130 241 241 224 T T F <---
151282024-02-03T17:16:03.758Z 18 360-379 138 138 134 241 241 234 T T F <---
151292024-02-03T17:16:03.758Z 19 380-399 134 134 134 234 234 234 F F F
151302024-02-03T17:16:03.758Z 20 400-419 138 138 135 239 239 235 F F F <---
151312024-02-03T17:16:03.758Z 21 420-439 135 135 135 235 235 235 F F F
151322024-02-03T17:16:03.758Z 22 440-459 138 138 136 239 239 237 F F F <---
151332024-02-03T17:16:03.758Z 23 460-479 138 138 130 241 241 223 F F F <---
151342024-02-03T17:16:03.758Z 24 480-499 135 135 135 235 235 235 F F F
151352024-02-03T17:16:03.758Z 25 500-519 131 131 131 227 227 227 F F F
151362024-02-03T17:16:03.758Z 26 520-539 138 138 136 241 241 237 F F F <---
151372024-02-03T17:16:03.758Z 27 540-559 138 138 131 241 241 227 F F F <---
151382024-02-03T17:16:03.758Z 28 560-579 134 134 134 234 234 234 T T F <---
151392024-02-03T17:16:03.758Z 29 580-599 137 137 137 238 238 238 F F F
151402024-02-03T17:16:03.758ZMax gen: 138, Max flush: 241
151412024-02-03T17:16:03.758ZError: Difference in extent metadata found!
151422024-02-03T17:16:03.758ZA Difference in extent metadata is expected here
151432024-02-03T17:16:03.758ZOn loop 69
151442024-02-03T17:16:03.758Z
151452024-02-03T17:16:03.758ZVerifying data now
151462024-02-03T17:16:09.212ZNo change after verify
151472024-02-03T17:16:09.212ZLoop: 69 Downstairs dump after verify (and repair):
151482024-02-03T17:16:09.235ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
151492024-02-03T17:16:09.235Z 0 000-019 136 136 136 237 237 237 F F F
151502024-02-03T17:16:09.235Z 1 020-039 136 136 136 237 237 237 F F F
151512024-02-03T17:16:09.235Z 2 040-059 130 130 130 225 225 225 F F F
151522024-02-03T17:16:09.235Z 3 060-079 135 135 135 235 235 235 F F F
151532024-02-03T17:16:09.245Z 4 080-099 132 132 132 230 230 230 F F F
151542024-02-03T17:16:09.245Z 5 100-119 138 138 138 241 241 241 F F F
151552024-02-03T17:16:09.245Z 6 120-139 138 138 138 241 241 241 F F F
151562024-02-03T17:16:09.245Z 7 140-159 136 136 136 237 237 237 F F F
151572024-02-03T17:16:09.245Z 8 160-179 137 137 137 238 238 238 F F F
151582024-02-03T17:16:09.245Z 9 180-199 138 138 138 239 239 239 F F F
151592024-02-03T17:16:09.245Z 10 200-219 139 139 139 242 242 242 F F F
151602024-02-03T17:16:09.245Z 11 220-239 139 139 139 242 242 242 F F F
151612024-02-03T17:16:09.246Z 12 240-259 138 138 138 240 240 240 F F F
151622024-02-03T17:16:09.246Z 13 260-279 139 139 139 242 242 242 F F F
151632024-02-03T17:16:09.246Z 14 280-299 139 139 139 242 242 242 F F F
151642024-02-03T17:16:09.246Z 15 300-319 135 135 135 235 235 235 F F F
151652024-02-03T17:16:09.246Z 16 320-339 139 139 139 242 242 242 F F F
151662024-02-03T17:16:09.246Z 17 340-359 139 139 139 242 242 242 F F F
151672024-02-03T17:16:09.246Z 18 360-379 139 139 139 242 242 242 F F F
151682024-02-03T17:16:09.246Z 19 380-399 134 134 134 234 234 234 F F F
151692024-02-03T17:16:09.246Z 20 400-419 138 138 138 239 239 239 F F F
151702024-02-03T17:16:09.246Z 21 420-439 135 135 135 235 235 235 F F F
151712024-02-03T17:16:09.246Z 22 440-459 138 138 138 239 239 239 F F F
151722024-02-03T17:16:09.246Z 23 460-479 138 138 138 241 241 241 F F F
151732024-02-03T17:16:09.247Z 24 480-499 135 135 135 235 235 235 F F F
151742024-02-03T17:16:09.247Z 25 500-519 131 131 131 227 227 227 F F F
151752024-02-03T17:16:09.247Z 26 520-539 138 138 138 241 241 241 F F F
151762024-02-03T17:16:09.247Z 27 540-559 138 138 138 241 241 241 F F F
151772024-02-03T17:16:09.247Z 28 560-579 139 139 139 242 242 242 F F F
151782024-02-03T17:16:09.247Z 29 580-599 137 137 137 238 238 238 F F F
151792024-02-03T17:16:09.247ZMax gen: 139, Max flush: 242
151802024-02-03T17:16:09.247Z
151812024-02-03T17:16:09.247ZBegin loop 70
151822024-02-03T17:16:09.247ZDownstairs to restart: 0
151832024-02-03T17:16:09.264Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.833985199Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695}
151842024-02-03T17:16:09.264Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.834531484Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695}
151852024-02-03T17:16:09.271Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.834561966Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695}
151862024-02-03T17:16:09.271Z{"msg":"Crucible 8494480e-7f3e-42f8-989c-1cf789211002 has session id: 2b9a9eb4-711f-46b2-bfd1-3f546e45af18","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.834678748Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":Crucible runtime is spawned
151872024-02-03T17:16:09.271Z1695,"session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
151882024-02-03T17:16:09.271Z{"msg":"Upstairs opts: Upstairs UUID: 8494480e-7f3e-42f8-989c-1cf789211002, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.834838251Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
151892024-02-03T17:16:09.271Z{"msg":"Crucible stats registered with UUID: 8494480e-7f3e-42f8-989c-1cf789211002","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.834921511Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
151902024-02-03T17:16:09.271Z{"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.834963733Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695}
151912024-02-03T17:16:09.271Z{"msg":"8494480e-7f3e-42f8-989c-1cf789211002 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.835155782Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
151922024-02-03T17:16:09.271Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.835228131Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
151932024-02-03T17:16:09.271Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.835256731Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
151942024-02-03T17:16:09.272Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.835280211Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
151952024-02-03T17:16:09.272Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.835360955Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
151962024-02-03T17:16:09.272Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.835411242Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
151972024-02-03T17:16:09.272Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.835569548Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
151982024-02-03T17:16:09.272Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.835607646Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"":"io task","client":"2","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
151992024-02-03T17:16:09.272Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.835632496Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"":"io task","client":"0","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152002024-02-03T17:16:09.272Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.835655858Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"":"io task","client":"1","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152012024-02-03T17:16:09.272Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.835692323Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"":"io task","client":"0","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152022024-02-03T17:16:09.272Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.835733676Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"":"io task","client":"1","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152032024-02-03T17:16:09.272Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.835959734Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"":"io task","client":"2","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152042024-02-03T17:16:09.272Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.83721825Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152052024-02-03T17:16:09.272Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.837348413Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152062024-02-03T17:16:09.272Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.837479858Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152072024-02-03T17:16:09.272Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.837506679Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152082024-02-03T17:16:09.272Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.837528985Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152092024-02-03T17:16:09.272Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.837583878Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152102024-02-03T17:16:09.277Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.838245757Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152112024-02-03T17:16:09.277Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.838746584Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152122024-02-03T17:16:09.277Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.839216083Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152132024-02-03T17:16:09.277Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.839610135Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152142024-02-03T17:16:09.277Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.8396913Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152152024-02-03T17:16:09.278Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.839721112Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152162024-02-03T17:16:09.278Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.83974254Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152172024-02-03T17:16:09.278Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.83979144Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152182024-02-03T17:16:09.278Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.839816746Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152192024-02-03T17:16:09.278Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.840084375Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152202024-02-03T17:16:09.278Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.840146731Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152212024-02-03T17:16:09.278Z{"msg":"[0]R flush_numbers[0..12]: [237, 237, 225, 235, 230, 241, 241, 237, 238, 239, 242, 242]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.840174036Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152222024-02-03T17:16:09.278Z{"msg":"[0]R generation[0..12]: [136, 136, 130, 135, 132, 138, 138, 136, 137, 138, 139, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.84019796Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152232024-02-03T17:16:09.278Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.840222186Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152242024-02-03T17:16:09.278Z{"msg":"[1]R flush_numbers[0..12]: [237, 237, 225, 235, 230, 241, 241, 237, 238, 239, 242, 242]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.84027178Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152252024-02-03T17:16:09.278Z{"msg":"[1]R generation[0..12]: [136, 136, 130, 135, 132, 138, 138, 136, 137, 138, 139, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.840297927Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152262024-02-03T17:16:09.278Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.840346707Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152272024-02-03T17:16:09.278Z{"msg":"[2]R flush_numbers[0..12]: [237, 237, 225, 235, 230, 241, 241, 237, 238, 239, 242, 242]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.840372782Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152282024-02-03T17:16:09.278Z{"msg":"[2]R generation[0..12]: [136, 136, 130, 135, 132, 138, 138, 136, 137, 138, 139, 139]"Wait for a query_work_queue command to finish before sending IO
152292024-02-03T17:16:09.278Z,"v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.840425915Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152302024-02-03T17:16:09.278Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.840452862Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152312024-02-03T17:16:09.278Z{"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.840510383Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152322024-02-03T17:16:09.278Z{"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.840537055Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152332024-02-03T17:16:09.278Z{"msg":"Next flush: 243","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.840587585Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152342024-02-03T17:16:09.278Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.840612294Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152352024-02-03T17:16:09.278Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.840660939Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152362024-02-03T17:16:09.278Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.840683461Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152372024-02-03T17:16:09.278Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.840705538Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152382024-02-03T17:16:09.279Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.840727608Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152392024-02-03T17:16:09.279Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.840774558Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152402024-02-03T17:16:09.279Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.840797744Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152412024-02-03T17:16:09.279Z{"msg":"8494480e-7f3e-42f8-989c-1cf789211002 is now active with session: 2b9a9eb4-711f-46b2-bfd1-3f546e45af18","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.840820992Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152422024-02-03T17:16:09.279Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.840865837Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695,"session_id":"2b9a9eb4-711f-46b2-bfd1-3f546e45af18"}
152432024-02-03T17:16:09.279Z{"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:08.840888954Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695}
152442024-02-03T17:16:09.279ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
152452024-02-03T17:16:09.279ZLoading write count information from file "/tmp/test_repair_verify.data"
152462024-02-03T17:16:09.279ZRun Repair workload
152472024-02-03T17:16:09.279Z01/30 Write block 0 len 28672 data: 3 4 7 8 9 9 12
152482024-02-03T17:16:09.279Z02/30 Write block 314 len 28672 data: 12 13 13 14 11 14 13
152492024-02-03T17:16:09.282Z03/30 Write block 375 len 40960 data: 12 14 15 12 14 12 10 11 11 10
152502024-02-03T17:16:09.282Z04/30 Read block 402 len 20480
152512024-02-03T17:16:09.282Z05/30 Write block 246 len 32768 data: 10 7 12 12 11 11 10 7
152522024-02-03T17:16:09.282Z06/30 Read block 272 len 32768
152532024-02-03T17:16:09.289Z07/30 Read block 209 len 24576
152542024-02-03T17:16:09.292Z08/30 Flush
152552024-02-03T17:16:09.292Z09/30 Write block 100 len 4096 data: 11
152562024-02-03T17:16:09.292Z10/30 Flush
152572024-02-03T17:16:09.295Z11/30 Write block 325 len 36864 data: 11 9 14 13 13 12 12 9 10
152582024-02-03T17:16:09.298Z12/30 Write block 471 len 8192 data: 9 9
152592024-02-03T17:16:09.298Z13/30 Read block 376 len 28672
152602024-02-03T17:16:09.302Z14/30 Write block 150 len 16384 data: 10 12 13 10
152612024-02-03T17:16:09.302Z15/30 Write block 567 len 24576 data: 11 14 12 12 10 9
152622024-02-03T17:16:09.302Z16/30 Read block 183 len 28672
152632024-02-03T17:16:09.305Z17/30 Read block 109 len 8192
152642024-02-03T17:16:09.305Z18/30 Read block 509 len 40960
152652024-02-03T17:16:09.309Z19/30 Read block 529 len 40960
152662024-02-03T17:16:09.309Z20/30 Write block 272 len 8192 data: 14 13
152672024-02-03T17:16:09.309Z21/30 Write block 315 len 16384 data: 14 14 15 12
152682024-02-03T17:16:09.312Z22/30 Read block 228 len 8192
152692024-02-03T17:16:09.312Z23/30 Write block 455 len 28672 data: 7 7 8 10 12 15 13
152702024-02-03T17:16:09.312Z24/30 Read block 130 len 40960
152712024-02-03T17:16:09.315Z25/30 Read block 568 len 12288
152722024-02-03T17:16:09.318Z26/30 Read block 273 len 36864
152732024-02-03T17:16:09.318Z27/30 Read block 548 len 20480
152742024-02-03T17:16:09.321Z28/30 Write block 288 len 16384 data: 10 12 11 10
152752024-02-03T17:16:09.321Z29/30 Read block 244 len 24576
152762024-02-03T17:16:09.324Z30/30 Read block 20 len 24576
152772024-02-03T17:16:09.325Z----------------------------------------------------------------
152782024-02-03T17:16:09.325Z Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30
152792024-02-03T17:16:09.325ZStates: Active Active Active
152802024-02-03T17:16:09.325ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
152812024-02-03T17:16:09.325Z 1 Acked 1000 Write 7 Sent Done Done false
152822024-02-03T17:16:09.325Z 2 Acked 1001 Write 7 Sent Done Done false
152832024-02-03T17:16:09.325Z 3 Acked 1002 Write 10 Sent Done Done false
152842024-02-03T17:16:09.325Z 4 Acked 1003 Read 5 Sent Done Done false
152852024-02-03T17:16:09.325Z 5 Acked 1004 Write 8 Sent Done Done false
152862024-02-03T17:16:09.325Z 6 Acked 1005 Read 8 Sent Done Done false
152872024-02-03T17:16:09.325Z 7 Acked 1006 Read 6 Sent Done Done false
152882024-02-03T17:16:09.325Z 8 Acked 1007 Flush 0 Sent Done Done false
152892024-02-03T17:16:09.325Z 9 Acked 1008 Write 1 Sent Done Done false
152902024-02-03T17:16:09.325Z 10 Acked 1009 Flush 0 Sent Done Done false
152912024-02-03T17:16:09.325Z 11 Acked 1010 Write 9 Sent Done Done false
152922024-02-03T17:16:09.325Z 12 Acked 1011 Write 2 Sent Done Done false
152932024-02-03T17:16:09.325Z 13 Acked 1012 Read 7 Sent Done Done false
152942024-02-03T17:16:09.325Z 14 Acked 1013 Write 4 Sent Done Done false
152952024-02-03T17:16:09.325Z 15 Acked 1014 Write 6 Sent Done Done false
152962024-02-03T17:16:09.325Z 16 Acked 1015 Read 7 Sent Done Done false
152972024-02-03T17:16:09.325Z 17 Acked 1016 Read 2 Sent Done Done false
152982024-02-03T17:16:09.325Z 18 Acked 1017 Read 10 Sent Done Done false
152992024-02-03T17:16:09.325Z 19 Acked 1018 Read 10 Sent Done Done false
153002024-02-03T17:16:09.325Z 20 Acked 1019 Write 2 Sent Done Done false
153012024-02-03T17:16:09.326Z 21 Acked 1020 Write 4 Sent Done Done false
153022024-02-03T17:16:09.326Z 22 Acked 1021 Read 2 Sent Done Done false
153032024-02-03T17:16:09.326Z 23 Acked 1022 Write 7 Sent Done Done false
153042024-02-03T17:16:09.326Z 24 Acked 1023 Read 10 Sent Done Done false
153052024-02-03T17:16:09.326Z 25 Acked 1024 Read 3 Sent Done Done false
153062024-02-03T17:16:09.326Z 26 Acked 1025 Read 9 Sent Done Done false
153072024-02-03T17:16:09.326Z 27 Acked 1026 Read 5 Sent Done Done false
153082024-02-03T17:16:09.326Z 28 Acked 1027 Write 4 Sent Done Done false
153092024-02-03T17:16:09.326Z 29 Acked 1028 Read 6 Sent Done Done false
153102024-02-03T17:16:09.326Z 30 Acked 1029 Read 6 Sent Sent Done false
153112024-02-03T17:16:09.326Z STATES DS:0 DS:1 DS:2 TOTAL
153122024-02-03T17:16:09.326Z New 0 0 0 0
153132024-02-03T17:16:09.326Z Sent 30 1 0 31
153142024-02-03T17:16:09.326Z Done 0 29 30 59
153152024-02-03T17:16:09.326Z Skipped 0 0 0 0
153162024-02-03T17:16:09.326Z Error 0 0 0 0
153172024-02-03T17:16:09.326ZLast Flush: 0 1009 1009
153182024-02-03T17:16:09.326ZDownstairs last five completed:
153192024-02-03T17:16:09.326ZUpstairs last five completed: 30 29 28 27 26
153202024-02-03T17:16:09.326Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:16:08.894166216Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1695}
153212024-02-03T17:16:09.326ZWrote out file "/tmp/test_repair_verify.data"
153222024-02-03T17:16:09.330Z
153232024-02-03T17:16:09.336ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
153242024-02-03T17:16:09.366ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
153252024-02-03T17:16:09.366Z 0 000-019 136 140 140 237 243 243 F F F <---
153262024-02-03T17:16:09.366Z 1 020-039 136 136 136 237 237 237 F F F
153272024-02-03T17:16:09.366Z 2 040-059 130 130 130 225 225 225 F F F
153282024-02-03T17:16:09.366Z 3 060-079 135 135 135 235 235 235 F F F
153292024-02-03T17:16:09.366Z 4 080-099 132 132 132 230 230 230 F F F
153302024-02-03T17:16:09.366Z 5 100-119 138 140 140 241 244 244 F F F <---
153312024-02-03T17:16:09.366Z 6 120-139 138 138 138 241 241 241 F F F
153322024-02-03T17:16:09.366Z 7 140-159 136 136 136 237 237 237 F T T <---
153332024-02-03T17:16:09.366Z 8 160-179 137 137 137 238 238 238 F F F
153342024-02-03T17:16:09.366Z 9 180-199 138 138 138 239 239 239 F F F
153352024-02-03T17:16:09.366Z 10 200-219 139 139 139 242 242 242 F F F
153362024-02-03T17:16:09.370Z 11 220-239 139 139 139 242 242 242 F F F
153372024-02-03T17:16:09.370Z 12 240-259 138 140 140 240 243 243 F F F <---
153382024-02-03T17:16:09.370Z 13 260-279 139 139 139 242 242 242 F T T <---
153392024-02-03T17:16:09.370Z 14 280-299 139 139 139 242 242 242 F T T <---
153402024-02-03T17:16:09.370Z 15 300-319 135 140 140 235 243 243 F T T <---
153412024-02-03T17:16:09.370Z 16 320-339 139 140 140 242 243 243 F T T <---
153422024-02-03T17:16:09.370Z 17 340-359 139 139 139 242 242 242 F F F
153432024-02-03T17:16:09.370Z 18 360-379 139 140 140 242 243 243 F F F <---
153442024-02-03T17:16:09.370Z 19 380-399 134 140 140 234 243 243 F F F <---
153452024-02-03T17:16:09.370Z 20 400-419 138 138 138 239 239 239 F F F
153462024-02-03T17:16:09.370Z 21 420-439 135 135 135 235 235 235 F F F
153472024-02-03T17:16:09.370Z 22 440-459 138 138 138 239 239 239 F T T <---
153482024-02-03T17:16:09.370Z 23 460-479 138 138 138 241 241 241 F T T <---
153492024-02-03T17:16:09.370Z 24 480-499 135 135 135 235 235 235 F F F
153502024-02-03T17:16:09.370Z 25 500-519 131 131 131 227 227 227 F F F
153512024-02-03T17:16:09.370Z 26 520-539 138 138 138 241 241 241 F F F
153522024-02-03T17:16:09.370Z 27 540-559 138 138 138 241 241 241 F F F
153532024-02-03T17:16:09.370Z 28 560-579 139 139 139 242 242 242 F T T <---
153542024-02-03T17:16:09.370Z 29 580-599 137 137 137 238 238 238 F F F
153552024-02-03T17:16:09.371ZMax gen: 140, Max flush: 244
153562024-02-03T17:16:09.371ZError: Difference in extent metadata found!
153572024-02-03T17:16:09.371ZA Difference in extent metadata is expected here
153582024-02-03T17:16:09.371ZOn loop 70
153592024-02-03T17:16:09.371Z
153602024-02-03T17:16:09.374ZVerifying data now
153612024-02-03T17:16:14.523ZNo change after verify
153622024-02-03T17:16:14.523ZLoop: 70 Downstairs dump after verify (and repair):
153632024-02-03T17:16:14.549ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
153642024-02-03T17:16:14.549Z 0 000-019 140 140 140 243 243 243 F F F
153652024-02-03T17:16:14.549Z 1 020-039 136 136 136 237 237 237 F F F
153662024-02-03T17:16:14.549Z 2 040-059 130 130 130 225 225 225 F F F
153672024-02-03T17:16:14.549Z 3 060-079 135 135 135 235 235 235 F F F
153682024-02-03T17:16:14.549Z 4 080-099 132 132 132 230 230 230 F F F
153692024-02-03T17:16:14.549Z 5 100-119 140 140 140 244 244 244 F F F
153702024-02-03T17:16:14.556Z 6 120-139 138 138 138 241 241 241 F F F
153712024-02-03T17:16:14.556Z 7 140-159 141 141 141 245 245 245 F F F
153722024-02-03T17:16:14.556Z 8 160-179 137 137 137 238 238 238 F F F
153732024-02-03T17:16:14.556Z 9 180-199 138 138 138 239 239 239 F F F
153742024-02-03T17:16:14.556Z 10 200-219 139 139 139 242 242 242 F F F
153752024-02-03T17:16:14.556Z 11 220-239 139 139 139 242 242 242 F F F
153762024-02-03T17:16:14.556Z 12 240-259 140 140 140 243 243 243 F F F
153772024-02-03T17:16:14.556Z 13 260-279 141 141 141 245 245 245 F F F
153782024-02-03T17:16:14.556Z 14 280-299 141 141 141 245 245 245 F F F
153792024-02-03T17:16:14.556Z 15 300-319 141 141 141 245 245 245 F F F
153802024-02-03T17:16:14.556Z 16 320-339 141 141 141 245 245 245 F F F
153812024-02-03T17:16:14.556Z 17 340-359 139 139 139 242 242 242 F F F
153822024-02-03T17:16:14.556Z 18 360-379 140 140 140 243 243 243 F F F
153832024-02-03T17:16:14.556Z 19 380-399 140 140 140 243 243 243 F F F
153842024-02-03T17:16:14.556Z 20 400-419 138 138 138 239 239 239 F F F
153852024-02-03T17:16:14.556Z 21 420-439 135 135 135 235 235 235 F F F
153862024-02-03T17:16:14.556Z 22 440-459 141 141 141 245 245 245 F F F
153872024-02-03T17:16:14.556Z 23 460-479 141 141 141 245 245 245 F F F
153882024-02-03T17:16:14.556Z 24 480-499 135 135 135 235 235 235 F F F
153892024-02-03T17:16:14.557Z 25 500-519 131 131 131 227 227 227 F F F
153902024-02-03T17:16:14.557Z 26 520-539 138 138 138 241 241 241 F F F
153912024-02-03T17:16:14.557Z 27 540-559 138 138 138 241 241 241 F F F
153922024-02-03T17:16:14.557Z 28 560-579 141 141 141 245 245 245 F F F
153932024-02-03T17:16:14.557Z 29 580-599 137 137 137 238 238 238 F F F
153942024-02-03T17:16:14.557ZMax gen: 141, Max flush: 245
153952024-02-03T17:16:14.557Z
153962024-02-03T17:16:14.557ZBegin loop 71
153972024-02-03T17:16:14.557ZDownstairs to restart: 1
153982024-02-03T17:16:14.593Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.153035933Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703}
153992024-02-03T17:16:14.593Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.153690004Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703}
154002024-02-03T17:16:14.593Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.15374787Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703}
154012024-02-03T17:16:14.604Z{"msg":"Crucible 3cdd83b6-715b-4b03-96a6-4dd263c62ade has session id: fed904d4-9dbb-4edd-b30a-082f740f557e","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.153778331Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154022024-02-03T17:16:14.604Z{"msg":"Upstairs opts: Upstairs UUID: 3cdd83b6-715b-4b03-96a6-4dd263c62ade, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.15397808Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154032024-02-03T17:16:14.604Z{"msg":"Crucible stats registered with UUID: 3cdd83b6-715b-4b03-96a6-4dd263c62ade","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.154028005Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154042024-02-03T17:16:14.604ZCrucible runtime is spawned
154052024-02-03T17:16:14.604Z{"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.154290791Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703}
154062024-02-03T17:16:14.604Z{"msg":"3cdd83b6-715b-4b03-96a6-4dd263c62ade active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.154401631Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154072024-02-03T17:16:14.604Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.154468811Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154082024-02-03T17:16:14.604Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.15449705Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154092024-02-03T17:16:14.604Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.154519831Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154102024-02-03T17:16:14.604Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.154576973Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154112024-02-03T17:16:14.604Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.154647583Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154122024-02-03T17:16:14.604Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.154676372Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154132024-02-03T17:16:14.604Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.154698373Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"":"io task","client":"2","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154142024-02-03T17:16:14.605Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.154836944Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"":"io task","client":"0","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154152024-02-03T17:16:14.605Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.155023658Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"":"io task","client":"1","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154162024-02-03T17:16:14.605Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.155122787Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"":"io task","client":"2","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154172024-02-03T17:16:14.605Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.155186223Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"":"io task","client":"0","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154182024-02-03T17:16:14.605Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.155262283Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"":"io task","client":"1","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154192024-02-03T17:16:14.605Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.156656354Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154202024-02-03T17:16:14.605Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.15673041Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154212024-02-03T17:16:14.605Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.15675809Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154222024-02-03T17:16:14.605Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.156780725Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154232024-02-03T17:16:14.605Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.156832754Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154242024-02-03T17:16:14.605Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.156933874Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154252024-02-03T17:16:14.605Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.157965728Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154262024-02-03T17:16:14.605Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.158394443Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154272024-02-03T17:16:14.605Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.158479269Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154282024-02-03T17:16:14.605Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.158878555Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154292024-02-03T17:16:14.605Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.159048144Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154302024-02-03T17:16:14.605Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.159096555Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154312024-02-03T17:16:14.605Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.159349199Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154322024-02-03T17:16:14.605Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.15949927Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154332024-02-03T17:16:14.605Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.159649537Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154342024-02-03T17:16:14.605Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.159686227Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154352024-02-03T17:16:14.605Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.159713858Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154362024-02-03T17:16:14.605Z{"msg":"[0]R flush_numbers[0..12]: [243, 237, 225, 235, 230, 244, 241, 245, 238, 239, 242, 242]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.159772626Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154372024-02-03T17:16:14.605Z{"msg":"[0]R generation[0..12]: [140, 136, 130, 135, 132, 140, 138, 141, 137, 138, 139, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.159799517Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154382024-02-03T17:16:14.606Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.15985158Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154392024-02-03T17:16:14.606Z{"msg":"[1]R flush_numbers[0..12]: [243, 237, 225, 235, 230, 244, 241, 245, 238, 239, 242, 242]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.159877322Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154402024-02-03T17:16:14.606Z{"Wait for a query_work_queue command to finish before sending IO
154412024-02-03T17:16:14.606Zmsg":"[1]R generation[0..12]: [140, 136, 130, 135, 132, 140, 138, 141, 137, 138, 139, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.159956945Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154422024-02-03T17:16:14.606Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.160040397Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154432024-02-03T17:16:14.606Z{"msg":"[2]R flush_numbers[0..12]: [243, 237, 225, 235, 230, 244, 241, 245, 238, 239, 242, 242]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.160070572Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154442024-02-03T17:16:14.606Z{"msg":"[2]R generation[0..12]: [140, 136, 130, 135, 132, 140, 138, 141, 137, 138, 139, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.160124661Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154452024-02-03T17:16:14.606Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.160222165Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154462024-02-03T17:16:14.606Z{"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.160297823Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154472024-02-03T17:16:14.606Z{"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.160324532Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154482024-02-03T17:16:14.606Z{"msg":"Next flush: 246","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.160386317Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154492024-02-03T17:16:14.606Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.160430282Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154502024-02-03T17:16:14.606Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.160504594Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154512024-02-03T17:16:14.606Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.160604682Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154522024-02-03T17:16:14.606Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.160646488Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154532024-02-03T17:16:14.606Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.160719158Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154542024-02-03T17:16:14.606Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.160746483Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154552024-02-03T17:16:14.606Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.160767308Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154562024-02-03T17:16:14.606Z{"msg":"3cdd83b6-715b-4b03-96a6-4dd263c62ade is now active with session: fed904d4-9dbb-4edd-b30a-082f740f557e","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.160820941Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154572024-02-03T17:16:14.606Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.160858437Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703,"session_id":"fed904d4-9dbb-4edd-b30a-082f740f557e"}
154582024-02-03T17:16:14.606Z{"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:14.160920281Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703}
154592024-02-03T17:16:14.606ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
154602024-02-03T17:16:14.606ZLoading write count information from file "/tmp/test_repair_verify.data"
154612024-02-03T17:16:14.606ZRun Repair workload
154622024-02-03T17:16:14.607Z01/30 Write block 154 len 8192 data: 9 9
154632024-02-03T17:16:14.607Z02/30 Write block 87 len 12288 data: 9 8 10
154642024-02-03T17:16:14.607Z03/30 Write block 142 len 20480 data: 15 13 9 8 8
154652024-02-03T17:16:14.607Z04/30 Read block 226 len 40960
154662024-02-03T17:16:14.607Z05/30 Read block 30 len 28672
154672024-02-03T17:16:14.607Z06/30 Write block 495 len 12288 data: 8 8 8
154682024-02-03T17:16:14.610Z07/30 Write block 516 len 28672 data: 13 11 8 8 9 7 9
154692024-02-03T17:16:14.610Z08/30 Write block 49 len 40960 data: 12 11 11 12 9 9 9 10 9 7
154702024-02-03T17:16:14.610Z09/30 Read block 42 len 8192
154712024-02-03T17:16:14.610Z10/30 Read block 433 len 24576
154722024-02-03T17:16:14.611Z11/30 Read block 220 len 16384
154732024-02-03T17:16:14.611Z12/30 Write block 285 len 40960 data: 14 14 13 11 13 12 11 9 10 11
154742024-02-03T17:16:14.611Z13/30 Flush
154752024-02-03T17:16:14.615Z14/30 Write block 14 len 8192 data: 13 10
154762024-02-03T17:16:14.621Z15/30 Read block 238 len 24576
154772024-02-03T17:16:14.621Z16/30 Read block 355 len 28672
154782024-02-03T17:16:14.624Z17/30 Write block 228 len 24576 data: 10 7 7 9 9 9
154792024-02-03T17:16:14.624Z18/30 Read block 382 len 28672
154802024-02-03T17:16:14.624Z19/30 Read block 355 len 32768
154812024-02-03T17:16:14.627Z20/30 Write block 426 len 32768 data: 9 12 12 15 15 12 13 10
154822024-02-03T17:16:14.627Z21/30 Write block 490 len 28672 data: 6 8 7 7 8 9 9
154832024-02-03T17:16:14.627Z22/30 Read block 519 len 32768
154842024-02-03T17:16:14.630Z23/30 Flush
154852024-02-03T17:16:14.633Z24/30 Flush
154862024-02-03T17:16:14.633Z25/30 Write block 126 len 4096 data: 7
154872024-02-03T17:16:14.636Z26/30 Write block 305 len 32768 data: 10 11 9 8 10 12 10 11
154882024-02-03T17:16:14.636Z27/30 Write block 392 len 8192 data: 17 16
154892024-02-03T17:16:14.636Z28/30 Read block 333 len 36864
154902024-02-03T17:16:14.639Z29/30 Write block 390 len 36864 data: 12 13 18 17 13 11 11 9 12
154912024-02-03T17:16:14.643Z30/30 Read block 298 len 32768
154922024-02-03T17:16:14.643Z----------------------------------------------------------------
154932024-02-03T17:16:14.643Z Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:30
154942024-02-03T17:16:14.643ZStates: Active Active Active
154952024-02-03T17:16:14.643ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
154962024-02-03T17:16:14.643Z 1 Acked 1000 Write 2 Done Done Done false
154972024-02-03T17:16:14.643Z 2 Acked 1001 Write 3 Done Done Done false
154982024-02-03T17:16:14.643Z 3 Acked 1002 Write 5 Done Done Done false
154992024-02-03T17:16:14.643Z 4 Acked 1003 Read 10 Done Done Done false
155002024-02-03T17:16:14.643Z 5 Acked 1004 Read 7 Done Sent Done false
155012024-02-03T17:16:14.643Z 6 Acked 1005 Write 3 Done Sent Done false
155022024-02-03T17:16:14.643Z 7 Acked 1006 Write 7 Done Sent Done false
155032024-02-03T17:16:14.643Z 8 Acked 1007 Write 10 Done Sent Done false
155042024-02-03T17:16:14.643Z 9 Acked 1008 Read 2 Done Sent Done false
155052024-02-03T17:16:14.643Z 10 Acked 1009 Read 6 Done Sent Done false
155062024-02-03T17:16:14.643Z 11 Acked 1010 Read 4 Done Sent Done false
155072024-02-03T17:16:14.643Z 12 Acked 1011 Write 10 Done Sent Done false
155082024-02-03T17:16:14.643Z 13 Acked 1012 Flush 0 Done Sent Done false
155092024-02-03T17:16:14.644Z 14 Acked 1013 Write 2 Done Sent Done false
155102024-02-03T17:16:14.644Z 15 Acked 1014 Read 6 Done Sent Done false
155112024-02-03T17:16:14.644Z 16 Acked 1015 Read 7 Done Sent Done false
155122024-02-03T17:16:14.644Z 17 Acked 1016 Write 6 Done Sent Done false
155132024-02-03T17:16:14.644Z 18 Acked 1017 Read 7 Done Sent Done false
155142024-02-03T17:16:14.644Z 19 Acked 1018 Read 8 Done Sent Done false
155152024-02-03T17:16:14.644Z 20 Acked 1019 Write 8 Done Sent Done false
155162024-02-03T17:16:14.644Z 21 Acked 1020 Write 7 Done Sent Done false
155172024-02-03T17:16:14.644Z 22 Acked 1021 Read 8 Done Sent Done false
155182024-02-03T17:16:14.644Z 23 Acked 1022 Flush 0 Done Sent Done false
155192024-02-03T17:16:14.644Z 24 Acked 1023 Flush 0 Done Sent Done false
155202024-02-03T17:16:14.644Z 25 Acked 1024 Write 1 Done Sent Done false
155212024-02-03T17:16:14.644Z 26 Acked 1025 Write 8 Done Sent Done false
155222024-02-03T17:16:14.644Z 27 Acked 1026 Write 2 Done Sent Done false
155232024-02-03T17:16:14.644Z 28 Acked 1027 Read 9 Done Sent Done false
155242024-02-03T17:16:14.644Z 29 Acked 1028 Write 9 Done Sent Done false
155252024-02-03T17:16:14.644Z 30 Acked 1029 Read 8 Sent Sent Done false
155262024-02-03T17:16:14.644Z STATES DS:0 DS:1 DS:2 TOTAL
155272024-02-03T17:16:14.644Z New 0 0 0 0
155282024-02-03T17:16:14.644Z Sent 1 26 0 27
155292024-02-03T17:16:14.644Z Done 29 4 30 63
155302024-02-03T17:16:14.644Z Skipped 0 0 0 0
155312024-02-03T17:16:14.644Z Error 0 0 0 0
155322024-02-03T17:16:14.644ZLast Flush: 1023 0 1023
155332024-02-03T17:16:14.644ZDownstairs last five completed:
155342024-02-03T17:16:14.645ZUpstairs last five completed: 30 29 28 27 26
155352024-02-03T17:16:14.648Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:16:14.213467222Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1703}
155362024-02-03T17:16:14.648ZWrote out file "/tmp/test_repair_verify.data"
155372024-02-03T17:16:14.651Z
155382024-02-03T17:16:14.656ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
155392024-02-03T17:16:14.681ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
155402024-02-03T17:16:14.682Z 0 000-019 142 140 142 247 243 247 F F F <---
155412024-02-03T17:16:14.682Z 1 020-039 136 136 136 237 237 237 F F F
155422024-02-03T17:16:14.682Z 2 040-059 142 130 142 246 225 246 F F F <---
155432024-02-03T17:16:14.682Z 3 060-079 135 135 135 235 235 235 F F F
155442024-02-03T17:16:14.682Z 4 080-099 142 132 142 246 230 246 F T F <---
155452024-02-03T17:16:14.682Z 5 100-119 140 140 140 244 244 244 F F F
155462024-02-03T17:16:14.682Z 6 120-139 138 138 138 241 241 241 T F T <---
155472024-02-03T17:16:14.682Z 7 140-159 142 141 142 246 245 246 F T F <---
155482024-02-03T17:16:14.682Z 8 160-179 137 137 137 238 238 238 F F F
155492024-02-03T17:16:14.685Z 9 180-199 138 138 138 239 239 239 F F F
155502024-02-03T17:16:14.685Z 10 200-219 139 139 139 242 242 242 F F F
155512024-02-03T17:16:14.685Z 11 220-239 142 139 142 247 242 247 F F F <---
155522024-02-03T17:16:14.685Z 12 240-259 140 140 140 243 243 243 F F F
155532024-02-03T17:16:14.685Z 13 260-279 141 141 141 245 245 245 F F F
155542024-02-03T17:16:14.685Z 14 280-299 142 141 142 246 245 246 F F F <---
155552024-02-03T17:16:14.685Z 15 300-319 141 141 141 245 245 245 T F T <---
155562024-02-03T17:16:14.685Z 16 320-339 141 141 141 245 245 245 F F F
155572024-02-03T17:16:14.685Z 17 340-359 139 139 139 242 242 242 F F F
155582024-02-03T17:16:14.685Z 18 360-379 140 140 140 243 243 243 F F F
155592024-02-03T17:16:14.685Z 19 380-399 140 140 140 243 243 243 T F T <---
155602024-02-03T17:16:14.685Z 20 400-419 138 138 138 239 239 239 F F F
155612024-02-03T17:16:14.685Z 21 420-439 142 135 142 247 235 247 F F F <---
155622024-02-03T17:16:14.686Z 22 440-459 141 141 141 245 245 245 F F F
155632024-02-03T17:16:14.686Z 23 460-479 141 141 141 245 245 245 F F F
155642024-02-03T17:16:14.686Z 24 480-499 142 135 142 247 235 247 F F F <---
155652024-02-03T17:16:14.686Z 25 500-519 142 131 142 246 227 246 F F F <---
155662024-02-03T17:16:14.686Z 26 520-539 142 138 142 246 241 246 F F F <---
155672024-02-03T17:16:14.686Z 27 540-559 138 138 138 241 241 241 F F F
155682024-02-03T17:16:14.686Z 28 560-579 141 141 141 245 245 245 F F F
155692024-02-03T17:16:14.686Z 29 580-599 137 137 137 238 238 238 F F F
155702024-02-03T17:16:14.686ZMax gen: 142, Max flush: 247
155712024-02-03T17:16:14.686ZError: Difference in extent metadata found!
155722024-02-03T17:16:14.689ZA Difference in extent metadata is expected here
155732024-02-03T17:16:14.689ZOn loop 71
155742024-02-03T17:16:14.689Z
155752024-02-03T17:16:14.692ZVerifying data now
155762024-02-03T17:16:20.815ZNo change after verify
155772024-02-03T17:16:20.815ZLoop: 71 Downstairs dump after verify (and repair):
155782024-02-03T17:16:20.838ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
155792024-02-03T17:16:20.838Z 0 000-019 142 142 142 247 247 247 F F F
155802024-02-03T17:16:20.839Z 1 020-039 136 136 136 237 237 237 F F F
155812024-02-03T17:16:20.839Z 2 040-059 142 142 142 246 246 246 F F F
155822024-02-03T17:16:20.839Z 3 060-079 135 135 135 235 235 235 F F F
155832024-02-03T17:16:20.839Z 4 080-099 142 142 142 246 246 246 F F F
155842024-02-03T17:16:20.839Z 5 100-119 140 140 140 244 244 244 F F F
155852024-02-03T17:16:20.839Z 6 120-139 143 143 143 248 248 248 F F F
155862024-02-03T17:16:20.839Z 7 140-159 142 142 142 246 246 246 F F F
155872024-02-03T17:16:20.839Z 8 160-179 137 137 137 238 238 238 F F F
155882024-02-03T17:16:20.842Z 9 180-199 138 138 138 239 239 239 F F F
155892024-02-03T17:16:20.842Z 10 200-219 139 139 139 242 242 242 F F F
155902024-02-03T17:16:20.842Z 11 220-239 142 142 142 247 247 247 F F F
155912024-02-03T17:16:20.842Z 12 240-259 140 140 140 243 243 243 F F F
155922024-02-03T17:16:20.842Z 13 260-279 141 141 141 245 245 245 F F F
155932024-02-03T17:16:20.842Z 14 280-299 142 142 142 246 246 246 F F F
155942024-02-03T17:16:20.842Z 15 300-319 143 143 143 248 248 248 F F F
155952024-02-03T17:16:20.843Z 16 320-339 141 141 141 245 245 245 F F F
155962024-02-03T17:16:20.843Z 17 340-359 139 139 139 242 242 242 F F F
155972024-02-03T17:16:20.843Z 18 360-379 140 140 140 243 243 243 F F F
155982024-02-03T17:16:20.843Z 19 380-399 143 143 143 248 248 248 F F F
155992024-02-03T17:16:20.843Z 20 400-419 138 138 138 239 239 239 F F F
156002024-02-03T17:16:20.843Z 21 420-439 142 142 142 247 247 247 F F F
156012024-02-03T17:16:20.843Z 22 440-459 141 141 141 245 245 245 F F F
156022024-02-03T17:16:20.843Z 23 460-479 141 141 141 245 245 245 F F F
156032024-02-03T17:16:20.843Z 24 480-499 142 142 142 247 247 247 F F F
156042024-02-03T17:16:20.843Z 25 500-519 142 142 142 246 246 246 F F F
156052024-02-03T17:16:20.843Z 26 520-539 142 142 142 246 246 246 F F F
156062024-02-03T17:16:20.843Z 27 540-559 138 138 138 241 241 241 F F F
156072024-02-03T17:16:20.843Z 28 560-579 141 141 141 245 245 245 F F F
156082024-02-03T17:16:20.843Z 29 580-599 137 137 137 238 238 238 F F F
156092024-02-03T17:16:20.843ZMax gen: 143, Max flush: 248
156102024-02-03T17:16:20.846Z
156112024-02-03T17:16:20.846ZBegin loop 72
156122024-02-03T17:16:20.846ZDownstairs to restart: 1
156132024-02-03T17:16:20.879Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.447067914Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711}
156142024-02-03T17:16:20.879Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.447545089Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711}
156152024-02-03T17:16:20.885Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.44757385Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711}
156162024-02-03T17:16:20.885Z{"msg":"Crucible db309c0f-e4e0-4973-a6d5-728a1db39264 has session id: 21c11d9c-a507-4b01-9234-0d4e7ff3aa64","v":0,"name":"crucible","level":30Crucible runtime is spawned
156172024-02-03T17:16:20.885Z,"time":"2024-02-03T17:16:20.447654736Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156182024-02-03T17:16:20.885Z{"msg":"Upstairs opts: Upstairs UUID: db309c0f-e4e0-4973-a6d5-728a1db39264, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.447749136Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156192024-02-03T17:16:20.885Z{"msg":"Crucible stats registered with UUID: db309c0f-e4e0-4973-a6d5-728a1db39264","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.447810492Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156202024-02-03T17:16:20.885Z{"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.447834724Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711}
156212024-02-03T17:16:20.885Z{"msg":"db309c0f-e4e0-4973-a6d5-728a1db39264 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.447945946Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156222024-02-03T17:16:20.885Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.44802122Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156232024-02-03T17:16:20.885Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.448049681Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156242024-02-03T17:16:20.885Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.448072223Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156252024-02-03T17:16:20.885Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.448149612Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156262024-02-03T17:16:20.885Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.448194291Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156272024-02-03T17:16:20.885Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.448240505Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156282024-02-03T17:16:20.885Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.448383777Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"":"io task","client":"0","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156292024-02-03T17:16:20.885Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.44867783Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"":"io task","client":"1","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156302024-02-03T17:16:20.885Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.448802453Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"":"io task","client":"2","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156312024-02-03T17:16:20.886Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.448850508Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"":"io task","client":"0","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156322024-02-03T17:16:20.886Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.448890577Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"":"io task","client":"2","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156332024-02-03T17:16:20.886Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.448932527Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"":"io task","client":"1","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156342024-02-03T17:16:20.886Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.450023195Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156352024-02-03T17:16:20.886Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.450155181Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156362024-02-03T17:16:20.886Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.450210677Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156372024-02-03T17:16:20.886Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.450262396Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156382024-02-03T17:16:20.886Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.450312891Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156392024-02-03T17:16:20.886Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.450453068Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156402024-02-03T17:16:20.886Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.451206286Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156412024-02-03T17:16:20.886Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.451269343Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156422024-02-03T17:16:20.886Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.451591131Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156432024-02-03T17:16:20.886Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.452244442Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156442024-02-03T17:16:20.886Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.452289886Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156452024-02-03T17:16:20.886Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.452359473Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156462024-02-03T17:16:20.886Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.452584174Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156472024-02-03T17:16:20.886Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.452707351Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156482024-02-03T17:16:20.886Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.452753895Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156492024-02-03T17:16:20.886Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.452791042Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156502024-02-03T17:16:20.886Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.452868851Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156512024-02-03T17:16:20.886Z{"msg":"[0]R flush_numbers[0..12]: [247, 237, 246, 235, 246, 244, 248, 246, 238, 239, 242, 247]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.45295247Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156522024-02-03T17:16:20.886Z{"msg":"[0]R generation[0..12]: [142, 136, 142, 135, 142, 140, 143, 142, 137, 138, 139, 142]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.453033845Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156532024-02-03T17:16:20.886Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.453118958Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156542024-02-03T17:16:20.886Z{"msg":"[1]R flush_numbers[0..12]: [247, 237, 246, 235, 246, 244, 248, 246, 238, 239, 242, 247]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
156552024-02-03T17:16:20.887Z,"time":"2024-02-03T17:16:20.453274171Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156562024-02-03T17:16:20.887Z{"msg":"[1]R generation[0..12]: [142, 136, 142, 135, 142, 140, 143, 142, 137, 138, 139, 142]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.453473377Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156572024-02-03T17:16:20.887Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.453517603Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156582024-02-03T17:16:20.887Z{"msg":"[2]R flush_numbers[0..12]: [247, 237, 246, 235, 246, 244, 248, 246, 238, 239, 242, 247]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.453584517Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156592024-02-03T17:16:20.887Z{"msg":"[2]R generation[0..12]: [142, 136, 142, 135, 142, 140, 143, 142, 137, 138, 139, 142]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.453612429Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156602024-02-03T17:16:20.887Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.453662969Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156612024-02-03T17:16:20.887Z{"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.453687091Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156622024-02-03T17:16:20.887Z{"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.453710804Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156632024-02-03T17:16:20.887Z{"msg":"Next flush: 249","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.453744376Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156642024-02-03T17:16:20.887Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.453805679Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156652024-02-03T17:16:20.887Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.453831768Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156662024-02-03T17:16:20.887Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.453853419Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156672024-02-03T17:16:20.887Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.453875951Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156682024-02-03T17:16:20.887Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.453933087Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156692024-02-03T17:16:20.887Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.45399963Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156702024-02-03T17:16:20.891Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.454025412Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156712024-02-03T17:16:20.891Z{"msg":"db309c0f-e4e0-4973-a6d5-728a1db39264 is now active with session: 21c11d9c-a507-4b01-9234-0d4e7ff3aa64","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.45404913Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156722024-02-03T17:16:20.891Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.454069418Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711,"session_id":"21c11d9c-a507-4b01-9234-0d4e7ff3aa64"}
156732024-02-03T17:16:20.891Z{"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:20.454089916Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711}
156742024-02-03T17:16:20.891ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
156752024-02-03T17:16:20.891ZLoading write count information from file "/tmp/test_repair_verify.data"
156762024-02-03T17:16:20.891ZRun Repair workload
156772024-02-03T17:16:20.891Z01/30 Write block 264 len 20480 data: 17 16 14 14 17
156782024-02-03T17:16:20.891Z02/30 Read block 400 len 36864
156792024-02-03T17:16:20.891Z03/30 Read block 440 len 24576
156802024-02-03T17:16:20.894Z04/30 Write block 176 len 24576 data: 9 13 11 11 11 14
156812024-02-03T17:16:20.894Z05/30 Read block 180 len 28672
156822024-02-03T17:16:20.894Z06/30 Read block 460 len 32768
156832024-02-03T17:16:20.897Z07/30 Read block 592 len 16384
156842024-02-03T17:16:20.897Z08/30 Write block 38 len 32768 data: 10 10 8 7 10 9 9 11
156852024-02-03T17:16:20.900Z09/30 Read block 109 len 16384
156862024-02-03T17:16:20.900Z10/30 Read block 313 len 20480
156872024-02-03T17:16:20.903Z11/30 Write block 115 len 40960 data: 8 10 14 12 11 10 9 10 8 7
156882024-02-03T17:16:20.906Z12/30 Write block 123 len 32768 data: 9 8 6 8 9 10 11 11
156892024-02-03T17:16:20.906Z13/30 Read block 146 len 4096
156902024-02-03T17:16:20.909Z14/30 Write block 413 len 40960 data: 11 12 12 11 12 11 10 10 12 10
156912024-02-03T17:16:20.909Z15/30 Read block 543 len 4096
156922024-02-03T17:16:20.909Z16/30 Flush
156932024-02-03T17:16:20.915Z17/30 Write block 55 len 20480 data: 10 11 10 8 9
156942024-02-03T17:16:20.918Z18/30 Write block 466 len 12288 data: 12 11 10
156952024-02-03T17:16:20.918Z19/30 Write block 433 len 4096 data: 11
156962024-02-03T17:16:20.918Z20/30 Read block 63 len 32768
156972024-02-03T17:16:20.921Z21/30 Flush
156982024-02-03T17:16:20.925Z22/30 Write block 190 len 36864 data: 21 16 17 13 14 15 14 12 11
156992024-02-03T17:16:20.925Z23/30 Read block 122 len 4096
157002024-02-03T17:16:20.928Z24/30 Read block 481 len 28672
157012024-02-03T17:16:20.928Z25/30 Read block 364 len 16384
157022024-02-03T17:16:20.931Z26/30 Flush
157032024-02-03T17:16:20.931Z27/30 Write block 439 len 28672 data: 14 15 16 18 15 12 14
157042024-02-03T17:16:20.934Z28/30 Write block 312 len 36864 data: 12 11 13 15 15 16 13 15 14
157052024-02-03T17:16:20.940Z29/30 Write block 394 len 36864 data: 14 12 12 10 13 14 12 9 8
157062024-02-03T17:16:20.940Z30/30 Write block 88 len 8192 data: 9 11
157072024-02-03T17:16:20.940Z----------------------------------------------------------------
157082024-02-03T17:16:20.940Z Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:30
157092024-02-03T17:16:20.940ZStates: Active Active Active
157102024-02-03T17:16:20.940ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
157112024-02-03T17:16:20.940Z 1 Acked 1000 Write 5 Done Sent Done false
157122024-02-03T17:16:20.940Z 2 Acked 1001 Read 9 Done Sent Done false
157132024-02-03T17:16:20.940Z 3 Acked 1002 Read 6 Done Sent Done false
157142024-02-03T17:16:20.940Z 4 Acked 1003 Write 6 Done Sent Done false
157152024-02-03T17:16:20.940Z 5 Acked 1004 Read 7 Done Sent Done false
157162024-02-03T17:16:20.940Z 6 Acked 1005 Read 8 Done Sent Done false
157172024-02-03T17:16:20.940Z 7 Acked 1006 Read 4 Done Sent Done false
157182024-02-03T17:16:20.940Z 8 Acked 1007 Write 8 Done Sent Done false
157192024-02-03T17:16:20.940Z 9 Acked 1008 Read 4 Done Sent Done false
157202024-02-03T17:16:20.941Z 10 Acked 1009 Read 5 Done Sent Done false
157212024-02-03T17:16:20.941Z 11 Acked 1010 Write 10 Done Sent Done false
157222024-02-03T17:16:20.941Z 12 Acked 1011 Write 8 Done Sent Done false
157232024-02-03T17:16:20.941Z 13 Acked 1012 Read 1 Done Sent Done false
157242024-02-03T17:16:20.941Z 14 Acked 1013 Write 10 Done Sent Done false
157252024-02-03T17:16:20.941Z 15 Acked 1014 Read 1 Done Sent Done false
157262024-02-03T17:16:20.941Z 16 Acked 1015 Flush 0 Done Sent Done false
157272024-02-03T17:16:20.941Z 17 Acked 1016 Write 5 Done Sent Done false
157282024-02-03T17:16:20.941Z 18 Acked 1017 Write 3 Done Sent Done false
157292024-02-03T17:16:20.941Z 19 Acked 1018 Write 1 Done Sent Done false
157302024-02-03T17:16:20.941Z 20 Acked 1019 Read 8 Done Sent Done false
157312024-02-03T17:16:20.941Z 21 Acked 1020 Flush 0 Done Sent Done false
157322024-02-03T17:16:20.941Z 22 Acked 1021 Write 9 Done Sent Done false
157332024-02-03T17:16:20.941Z 23 Acked 1022 Read 1 Done Sent Done false
157342024-02-03T17:16:20.941Z 24 Acked 1023 Read 7 Done Sent Done false
157352024-02-03T17:16:20.941Z 25 Acked 1024 Read 4 Done Sent Done false
157362024-02-03T17:16:20.941Z 26 Acked 1025 Flush 0 Done Sent Done false
157372024-02-03T17:16:20.941Z 27 Acked 1026 Write 7 Done Sent Done false
157382024-02-03T17:16:20.941Z 28 Acked 1027 Write 9 Done Sent Done false
157392024-02-03T17:16:20.941Z 29 Acked 1028 Write 9 Sent Sent Sent false
157402024-02-03T17:16:20.941Z 30 Acked 1029 Write 2 Sent Sent Sent false
157412024-02-03T17:16:20.941Z STATES DS:0 DS:1 DS:2 TOTAL
157422024-02-03T17:16:20.941Z New 0 0 0 0
157432024-02-03T17:16:20.941Z Sent 2 30 2 34
157442024-02-03T17:16:20.941Z Done 28 0 28 56
157452024-02-03T17:16:20.942Z Skipped 0 0 0 0
157462024-02-03T17:16:20.942Z Error 0 0 0 0
157472024-02-03T17:16:20.942ZLast Flush: 1025 0 1025
157482024-02-03T17:16:20.942ZDownstairs last five completed:
157492024-02-03T17:16:20.942ZUpstairs last five completed: 30 29 28 27 26
157502024-02-03T17:16:20.946Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:16:20.508222206Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1711}
157512024-02-03T17:16:20.946ZWrote out file "/tmp/test_repair_verify.data"
157522024-02-03T17:16:20.946Z
157532024-02-03T17:16:20.953ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
157542024-02-03T17:16:20.978ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
157552024-02-03T17:16:20.978Z 0 000-019 142 142 142 247 247 247 F F F
157562024-02-03T17:16:20.982Z 1 020-039 144 136 144 249 237 249 F F F <---
157572024-02-03T17:16:20.982Z 2 040-059 144 142 144 250 246 250 F F F <---
157582024-02-03T17:16:20.982Z 3 060-079 135 135 135 235 235 235 F F F
157592024-02-03T17:16:20.982Z 4 080-099 142 142 142 246 246 246 T F T <---
157602024-02-03T17:16:20.982Z 5 100-119 144 140 144 249 244 249 F F F <---
157612024-02-03T17:16:20.982Z 6 120-139 144 143 144 249 248 249 F F F <---
157622024-02-03T17:16:20.982Z 7 140-159 142 142 142 246 246 246 F F F
157632024-02-03T17:16:20.982Z 8 160-179 144 137 144 249 238 249 F F F <---
157642024-02-03T17:16:20.982Z 9 180-199 144 138 144 251 239 251 F F F <---
157652024-02-03T17:16:20.982Z 10 200-219 139 139 139 242 242 242 F F F
157662024-02-03T17:16:20.982Z 11 220-239 142 142 142 247 247 247 F F F
157672024-02-03T17:16:20.982Z 12 240-259 140 140 140 243 243 243 F F F
157682024-02-03T17:16:20.982Z 13 260-279 144 141 144 249 245 249 F F F <---
157692024-02-03T17:16:20.982Z 14 280-299 142 142 142 246 246 246 F F F
157702024-02-03T17:16:20.982Z 15 300-319 143 143 143 248 248 248 T F T <---
157712024-02-03T17:16:20.982Z 16 320-339 141 141 141 245 245 245 T F T <---
157722024-02-03T17:16:20.982Z 17 340-359 139 139 139 242 242 242 F F F
157732024-02-03T17:16:20.982Z 18 360-379 140 140 140 243 243 243 F F F
157742024-02-03T17:16:20.983Z 19 380-399 143 143 143 248 248 248 T F T <---
157752024-02-03T17:16:20.983Z 20 400-419 144 138 144 249 239 249 T F T <---
157762024-02-03T17:16:20.983Z 21 420-439 144 142 144 250 247 250 T F T <---
157772024-02-03T17:16:20.983Z 22 440-459 141 141 141 245 245 245 T F T <---
157782024-02-03T17:16:20.983Z 23 460-479 144 141 144 250 245 250 F F F <---
157792024-02-03T17:16:20.983Z 24 480-499 142 142 142 247 247 247 F F F
157802024-02-03T17:16:20.983Z 25 500-519 142 142 142 246 246 246 F F F
157812024-02-03T17:16:20.983Z 26 520-539 142 142 142 246 246 246 F F F
157822024-02-03T17:16:20.983Z 27 540-559 138 138 138 241 241 241 F F F
157832024-02-03T17:16:20.983Z 28 560-579 141 141 141 245 245 245 F F F
157842024-02-03T17:16:20.983Z 29 580-599 137 137 137 238 238 238 F F F
157852024-02-03T17:16:20.983ZMax gen: 144, Max flush: 251
157862024-02-03T17:16:20.983ZError: Difference in extent metadata found!
157872024-02-03T17:16:20.986ZA Difference in extent metadata is expected here
157882024-02-03T17:16:20.986ZOn loop 72
157892024-02-03T17:16:20.986Z
157902024-02-03T17:16:20.986ZVerifying data now
157912024-02-03T17:16:26.278ZNo change after verify
157922024-02-03T17:16:26.278ZLoop: 72 Downstairs dump after verify (and repair):
157932024-02-03T17:16:26.301ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
157942024-02-03T17:16:26.301Z 0 000-019 142 142 142 247 247 247 F F F
157952024-02-03T17:16:26.301Z 1 020-039 144 144 144 249 249 249 F F F
157962024-02-03T17:16:26.301Z 2 040-059 144 144 144 250 250 250 F F F
157972024-02-03T17:16:26.304Z 3 060-079 135 135 135 235 235 235 F F F
157982024-02-03T17:16:26.305Z 4 080-099 145 145 145 252 252 252 F F F
157992024-02-03T17:16:26.305Z 5 100-119 144 144 144 249 249 249 F F F
158002024-02-03T17:16:26.305Z 6 120-139 144 144 144 249 249 249 F F F
158012024-02-03T17:16:26.305Z 7 140-159 142 142 142 246 246 246 F F F
158022024-02-03T17:16:26.305Z 8 160-179 144 144 144 249 249 249 F F F
158032024-02-03T17:16:26.305Z 9 180-199 144 144 144 251 251 251 F F F
158042024-02-03T17:16:26.305Z 10 200-219 139 139 139 242 242 242 F F F
158052024-02-03T17:16:26.305Z 11 220-239 142 142 142 247 247 247 F F F
158062024-02-03T17:16:26.305Z 12 240-259 140 140 140 243 243 243 F F F
158072024-02-03T17:16:26.305Z 13 260-279 144 144 144 249 249 249 F F F
158082024-02-03T17:16:26.305Z 14 280-299 142 142 142 246 246 246 F F F
158092024-02-03T17:16:26.305Z 15 300-319 145 145 145 252 252 252 F F F
158102024-02-03T17:16:26.305Z 16 320-339 145 145 145 252 252 252 F F F
158112024-02-03T17:16:26.305Z 17 340-359 139 139 139 242 242 242 F F F
158122024-02-03T17:16:26.305Z 18 360-379 140 140 140 243 243 243 F F F
158132024-02-03T17:16:26.305Z 19 380-399 145 145 145 252 252 252 F F F
158142024-02-03T17:16:26.305Z 20 400-419 145 145 145 252 252 252 F F F
158152024-02-03T17:16:26.305Z 21 420-439 145 145 145 252 252 252 F F F
158162024-02-03T17:16:26.305Z 22 440-459 145 145 145 252 252 252 F F F
158172024-02-03T17:16:26.305Z 23 460-479 144 144 144 250 250 250 F F F
158182024-02-03T17:16:26.305Z 24 480-499 142 142 142 247 247 247 F F F
158192024-02-03T17:16:26.305Z 25 500-519 142 142 142 246 246 246 F F F
158202024-02-03T17:16:26.306Z 26 520-539 142 142 142 246 246 246 F F F
158212024-02-03T17:16:26.306Z 27 540-559 138 138 138 241 241 241 F F F
158222024-02-03T17:16:26.306Z 28 560-579 141 141 141 245 245 245 F F F
158232024-02-03T17:16:26.306Z 29 580-599 137 137 137 238 238 238 F F F
158242024-02-03T17:16:26.306ZMax gen: 145, Max flush: 252
158252024-02-03T17:16:26.311Z
158262024-02-03T17:16:26.311ZBegin loop 73
158272024-02-03T17:16:26.311ZDownstairs to restart: 2
158282024-02-03T17:16:26.341Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.91195227Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719}
158292024-02-03T17:16:26.344Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.912507154Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719}
158302024-02-03T17:16:26.344Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.912626547Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719}
158312024-02-03T17:16:26.344ZCrucible runtime is spawned
158322024-02-03T17:16:26.344Z{"msg":"Crucible b8b5b1ee-46d2-47bd-bde6-2bddeda2136f has session id: 1b2f9554-365f-4995-a924-ced2a7c184d9","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.912852334Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158332024-02-03T17:16:26.345Z{"msg":"Upstairs opts: Upstairs UUID: b8b5b1ee-46d2-47bd-bde6-2bddeda2136f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.912924547Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158342024-02-03T17:16:26.345Z{"msg":"Crucible stats registered with UUID: b8b5b1ee-46d2-47bd-bde6-2bddeda2136f","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.913024487Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158352024-02-03T17:16:26.345Z{"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.913103822Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719}
158362024-02-03T17:16:26.345Z{"msg":"b8b5b1ee-46d2-47bd-bde6-2bddeda2136f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.913138121Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158372024-02-03T17:16:26.345Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.91321064Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158382024-02-03T17:16:26.345Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.913253679Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158392024-02-03T17:16:26.345Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.913291757Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158402024-02-03T17:16:26.345Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.91333231Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158412024-02-03T17:16:26.345Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.913372281Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158422024-02-03T17:16:26.345Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.913453107Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158432024-02-03T17:16:26.345Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.913563889Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"":"io task","client":"0","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158442024-02-03T17:16:26.345Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.913648557Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"":"io task","client":"1","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158452024-02-03T17:16:26.345Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.913758427Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"":"io task","client":"2","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158462024-02-03T17:16:26.345Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.913803174Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"":"io task","client":"1","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158472024-02-03T17:16:26.345Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.913906641Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"":"io task","client":"0","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158482024-02-03T17:16:26.345Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.913949299Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"":"io task","client":"2","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158492024-02-03T17:16:26.345Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.915120504Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158502024-02-03T17:16:26.345Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.915183304Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158512024-02-03T17:16:26.345Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.915231101Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158522024-02-03T17:16:26.345Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.915338887Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158532024-02-03T17:16:26.351Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.915675868Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158542024-02-03T17:16:26.351Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.915771963Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158552024-02-03T17:16:26.351Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.91646865Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158562024-02-03T17:16:26.351Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.91682869Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158572024-02-03T17:16:26.351Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.916930371Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158582024-02-03T17:16:26.351Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.917242391Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158592024-02-03T17:16:26.351Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.917319394Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158602024-02-03T17:16:26.351Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.917347378Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158612024-02-03T17:16:26.351Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.91769723Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158622024-02-03T17:16:26.351Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.91779456Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158632024-02-03T17:16:26.351Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.917844036Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158642024-02-03T17:16:26.351Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.917953633Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158652024-02-03T17:16:26.351Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.917996909Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158662024-02-03T17:16:26.351Z{"msg":"[0]R flush_numbers[0..12]: [247, 249, 250, 235, 252, 249, 249, 246, 249, 251, 242, 247]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.918077909Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158672024-02-03T17:16:26.351Z{"msg":"[0]R generation[0..12]: [142, 144, 144, 135, 145, 144, 144, 142, 144, 144, 139, 142]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.918160673Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158682024-02-03T17:16:26.351Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
158692024-02-03T17:16:26.351Z,"time":"2024-02-03T17:16:25.918241572Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158702024-02-03T17:16:26.352Z{"msg":"[1]R flush_numbers[0..12]: [247, 249, 250, 235, 252, 249, 249, 246, 249, 251, 242, 247]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.918331406Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158712024-02-03T17:16:26.352Z{"msg":"[1]R generation[0..12]: [142, 144, 144, 135, 145, 144, 144, 142, 144, 144, 139, 142]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.918414742Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158722024-02-03T17:16:26.352Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.918499037Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158732024-02-03T17:16:26.352Z{"msg":"[2]R flush_numbers[0..12]: [247, 249, 250, 235, 252, 249, 249, 246, 249, 251, 242, 247]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.918567051Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158742024-02-03T17:16:26.352Z{"msg":"[2]R generation[0..12]: [142, 144, 144, 135, 145, 144, 144, 142, 144, 144, 139, 142]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.918593853Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158752024-02-03T17:16:26.352Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.918617316Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158762024-02-03T17:16:26.352Z{"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.91866524Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158772024-02-03T17:16:26.352Z{"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.918689775Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158782024-02-03T17:16:26.352Z{"msg":"Next flush: 253","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.918711308Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158792024-02-03T17:16:26.352Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.918766348Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158802024-02-03T17:16:26.352Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.918793619Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158812024-02-03T17:16:26.352Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.918813682Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158822024-02-03T17:16:26.352Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.918868405Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158832024-02-03T17:16:26.352Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.918894033Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158842024-02-03T17:16:26.352Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.918916312Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158852024-02-03T17:16:26.352Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.918938369Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158862024-02-03T17:16:26.352Z{"msg":"b8b5b1ee-46d2-47bd-bde6-2bddeda2136f is now active with session: 1b2f9554-365f-4995-a924-ced2a7c184d9","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.918959847Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158872024-02-03T17:16:26.352Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.918979375Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719,"session_id":"1b2f9554-365f-4995-a924-ced2a7c184d9"}
158882024-02-03T17:16:26.352Z{"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:25.919027972Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719}
158892024-02-03T17:16:26.352ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
158902024-02-03T17:16:26.352ZLoading write count information from file "/tmp/test_repair_verify.data"
158912024-02-03T17:16:26.352ZRun Repair workload
158922024-02-03T17:16:26.352Z01/30 Write block 349 len 8192 data: 12 9
158932024-02-03T17:16:26.357Z02/30 Write block 50 len 28672 data: 12 12 13 10 10 11 12
158942024-02-03T17:16:26.357Z03/30 Write block 232 len 28672 data: 10 10 12 11 9 8 7
158952024-02-03T17:16:26.357Z04/30 Write block 416 len 36864 data: 12 13 12 11 11 13 11 8 8
158962024-02-03T17:16:26.357Z05/30 Write block 293 len 40960 data: 11 12 11 12 11 11 10 8 9 9
158972024-02-03T17:16:26.360Z06/30 Read block 129 len 40960
158982024-02-03T17:16:26.360Z07/30 Read block 20 len 24576
158992024-02-03T17:16:26.363Z08/30 Write block 406 len 28672 data: 11 12 10 10 11 11 12
159002024-02-03T17:16:26.363Z09/30 Write block 216 len 20480 data: 8 7 7 7 6
159012024-02-03T17:16:26.367Z10/30 Write block 46 len 32768 data: 11 11 11 13 13 13 14 11
159022024-02-03T17:16:26.367Z11/30 Read block 507 len 4096
159032024-02-03T17:16:26.367Z12/30 Write block 99 len 4096 data: 11
159042024-02-03T17:16:26.373Z13/30 Read block 382 len 36864
159052024-02-03T17:16:26.373Z14/30 Write block 507 len 32768 data: 14 14 14 13 15 15 12 12
159062024-02-03T17:16:26.376Z15/30 Read block 27 len 32768
159072024-02-03T17:16:26.376Z16/30 Read block 468 len 8192
159082024-02-03T17:16:26.376Z17/30 Write block 268 len 8192 data: 18 14
159092024-02-03T17:16:26.376Z18/30 Read block 394 len 32768
159102024-02-03T17:16:26.379Z19/30 Read block 122 len 36864
159112024-02-03T17:16:26.379Z20/30 Write block 406 len 40960 data: 12 13 11 11 12 12 13 12 13 13
159122024-02-03T17:16:26.382Z21/30 Write block 453 len 4096 data: 8
159132024-02-03T17:16:26.382Z22/30 Write block 153 len 16384 data: 11 10 10 12
159142024-02-03T17:16:26.382Z23/30 Read block 225 len 36864
159152024-02-03T17:16:26.392Z24/30 Read block 250 len 24576
159162024-02-03T17:16:26.396Z25/30 Write block 574 len 36864 data: 9 8 8 7 7 8 7 8 7
159172024-02-03T17:16:26.396Z26/30 Read block 548 len 40960
159182024-02-03T17:16:26.396Z27/30 Read block 256 len 4096
159192024-02-03T17:16:26.396Z28/30 Write block 182 len 8192 data: 13 14
159202024-02-03T17:16:26.396Z29/30 Read block 323 len 20480
159212024-02-03T17:16:26.396Z30/30 Write block 193 len 32768 data: 14 15 16 15 13 12 7 7
159222024-02-03T17:16:26.396Z----------------------------------------------------------------
159232024-02-03T17:16:26.396Z Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:30
159242024-02-03T17:16:26.396ZStates: Active Active Active
159252024-02-03T17:16:26.396ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
159262024-02-03T17:16:26.396Z 1 Acked 1000 Write 2 Done Done Sent false
159272024-02-03T17:16:26.396Z 2 Acked 1001 Write 7 Done Done Sent false
159282024-02-03T17:16:26.396Z 3 Acked 1002 Write 7 Done Done Sent false
159292024-02-03T17:16:26.396Z 4 Acked 1003 Write 9 Done Done Sent false
159302024-02-03T17:16:26.397Z 5 Acked 1004 Write 10 Done Done Sent false
159312024-02-03T17:16:26.397Z 6 Acked 1005 Read 10 Done Done Sent false
159322024-02-03T17:16:26.397Z 7 Acked 1006 Read 6 Done Done Sent false
159332024-02-03T17:16:26.397Z 8 Acked 1007 Write 7 Done Done Sent false
159342024-02-03T17:16:26.397Z 9 Acked 1008 Write 5 Done Done Sent false
159352024-02-03T17:16:26.397Z 10 Acked 1009 Write 8 Done Done Sent false
159362024-02-03T17:16:26.397Z 11 Acked 1010 Read 1 Done Done Sent false
159372024-02-03T17:16:26.397Z 12 Acked 1011 Write 1 Done Done Sent false
159382024-02-03T17:16:26.397Z 13 Acked 1012 Read 9 Done Done Sent false
159392024-02-03T17:16:26.397Z 14 Acked 1013 Write 8 Done Done Sent false
159402024-02-03T17:16:26.397Z 15 Acked 1014 Read 8 Done Done Sent false
159412024-02-03T17:16:26.397Z 16 Acked 1015 Read 2 Done Done Sent false
159422024-02-03T17:16:26.397Z 17 Acked 1016 Write 2 Done Done Sent false
159432024-02-03T17:16:26.397Z 18 Acked 1017 Read 8 Done Done Sent false
159442024-02-03T17:16:26.397Z 19 Acked 1018 Read 9 Done Done Sent false
159452024-02-03T17:16:26.397Z 20 Acked 1019 Write 10 Done Done Sent false
159462024-02-03T17:16:26.397Z 21 Acked 1020 Write 1 Done Done Sent false
159472024-02-03T17:16:26.397Z 22 Acked 1021 Write 4 Done Done Sent false
159482024-02-03T17:16:26.397Z 23 Acked 1022 Read 9 Done Done Sent false
159492024-02-03T17:16:26.397Z 24 Acked 1023 Read 6 Done Done Sent false
159502024-02-03T17:16:26.397Z 25 Acked 1024 Write 9 Done Done Sent false
159512024-02-03T17:16:26.397Z 26 Acked 1025 Read 10 Done Done Sent false
159522024-02-03T17:16:26.397Z 27 Acked 1026 Read 1 Done Done Sent false
159532024-02-03T17:16:26.397Z 28 Acked 1027 Write 2 Done Done Sent false
159542024-02-03T17:16:26.398Z 29 Acked 1028 Read 5 Done Done Sent false
159552024-02-03T17:16:26.398Z 30 Acked 1029 Write 8 Sent Sent Sent false
159562024-02-03T17:16:26.398Z STATES DS:0 DS:1 DS:2 TOTAL
159572024-02-03T17:16:26.398Z New 0 0 0 0
159582024-02-03T17:16:26.398Z Sent 1 1 30 32
159592024-02-03T17:16:26.398Z Done 29 29 0 58
159602024-02-03T17:16:26.398Z Skipped 0 0 0 0
159612024-02-03T17:16:26.398Z Error 0 0 0 0
159622024-02-03T17:16:26.398ZLast Flush: 0 0 0
159632024-02-03T17:16:26.398ZDownstairs last five completed:
159642024-02-03T17:16:26.398ZUpstairs last five completed: 30 29 28 27 26
159652024-02-03T17:16:26.398Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:16:25.966540171Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1719}
159662024-02-03T17:16:26.398ZWrote out file "/tmp/test_repair_verify.data"
159672024-02-03T17:16:26.402Z
159682024-02-03T17:16:26.408ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
159692024-02-03T17:16:26.439ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
159702024-02-03T17:16:26.439Z 0 000-019 142 142 142 247 247 247 F F F
159712024-02-03T17:16:26.439Z 1 020-039 144 144 144 249 249 249 F F F
159722024-02-03T17:16:26.439Z 2 040-059 144 144 144 250 250 250 T T F <---
159732024-02-03T17:16:26.439Z 3 060-079 135 135 135 235 235 235 F F F
159742024-02-03T17:16:26.442Z 4 080-099 145 145 145 252 252 252 T T F <---
159752024-02-03T17:16:26.442Z 5 100-119 144 144 144 249 249 249 F F F
159762024-02-03T17:16:26.442Z 6 120-139 144 144 144 249 249 249 F F F
159772024-02-03T17:16:26.442Z 7 140-159 142 142 142 246 246 246 T T F <---
159782024-02-03T17:16:26.442Z 8 160-179 144 144 144 249 249 249 F F F
159792024-02-03T17:16:26.442Z 9 180-199 144 144 144 251 251 251 T T F <---
159802024-02-03T17:16:26.442Z 10 200-219 139 139 139 242 242 242 T T F <---
159812024-02-03T17:16:26.442Z 11 220-239 142 142 142 247 247 247 T T F <---
159822024-02-03T17:16:26.443Z 12 240-259 140 140 140 243 243 243 F F F
159832024-02-03T17:16:26.443Z 13 260-279 144 144 144 249 249 249 T T F <---
159842024-02-03T17:16:26.443Z 14 280-299 142 142 142 246 246 246 T T F <---
159852024-02-03T17:16:26.443Z 15 300-319 145 145 145 252 252 252 T T F <---
159862024-02-03T17:16:26.443Z 16 320-339 145 145 145 252 252 252 F F F
159872024-02-03T17:16:26.443Z 17 340-359 139 139 139 242 242 242 T T F <---
159882024-02-03T17:16:26.443Z 18 360-379 140 140 140 243 243 243 F F F
159892024-02-03T17:16:26.443Z 19 380-399 145 145 145 252 252 252 F F F
159902024-02-03T17:16:26.443Z 20 400-419 145 145 145 252 252 252 T T F <---
159912024-02-03T17:16:26.443Z 21 420-439 145 145 145 252 252 252 T T F <---
159922024-02-03T17:16:26.443Z 22 440-459 145 145 145 252 252 252 T T F <---
159932024-02-03T17:16:26.443Z 23 460-479 144 144 144 250 250 250 F F F
159942024-02-03T17:16:26.443Z 24 480-499 142 142 142 247 247 247 F F F
159952024-02-03T17:16:26.443Z 25 500-519 142 142 142 246 246 246 T T F <---
159962024-02-03T17:16:26.443Z 26 520-539 142 142 142 246 246 246 F F F
159972024-02-03T17:16:26.443Z 27 540-559 138 138 138 241 241 241 F F F
159982024-02-03T17:16:26.443Z 28 560-579 141 141 141 245 245 245 T T F <---
159992024-02-03T17:16:26.443Z 29 580-599 137 137 137 238 238 238 T T F <---
160002024-02-03T17:16:26.443ZMax gen: 145, Max flush: 252
160012024-02-03T17:16:26.443ZError: Difference in extent metadata found!
160022024-02-03T17:16:26.446ZA Difference in extent metadata is expected here
160032024-02-03T17:16:26.447ZOn loop 73
160042024-02-03T17:16:26.447Z
160052024-02-03T17:16:26.450ZVerifying data now
160062024-02-03T17:16:31.839ZNo change after verify
160072024-02-03T17:16:31.839ZLoop: 73 Downstairs dump after verify (and repair):
160082024-02-03T17:16:31.865ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
160092024-02-03T17:16:31.865Z 0 000-019 142 142 142 247 247 247 F F F
160102024-02-03T17:16:31.865Z 1 020-039 144 144 144 249 249 249 F F F
160112024-02-03T17:16:31.865Z 2 040-059 146 146 146 253 253 253 F F F
160122024-02-03T17:16:31.865Z 3 060-079 135 135 135 235 235 235 F F F
160132024-02-03T17:16:31.865Z 4 080-099 146 146 146 253 253 253 F F F
160142024-02-03T17:16:31.869Z 5 100-119 144 144 144 249 249 249 F F F
160152024-02-03T17:16:31.869Z 6 120-139 144 144 144 249 249 249 F F F
160162024-02-03T17:16:31.869Z 7 140-159 146 146 146 253 253 253 F F F
160172024-02-03T17:16:31.869Z 8 160-179 144 144 144 249 249 249 F F F
160182024-02-03T17:16:31.869Z 9 180-199 146 146 146 253 253 253 F F F
160192024-02-03T17:16:31.869Z 10 200-219 146 146 146 253 253 253 F F F
160202024-02-03T17:16:31.869Z 11 220-239 146 146 146 253 253 253 F F F
160212024-02-03T17:16:31.869Z 12 240-259 140 140 140 243 243 243 F F F
160222024-02-03T17:16:31.869Z 13 260-279 146 146 146 253 253 253 F F F
160232024-02-03T17:16:31.869Z 14 280-299 146 146 146 253 253 253 F F F
160242024-02-03T17:16:31.869Z 15 300-319 146 146 146 253 253 253 F F F
160252024-02-03T17:16:31.869Z 16 320-339 145 145 145 252 252 252 F F F
160262024-02-03T17:16:31.869Z 17 340-359 146 146 146 253 253 253 F F F
160272024-02-03T17:16:31.869Z 18 360-379 140 140 140 243 243 243 F F F
160282024-02-03T17:16:31.869Z 19 380-399 145 145 145 252 252 252 F F F
160292024-02-03T17:16:31.869Z 20 400-419 146 146 146 253 253 253 F F F
160302024-02-03T17:16:31.870Z 21 420-439 146 146 146 253 253 253 F F F
160312024-02-03T17:16:31.870Z 22 440-459 146 146 146 253 253 253 F F F
160322024-02-03T17:16:31.870Z 23 460-479 144 144 144 250 250 250 F F F
160332024-02-03T17:16:31.870Z 24 480-499 142 142 142 247 247 247 F F F
160342024-02-03T17:16:31.870Z 25 500-519 146 146 146 253 253 253 F F F
160352024-02-03T17:16:31.870Z 26 520-539 142 142 142 246 246 246 F F F
160362024-02-03T17:16:31.870Z 27 540-559 138 138 138 241 241 241 F F F
160372024-02-03T17:16:31.870Z 28 560-579 146 146 146 253 253 253 F F F
160382024-02-03T17:16:31.870Z 29 580-599 146 146 146 253 253 253 F F F
160392024-02-03T17:16:31.870ZMax gen: 146, Max flush: 253
160402024-02-03T17:16:31.870Z
160412024-02-03T17:16:31.870ZBegin loop 74
160422024-02-03T17:16:31.870ZDownstairs to restart: 0
160432024-02-03T17:16:31.898Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.469615175Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727}
160442024-02-03T17:16:31.902Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.47028911Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727}
160452024-02-03T17:16:31.902ZCrucible runtime is spawned
160462024-02-03T17:16:31.902Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.470384219Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727}
160472024-02-03T17:16:31.902Z{"msg":"Crucible 55d0da60-4013-443a-8179-4d702756b238 has session id: f6347761-36bb-4725-857a-6073b10de79d","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.470504569Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160482024-02-03T17:16:31.903Z{"msg":"Upstairs opts: Upstairs UUID: 55d0da60-4013-443a-8179-4d702756b238, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.470566185Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160492024-02-03T17:16:31.903Z{"msg":"Crucible stats registered with UUID: 55d0da60-4013-443a-8179-4d702756b238","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.47064677Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160502024-02-03T17:16:31.903Z{"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.47068913Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727}
160512024-02-03T17:16:31.903Z{"msg":"55d0da60-4013-443a-8179-4d702756b238 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.470766691Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160522024-02-03T17:16:31.903Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.470844911Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160532024-02-03T17:16:31.903Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.470890654Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160542024-02-03T17:16:31.903Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.470934156Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160552024-02-03T17:16:31.903Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.471126506Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160562024-02-03T17:16:31.903Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.471221748Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160572024-02-03T17:16:31.903Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.471268437Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160582024-02-03T17:16:31.903Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.471310616Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"":"io task","client":"1","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160592024-02-03T17:16:31.903Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.47148609Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"":"io task","client":"0","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160602024-02-03T17:16:31.903Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.471773338Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"":"io task","client":"2","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160612024-02-03T17:16:31.903Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.471866072Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"":"io task","client":"1","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160622024-02-03T17:16:31.903Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.471956773Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"":"io task","client":"2","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160632024-02-03T17:16:31.903Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.472071695Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"":"io task","client":"0","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160642024-02-03T17:16:31.903Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.472912103Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160652024-02-03T17:16:31.903Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.473024966Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160662024-02-03T17:16:31.903Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.473187298Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160672024-02-03T17:16:31.903Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.473225135Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160682024-02-03T17:16:31.903Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.473304968Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160692024-02-03T17:16:31.903Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.473394663Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160702024-02-03T17:16:31.909Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.474109443Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160712024-02-03T17:16:31.909Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.474297702Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160722024-02-03T17:16:31.909Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.474402991Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160732024-02-03T17:16:31.909Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.475056084Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160742024-02-03T17:16:31.909Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.475180156Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160752024-02-03T17:16:31.909Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.475231388Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160762024-02-03T17:16:31.909Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.475333371Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160772024-02-03T17:16:31.909Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.475397102Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160782024-02-03T17:16:31.909Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.475441358Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160792024-02-03T17:16:31.909Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.475505468Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160802024-02-03T17:16:31.909Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.475549009Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160812024-02-03T17:16:31.909Z{"msg":"[0]R flush_numbers[0..12]: [247, 249, 253, 235, 253, 249, 249, 253, 249, 253, 253, 253]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.475614272Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160822024-02-03T17:16:31.909Z{"msg":"[0]R generation[0..12]: [142, 144, 146, 135, 146, 144, 144, 146, 144, 146, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.47565889Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160832024-02-03T17:16:31.909Z{"msg":"Wait for a query_work_queue command to finish before sending IO
160842024-02-03T17:16:31.909Z[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.475760962Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160852024-02-03T17:16:31.909Z{"msg":"[1]R flush_numbers[0..12]: [247, 249, 253, 235, 253, 249, 249, 253, 249, 253, 253, 253]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.475829949Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160862024-02-03T17:16:31.909Z{"msg":"[1]R generation[0..12]: [142, 144, 146, 135, 146, 144, 144, 146, 144, 146, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.475874528Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160872024-02-03T17:16:31.909Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.475939467Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160882024-02-03T17:16:31.909Z{"msg":"[2]R flush_numbers[0..12]: [247, 249, 253, 235, 253, 249, 249, 253, 249, 253, 253, 253]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.475985681Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160892024-02-03T17:16:31.909Z{"msg":"[2]R generation[0..12]: [142, 144, 146, 135, 146, 144, 144, 146, 144, 146, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.476054201Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160902024-02-03T17:16:31.909Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.476082496Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160912024-02-03T17:16:31.909Z{"msg":"Max found gen is 147","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.476137428Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160922024-02-03T17:16:31.909Z{"msg":"Generation requested: 148 >= found:147","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.476164323Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160932024-02-03T17:16:31.910Z{"msg":"Next flush: 254","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.476220409Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160942024-02-03T17:16:31.910Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.476245647Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160952024-02-03T17:16:31.910Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.47626779Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160962024-02-03T17:16:31.910Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.476293352Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160972024-02-03T17:16:31.910Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.47638183Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160982024-02-03T17:16:31.910Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.476422328Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
160992024-02-03T17:16:31.910Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.476459561Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
161002024-02-03T17:16:31.910Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.476556336Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
161012024-02-03T17:16:31.910Z{"msg":"55d0da60-4013-443a-8179-4d702756b238 is now active with session: f6347761-36bb-4725-857a-6073b10de79d","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.47658747Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
161022024-02-03T17:16:31.910Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
161032024-02-03T17:16:31.910ZSet Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.476731623Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727,"session_id":"f6347761-36bb-4725-857a-6073b10de79d"}
161042024-02-03T17:16:31.910Z{"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:31.476791681Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727}
161052024-02-03T17:16:31.910ZLoading write count information from file "/tmp/test_repair_verify.data"
161062024-02-03T17:16:31.910ZRun Repair workload
161072024-02-03T17:16:31.910Z01/30 Write block 2 len 16384 data: 8 9 10 10
161082024-02-03T17:16:31.914Z02/30 Read block 502 len 36864
161092024-02-03T17:16:31.914Z03/30 Read block 33 len 4096
161102024-02-03T17:16:31.914Z04/30 Read block 543 len 4096
161112024-02-03T17:16:31.914Z05/30 Write block 497 len 16384 data: 9 8 7 5
161122024-02-03T17:16:31.917Z06/30 Write block 460 len 40960 data: 16 14 11 13 11 10 13 12 11 10
161132024-02-03T17:16:31.917Z07/30 Read block 580 len 24576
161142024-02-03T17:16:31.920Z08/30 Read block 40 len 40960
161152024-02-03T17:16:31.920Z09/30 Read block 267 len 8192
161162024-02-03T17:16:31.922Z10/30 Write block 36 len 16384 data: 13 11 11 11
161172024-02-03T17:16:31.922Z11/30 Write block 19 len 16384 data: 20 16 19 15
161182024-02-03T17:16:31.925Z12/30 Read block 152 len 8192
161192024-02-03T17:16:31.925Z13/30 Flush
161202024-02-03T17:16:31.933Z14/30 Flush
161212024-02-03T17:16:31.933Z15/30 Write block 537 len 16384 data: 7 8 7 7
161222024-02-03T17:16:31.933Z16/30 Write block 251 len 28672 data: 12 11 8 5 8 12 13
161232024-02-03T17:16:31.933Z17/30 Write block 167 len 24576 data: 19 16 16 16 13 10
161242024-02-03T17:16:31.934Z18/30 Flush
161252024-02-03T17:16:31.937Z19/30 Write block 210 len 36864 data: 13 11 11 10 5 6 9 8 8
161262024-02-03T17:16:31.941Z20/30 Write block 567 len 8192 data: 12 15
161272024-02-03T17:16:31.941Z21/30 Write block 568 len 4096 data: 16
161282024-02-03T17:16:31.941Z22/30 Read block 562 len 20480
161292024-02-03T17:16:31.946Z23/30 Read block 387 len 20480
161302024-02-03T17:16:31.946Z24/30 Write block 377 len 16384 data: 16 13 15 13
161312024-02-03T17:16:31.946Z25/30 Write block 259 len 28672 data: 15 18 18 17 17 18 17
161322024-02-03T17:16:31.957Z26/30 Read block 232 len 16384
161332024-02-03T17:16:31.957Z27/30 Write block 332 len 24576 data: 10 11 13 13 14 13
161342024-02-03T17:16:31.957Z28/30 Read block 11 len 16384
161352024-02-03T17:16:31.957Z29/30 Read block 473 len 28672
161362024-02-03T17:16:31.957Z30/30 Write block 281 len 36864 data: 14 16 18 14 15 15 14 12 14
161372024-02-03T17:16:31.969Z----------------------------------------------------------------
161382024-02-03T17:16:31.970Z Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30
161392024-02-03T17:16:31.970ZStates: Active Active Active
161402024-02-03T17:16:31.970ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
161412024-02-03T17:16:31.970Z 1 Acked 1000 Write 4 Done Done Done false
161422024-02-03T17:16:31.970Z 2 Acked 1001 Read 9 Done Done Done false
161432024-02-03T17:16:31.970Z 3 Acked 1002 Read 1 Done Done Done false
161442024-02-03T17:16:31.970Z 4 Acked 1003 Read 1 Done Done Done false
161452024-02-03T17:16:31.970Z 5 Acked 1004 Write 4 Done Done Done false
161462024-02-03T17:16:31.970Z 6 Acked 1005 Write 10 Done Done Done false
161472024-02-03T17:16:31.970Z 7 Acked 1006 Read 6 Done Done Done false
161482024-02-03T17:16:31.970Z 8 Acked 1007 Read 10 Done Done Done false
161492024-02-03T17:16:31.971Z 9 Acked 1008 Read 2 Sent Done Done false
161502024-02-03T17:16:31.971Z 10 Acked 1009 Write 4 Sent Done Done false
161512024-02-03T17:16:31.971Z 11 Acked 1010 Write 4 Sent Done Done false
161522024-02-03T17:16:31.971Z 12 Acked 1011 Read 2 Sent Done Done false
161532024-02-03T17:16:31.971Z 13 Acked 1012 Flush 0 Sent Done Done false
161542024-02-03T17:16:31.971Z 14 Acked 1013 Flush 0 Sent Done Done false
161552024-02-03T17:16:31.971Z 15 Acked 1014 Write 4 Sent Done Done false
161562024-02-03T17:16:31.971Z 16 Acked 1015 Write 7 Sent Done Done false
161572024-02-03T17:16:31.971Z 17 Acked 1016 Write 6 Sent Done Done false
161582024-02-03T17:16:31.971Z 18 Acked 1017 Flush 0 Sent Done Done false
161592024-02-03T17:16:31.971Z 19 Acked 1018 Write 9 Sent Done Done false
161602024-02-03T17:16:31.971Z 20 Acked 1019 Write 2 Sent Done Done false
161612024-02-03T17:16:31.971Z 21 Acked 1020 Write 1 Sent Done Done false
161622024-02-03T17:16:31.971Z 22 Acked 1021 Read 5 Sent Done Done false
161632024-02-03T17:16:31.972Z 23 Acked 1022 Read 5 Sent Done Done false
161642024-02-03T17:16:31.972Z 24 Acked 1023 Write 4 Sent Done Done false
161652024-02-03T17:16:31.972Z 25 Acked 1024 Write 7 Sent Done Done false
161662024-02-03T17:16:31.972Z 26 Acked 1025 Read 4 Sent Done Done false
161672024-02-03T17:16:31.972Z 27 Acked 1026 Write 6 Sent Done Done false
161682024-02-03T17:16:31.972Z 28 Acked 1027 Read 4 Sent Done Done false
161692024-02-03T17:16:31.972Z 29 Acked 1028 Read 7 Sent Done Done false
161702024-02-03T17:16:31.972Z 30 Acked 1029 Write 9 Sent Sent Sent false
161712024-02-03T17:16:31.972Z STATES DS:0 DS:1 DS:2 TOTAL
161722024-02-03T17:16:31.972Z New 0 0 0 0
161732024-02-03T17:16:31.972Z Sent 22 1 1 24
161742024-02-03T17:16:31.972Z Done 8 29 29 66
161752024-02-03T17:16:31.973Z Skipped 0 0 0 0
161762024-02-03T17:16:31.973Z Error 0 0 0 0
161772024-02-03T17:16:31.973ZLast Flush: 0 1017 1017
161782024-02-03T17:16:31.973ZDownstairs last five completed:
161792024-02-03T17:16:31.973ZUpstairs last five completed: 30 29 28 27 26
161802024-02-03T17:16:31.973Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:16:31.526205226Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1727}
161812024-02-03T17:16:31.973ZWrote out file "/tmp/test_repair_verify.data"
161822024-02-03T17:16:31.973Z
161832024-02-03T17:16:31.988ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
161842024-02-03T17:16:31.997ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
161852024-02-03T17:16:31.997Z 0 000-019 142 148 148 247 254 254 T F F <---
161862024-02-03T17:16:32.002Z 1 020-039 144 148 148 249 254 254 F F F <---
161872024-02-03T17:16:32.002Z 2 040-059 146 146 146 253 253 253 F F F
161882024-02-03T17:16:32.002Z 3 060-079 135 135 135 235 235 235 F F F
161892024-02-03T17:16:32.002Z 4 080-099 146 146 146 253 253 253 F F F
161902024-02-03T17:16:32.002Z 5 100-119 144 144 144 249 249 249 F F F
161912024-02-03T17:16:32.003Z 6 120-139 144 144 144 249 249 249 F F F
161922024-02-03T17:16:32.003Z 7 140-159 146 146 146 253 253 253 F F F
161932024-02-03T17:16:32.003Z 8 160-179 144 148 148 249 256 256 F F F <---
161942024-02-03T17:16:32.003Z 9 180-199 146 146 146 253 253 253 F F F
161952024-02-03T17:16:32.003Z 10 200-219 146 146 146 253 253 253 F T T <---
161962024-02-03T17:16:32.003Z 11 220-239 146 146 146 253 253 253 F F F
161972024-02-03T17:16:32.003Z 12 240-259 140 148 148 243 256 256 F T T <---
161982024-02-03T17:16:32.003Z 13 260-279 146 146 146 253 253 253 F T T <---
161992024-02-03T17:16:32.003Z 14 280-299 146 146 146 253 253 253 F T T <---
162002024-02-03T17:16:32.003Z 15 300-319 146 146 146 253 253 253 F F F
162012024-02-03T17:16:32.003Z 16 320-339 145 145 145 252 252 252 F T T <---
162022024-02-03T17:16:32.003Z 17 340-359 146 146 146 253 253 253 F F F
162032024-02-03T17:16:32.003Z 18 360-379 140 140 140 243 243 243 F T T <---
162042024-02-03T17:16:32.004Z 19 380-399 145 145 145 252 252 252 F T T <---
162052024-02-03T17:16:32.004Z 20 400-419 146 146 146 253 253 253 F F F
162062024-02-03T17:16:32.004Z 21 420-439 146 146 146 253 253 253 F F F
162072024-02-03T17:16:32.004Z 22 440-459 146 146 146 253 253 253 F F F
162082024-02-03T17:16:32.004Z 23 460-479 144 148 148 250 254 254 T F F <---
162092024-02-03T17:16:32.004Z 24 480-499 142 148 148 247 254 254 T F F <---
162102024-02-03T17:16:32.004Z 25 500-519 146 148 148 253 254 254 T F F <---
162112024-02-03T17:16:32.004Z 26 520-539 142 148 148 246 256 256 F F F <---
162122024-02-03T17:16:32.004Z 27 540-559 138 148 148 241 256 256 F F F <---
162132024-02-03T17:16:32.004Z 28 560-579 146 146 146 253 253 253 F T T <---
162142024-02-03T17:16:32.004Z 29 580-599 146 146 146 253 253 253 F F F
162152024-02-03T17:16:32.004ZMax gen: 148, Max flush: 256
162162024-02-03T17:16:32.004ZError: Difference in extent metadata found!
162172024-02-03T17:16:32.004ZA Difference in extent metadata is expected here
162182024-02-03T17:16:32.005ZOn loop 74
162192024-02-03T17:16:32.005Z
162202024-02-03T17:16:32.008ZVerifying data now
162212024-02-03T17:16:38.400ZNo change after verify
162222024-02-03T17:16:38.401ZLoop: 74 Downstairs dump after verify (and repair):
162232024-02-03T17:16:38.424ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
162242024-02-03T17:16:38.424Z 0 000-019 148 148 148 254 254 254 F F F
162252024-02-03T17:16:38.424Z 1 020-039 148 148 148 254 254 254 F F F
162262024-02-03T17:16:38.424Z 2 040-059 146 146 146 253 253 253 F F F
162272024-02-03T17:16:38.425Z 3 060-079 135 135 135 235 235 235 F F F
162282024-02-03T17:16:38.444Z 4 080-099 146 146 146 253 253 253 F F F
162292024-02-03T17:16:38.444Z 5 100-119 144 144 144 249 249 249 F F F
162302024-02-03T17:16:38.444Z 6 120-139 144 144 144 249 249 249 F F F
162312024-02-03T17:16:38.444Z 7 140-159 146 146 146 253 253 253 F F F
162322024-02-03T17:16:38.444Z 8 160-179 148 148 148 256 256 256 F F F
162332024-02-03T17:16:38.445Z 9 180-199 146 146 146 253 253 253 F F F
162342024-02-03T17:16:38.445Z 10 200-219 149 149 149 257 257 257 F F F
162352024-02-03T17:16:38.445Z 11 220-239 146 146 146 253 253 253 F F F
162362024-02-03T17:16:38.445Z 12 240-259 149 149 149 257 257 257 F F F
162372024-02-03T17:16:38.446Z 13 260-279 149 149 149 257 257 257 F F F
162382024-02-03T17:16:38.446Z 14 280-299 149 149 149 257 257 257 F F F
162392024-02-03T17:16:38.446Z 15 300-319 146 146 146 253 253 253 F F F
162402024-02-03T17:16:38.446Z 16 320-339 149 149 149 257 257 257 F F F
162412024-02-03T17:16:38.446Z 17 340-359 146 146 146 253 253 253 F F F
162422024-02-03T17:16:38.447Z 18 360-379 149 149 149 257 257 257 F F F
162432024-02-03T17:16:38.447Z 19 380-399 149 149 149 257 257 257 F F F
162442024-02-03T17:16:38.447Z 20 400-419 146 146 146 253 253 253 F F F
162452024-02-03T17:16:38.447Z 21 420-439 146 146 146 253 253 253 F F F
162462024-02-03T17:16:38.447Z 22 440-459 146 146 146 253 253 253 F F F
162472024-02-03T17:16:38.448Z 23 460-479 148 148 148 254 254 254 F F F
162482024-02-03T17:16:38.448Z 24 480-499 148 148 148 254 254 254 F F F
162492024-02-03T17:16:38.448Z 25 500-519 148 148 148 254 254 254 F F F
162502024-02-03T17:16:38.448Z 26 520-539 148 148 148 256 256 256 F F F
162512024-02-03T17:16:38.449Z 27 540-559 148 148 148 256 256 256 F F F
162522024-02-03T17:16:38.449Z 28 560-579 149 149 149 257 257 257 F F F
162532024-02-03T17:16:38.449Z 29 580-599 146 146 146 253 253 253 F F F
162542024-02-03T17:16:38.449ZMax gen: 149, Max flush: 257
162552024-02-03T17:16:38.449Z
162562024-02-03T17:16:38.450ZBegin loop 75
162572024-02-03T17:16:38.450ZDownstairs to restart: 2
162582024-02-03T17:16:38.485Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.024266479Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735}
162592024-02-03T17:16:38.485Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.024731338Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735}
162602024-02-03T17:16:38.486Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.02476336Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735}
162612024-02-03T17:16:38.486Z{"msg":"Crucible 4f264e89-2767-48da-ba81-f4ba98d60508 has session id: 8e6473c6-7e1a-4318-be5f-69eac841f0f4","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.024877372Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162622024-02-03T17:16:38.486Z{"msg":"Upstairs opts: Upstairs UUID: 4f264e89-2767-48da-ba81-f4ba98d60508, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
162632024-02-03T17:16:38.487Z,"time":"2024-02-03T17:16:38.02511716Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162642024-02-03T17:16:38.487Z{"msg":"Crucible stats registered with UUID: 4f264e89-2767-48da-ba81-f4ba98d60508","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.025249137Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162652024-02-03T17:16:38.487Z{"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.025451763Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735}
162662024-02-03T17:16:38.487Z{"msg":"4f264e89-2767-48da-ba81-f4ba98d60508 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.025571724Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162672024-02-03T17:16:38.488Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.025606286Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162682024-02-03T17:16:38.488Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.025630941Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162692024-02-03T17:16:38.488Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.025653436Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162702024-02-03T17:16:38.488Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.025676283Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162712024-02-03T17:16:38.488Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.025698382Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162722024-02-03T17:16:38.489Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.025732564Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162732024-02-03T17:16:38.489Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.025791823Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"":"io task","client":"0","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162742024-02-03T17:16:38.489Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.025819929Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"":"io task","client":"2","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162752024-02-03T17:16:38.489Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.025844992Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"":"io task","client":"1","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162762024-02-03T17:16:38.489Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.026274769Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"":"io task","client":"0","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162772024-02-03T17:16:38.490Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.026418583Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"":"io task","client":"2","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162782024-02-03T17:16:38.490Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.026468406Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"":"io task","client":"1","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162792024-02-03T17:16:38.490Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.027498388Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162802024-02-03T17:16:38.491Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.027570511Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162812024-02-03T17:16:38.491Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.02775761Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162822024-02-03T17:16:38.491Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.027879219Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162832024-02-03T17:16:38.491Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.028279836Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162842024-02-03T17:16:38.491Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.028348873Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162852024-02-03T17:16:38.491Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.028377388Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162862024-02-03T17:16:38.491Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.02921952Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162872024-02-03T17:16:38.491Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.029579527Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162882024-02-03T17:16:38.491Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.029645579Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162892024-02-03T17:16:38.491Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.029671587Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162902024-02-03T17:16:38.491Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.029693696Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162912024-02-03T17:16:38.492Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.030097989Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162922024-02-03T17:16:38.492Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.030169843Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162932024-02-03T17:16:38.492Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.030197749Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162942024-02-03T17:16:38.492Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.030642187Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162952024-02-03T17:16:38.492Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.030727922Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162962024-02-03T17:16:38.492Z{"msg":"[0]R flush_numbers[0..12]: [254, 254, 253, 235, 253, 249, 249, 253, 256, 253, 257, 253]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.030772119Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162972024-02-03T17:16:38.492Z{"msg":"[0]R generation[0..12]: [148, 148, 146, 135, 146, 144, 144, 146, 148, 146, 149, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.030828072Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162982024-02-03T17:16:38.492Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.030855689Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
162992024-02-03T17:16:38.492Z{"msg":"[1]R flush_numbers[0..12]: [254, 254, 253, 235, 253, 249, 249, 253, 256, 253, 257, 253]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.030904396Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
163002024-02-03T17:16:38.492Z{"msg":"[1]R generation[0..12]: [148, 148, 146, 135, 146, 144, 144, 146, 148, 146, 149, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.03093013Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
163012024-02-03T17:16:38.492Z{Wait for a query_work_queue command to finish before sending IO
163022024-02-03T17:16:38.492Z"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.030983137Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
163032024-02-03T17:16:38.493Z{"msg":"[2]R flush_numbers[0..12]: [254, 254, 253, 235, 253, 249, 249, 253, 256, 253, 257, 253]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.031009341Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
163042024-02-03T17:16:38.493Z{"msg":"[2]R generation[0..12]: [148, 148, 146, 135, 146, 144, 144, 146, 148, 146, 149, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.031059178Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
163052024-02-03T17:16:38.493Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.031085516Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
163062024-02-03T17:16:38.493Z{"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.031107374Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
163072024-02-03T17:16:38.493Z{"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.031133699Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
163082024-02-03T17:16:38.493Z{"msg":"Next flush: 258","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.031182199Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
163092024-02-03T17:16:38.493Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.031207011Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
163102024-02-03T17:16:38.493Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.031229574Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
163112024-02-03T17:16:38.493Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.031249817Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
163122024-02-03T17:16:38.493Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.031271733Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
163132024-02-03T17:16:38.493Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.031293726Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
163142024-02-03T17:16:38.493Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.031350291Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
163152024-02-03T17:16:38.494Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.031386746Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
163162024-02-03T17:16:38.494Z{"msg":"4f264e89-2767-48da-ba81-f4ba98d60508 is now active with session: 8e6473c6-7e1a-4318-be5f-69eac841f0f4","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.031410844Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
163172024-02-03T17:16:38.494Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.031431064Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735,"session_id":"8e6473c6-7e1a-4318-be5f-69eac841f0f4"}
163182024-02-03T17:16:38.494Z{"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:38.031451815Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735}
163192024-02-03T17:16:38.494ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
163202024-02-03T17:16:38.494ZLoading write count information from file "/tmp/test_repair_verify.data"
163212024-02-03T17:16:38.494ZRun Repair workload
163222024-02-03T17:16:38.494Z01/30 Write block 110 len 8192 data: 7 7
163232024-02-03T17:16:38.506Z02/30 Write block 454 len 4096 data: 8
163242024-02-03T17:16:38.506Z03/30 Flush
163252024-02-03T17:16:38.506Z04/30 Flush
163262024-02-03T17:16:38.506Z05/30 Write block 191 len 40960 data: 17 18 15 16 17 16 14 13 8 8
163272024-02-03T17:16:38.506Z06/30 Write block 247 len 4096 data: 8
163282024-02-03T17:16:38.507Z07/30 Flush
163292024-02-03T17:16:38.507Z08/30 Flush
163302024-02-03T17:16:38.507Z09/30 Flush
163312024-02-03T17:16:38.507Z10/30 Write block 115 len 4096 data: 9
163322024-02-03T17:16:38.507Z11/30 Read block 454 len 8192
163332024-02-03T17:16:38.507Z12/30 Read block 225 len 20480
163342024-02-03T17:16:38.507Z13/30 Write block 86 len 8192 data: 9 10
163352024-02-03T17:16:38.507Z14/30 Flush
163362024-02-03T17:16:38.507Z15/30 Write block 14 len 20480 data: 14 11 10 14 16
163372024-02-03T17:16:38.507Z16/30 Read block 398 len 40960
163382024-02-03T17:16:38.508Z17/30 Write block 263 len 36864 data: 18 19 18 15 15 19 15 14 13
163392024-02-03T17:16:38.508Z18/30 Read block 395 len 24576
163402024-02-03T17:16:38.508Z19/30 Read block 457 len 4096
163412024-02-03T17:16:38.508Z20/30 Read block 336 len 16384
163422024-02-03T17:16:38.508Z21/30 Flush
163432024-02-03T17:16:38.508Z22/30 Write block 74 len 36864 data: 7 10 12 11 10 11 9 12 11
163442024-02-03T17:16:38.508Z23/30 Write block 336 len 36864 data: 15 14 16 13 14 12 12 8 9
163452024-02-03T17:16:38.508Z24/30 Read block 469 len 8192
163462024-02-03T17:16:38.508Z25/30 Write block 464 len 4096 data: 12
163472024-02-03T17:16:38.508Z26/30 Read block 361 len 24576
163482024-02-03T17:16:38.508Z27/30 Write block 204 len 28672 data: 9 12 10 11 12 11 14
163492024-02-03T17:16:38.509Z28/30 Read block 115 len 28672
163502024-02-03T17:16:38.930Z29/30 Write block 376 len 36864 data: 15 17 14 16 14 11 12 12 11
163512024-02-03T17:16:38.930Z30/30 Read block 563 len 8192
163522024-02-03T17:16:38.930Z----------------------------------------------------------------
163532024-02-03T17:16:38.930Z Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:30
163542024-02-03T17:16:38.930ZStates: Active Active Active
163552024-02-03T17:16:38.930ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
163562024-02-03T17:16:38.930Z 1 Acked 1000 Write 2 Done Done Sent false
163572024-02-03T17:16:38.930Z 2 Acked 1001 Write 1 Done Done Sent false
163582024-02-03T17:16:38.930Z 3 Acked 1002 Flush 0 Done Done Sent false
163592024-02-03T17:16:38.930Z 4 Acked 1003 Flush 0 Done Done Sent false
163602024-02-03T17:16:38.930Z 5 Acked 1004 Write 10 Done Done Sent false
163612024-02-03T17:16:38.930Z 6 Acked 1005 Write 1 Done Done Sent false
163622024-02-03T17:16:38.931Z 7 Acked 1006 Flush 0 Done Done Sent false
163632024-02-03T17:16:38.931Z 8 Acked 1007 Flush 0 Done Done Sent false
163642024-02-03T17:16:38.931Z 9 Acked 1008 Flush 0 Done Done Sent false
163652024-02-03T17:16:38.931Z 10 Acked 1009 Write 1 Done Done Sent false
163662024-02-03T17:16:38.931Z 11 Acked 1010 Read 2 Done Done Sent false
163672024-02-03T17:16:38.931Z 12 Acked 1011 Read 5 Done Done Sent false
163682024-02-03T17:16:38.931Z 13 Acked 1012 Write 2 Done Done Sent false
163692024-02-03T17:16:38.931Z 14 Acked 1013 Flush 0 Done Done Sent false
163702024-02-03T17:16:38.931Z 15 Acked 1014 Write 5 Done Done Sent false
163712024-02-03T17:16:38.931Z 16 Acked 1015 Read 10 Done Done Sent false
163722024-02-03T17:16:38.931Z 17 Acked 1016 Write 9 Done Done Sent false
163732024-02-03T17:16:38.931Z 18 Acked 1017 Read 6 Done Done Sent false
163742024-02-03T17:16:38.931Z 19 Acked 1018 Read 1 Done Done Sent false
163752024-02-03T17:16:38.931Z 20 Acked 1019 Read 4 Done Done Sent false
163762024-02-03T17:16:38.931Z 21 Acked 1020 Flush 0 Done Done Sent false
163772024-02-03T17:16:38.931Z 22 Acked 1021 Write 9 Done Done Sent false
163782024-02-03T17:16:38.932Z 23 Acked 1022 Write 9 Done Done Sent false
163792024-02-03T17:16:38.932Z 24 Acked 1023 Read 2 Done Done Sent false
163802024-02-03T17:16:38.932Z 25 Acked 1024 Write 1 Done Done Sent false
163812024-02-03T17:16:38.932Z 26 Acked 1025 Read 6 Done Done Sent false
163822024-02-03T17:16:38.932Z 27 Acked 1026 Write 7 Done Done Sent false
163832024-02-03T17:16:38.932Z 28 Acked 1027 Read 7 Done Done Sent false
163842024-02-03T17:16:38.932Z 29 Acked 1028 Write 9 Done Done Sent false
163852024-02-03T17:16:38.932Z 30 Acked 1029 Read 2 Sent Done Sent false
163862024-02-03T17:16:38.932Z STATES DS:0 DS:1 DS:2 TOTAL
163872024-02-03T17:16:38.932Z New 0 0 0 0
163882024-02-03T17:16:38.932Z Sent 1 0 30 31
163892024-02-03T17:16:38.932Z Done 29 30 0 59
163902024-02-03T17:16:38.932Z Skipped 0 0 0 0
163912024-02-03T17:16:38.932Z Error 0 0 0 0
163922024-02-03T17:16:38.933ZLast Flush: 1020 1020 0
163932024-02-03T17:16:38.933ZDownstairs last five completed:
163942024-02-03T17:16:38.933ZUpstairs last five completed: 30 29 28 27 26
163952024-02-03T17:16:38.933Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:16:38.079296243Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1735}
163962024-02-03T17:16:38.933ZWrote out file "/tmp/test_repair_verify.data"
163972024-02-03T17:16:38.949Z
163982024-02-03T17:16:38.950ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
163992024-02-03T17:16:38.950ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
164002024-02-03T17:16:38.950Z 0 000-019 150 150 148 264 264 254 F F F <---
164012024-02-03T17:16:38.950Z 1 020-039 148 148 148 254 254 254 F F F
164022024-02-03T17:16:38.950Z 2 040-059 146 146 146 253 253 253 F F F
164032024-02-03T17:16:38.950Z 3 060-079 135 135 135 235 235 235 T T F <---
164042024-02-03T17:16:38.950Z 4 080-099 150 150 146 263 263 253 T T F <---
164052024-02-03T17:16:38.950Z 5 100-119 150 150 144 263 263 249 F F F <---
164062024-02-03T17:16:38.950Z 6 120-139 144 144 144 249 249 249 F F F
164072024-02-03T17:16:38.950Z 7 140-159 146 146 146 253 253 253 F F F
164082024-02-03T17:16:38.950Z 8 160-179 148 148 148 256 256 256 F F F
164092024-02-03T17:16:38.950Z 9 180-199 150 150 146 260 260 253 F F F <---
164102024-02-03T17:16:38.950Z 10 200-219 150 150 149 260 260 257 T T F <---
164112024-02-03T17:16:38.950Z 11 220-239 146 146 146 253 253 253 F F F
164122024-02-03T17:16:38.951Z 12 240-259 150 150 149 260 260 257 F F F <---
164132024-02-03T17:16:38.951Z 13 260-279 150 150 149 264 264 257 F F F <---
164142024-02-03T17:16:38.951Z 14 280-299 149 149 149 257 257 257 F F F
164152024-02-03T17:16:38.951Z 15 300-319 146 146 146 253 253 253 F F F
164162024-02-03T17:16:38.951Z 16 320-339 149 149 149 257 257 257 T T F <---
164172024-02-03T17:16:38.951Z 17 340-359 146 146 146 253 253 253 T T F <---
164182024-02-03T17:16:38.951Z 18 360-379 149 149 149 257 257 257 T T F <---
164192024-02-03T17:16:38.951Z 19 380-399 149 149 149 257 257 257 T T F <---
164202024-02-03T17:16:38.951Z 20 400-419 146 146 146 253 253 253 F F F
164212024-02-03T17:16:38.952Z 21 420-439 146 146 146 253 253 253 F F F
164222024-02-03T17:16:38.952Z 22 440-459 150 150 146 258 258 253 F F F <---
164232024-02-03T17:16:38.952Z 23 460-479 148 148 148 254 254 254 T T F <---
164242024-02-03T17:16:38.952Z 24 480-499 148 148 148 254 254 254 F F F
164252024-02-03T17:16:38.952Z 25 500-519 148 148 148 254 254 254 F F F
164262024-02-03T17:16:38.952Z 26 520-539 148 148 148 256 256 256 F F F
164272024-02-03T17:16:38.952Z 27 540-559 148 148 148 256 256 256 F F F
164282024-02-03T17:16:38.952Z 28 560-579 149 149 149 257 257 257 F F F
164292024-02-03T17:16:38.952Z 29 580-599 146 146 146 253 253 253 F F F
164302024-02-03T17:16:38.952ZMax gen: 150, Max flush: 264
164312024-02-03T17:16:38.952ZError: Difference in extent metadata found!
164322024-02-03T17:16:38.953ZA Difference in extent metadata is expected here
164332024-02-03T17:16:38.953ZOn loop 75
164342024-02-03T17:16:38.953Z
164352024-02-03T17:16:38.953ZVerifying data now
164362024-02-03T17:16:43.762ZNo change after verify
164372024-02-03T17:16:43.762ZLoop: 75 Downstairs dump after verify (and repair):
164382024-02-03T17:16:43.790ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
164392024-02-03T17:16:43.790Z 0 000-019 150 150 150 264 264 264 F F F
164402024-02-03T17:16:43.791Z 1 020-039 148 148 148 254 254 254 F F F
164412024-02-03T17:16:43.791Z 2 040-059 146 146 146 253 253 253 F F F
164422024-02-03T17:16:43.791Z 3 060-079 151 151 151 265 265 265 F F F
164432024-02-03T17:16:43.813Z 4 080-099 151 151 151 265 265 265 F F F
164442024-02-03T17:16:43.813Z 5 100-119 150 150 150 263 263 263 F F F
164452024-02-03T17:16:43.813Z 6 120-139 144 144 144 249 249 249 F F F
164462024-02-03T17:16:43.813Z 7 140-159 146 146 146 253 253 253 F F F
164472024-02-03T17:16:43.813Z 8 160-179 148 148 148 256 256 256 F F F
164482024-02-03T17:16:43.813Z 9 180-199 150 150 150 260 260 260 F F F
164492024-02-03T17:16:43.813Z 10 200-219 151 151 151 265 265 265 F F F
164502024-02-03T17:16:43.814Z 11 220-239 146 146 146 253 253 253 F F F
164512024-02-03T17:16:43.814Z 12 240-259 150 150 150 260 260 260 F F F
164522024-02-03T17:16:43.814Z 13 260-279 150 150 150 264 264 264 F F F
164532024-02-03T17:16:43.814Z 14 280-299 149 149 149 257 257 257 F F F
164542024-02-03T17:16:43.814Z 15 300-319 146 146 146 253 253 253 F F F
164552024-02-03T17:16:43.814Z 16 320-339 151 151 151 265 265 265 F F F
164562024-02-03T17:16:43.814Z 17 340-359 151 151 151 265 265 265 F F F
164572024-02-03T17:16:43.814Z 18 360-379 151 151 151 265 265 265 F F F
164582024-02-03T17:16:43.814Z 19 380-399 151 151 151 265 265 265 F F F
164592024-02-03T17:16:43.814Z 20 400-419 146 146 146 253 253 253 F F F
164602024-02-03T17:16:43.814Z 21 420-439 146 146 146 253 253 253 F F F
164612024-02-03T17:16:43.814Z 22 440-459 150 150 150 258 258 258 F F F
164622024-02-03T17:16:43.815Z 23 460-479 151 151 151 265 265 265 F F F
164632024-02-03T17:16:43.815Z 24 480-499 148 148 148 254 254 254 F F F
164642024-02-03T17:16:43.815Z 25 500-519 148 148 148 254 254 254 F F F
164652024-02-03T17:16:43.815Z 26 520-539 148 148 148 256 256 256 F F F
164662024-02-03T17:16:43.815Z 27 540-559 148 148 148 256 256 256 F F F
164672024-02-03T17:16:43.815Z 28 560-579 149 149 149 257 257 257 F F F
164682024-02-03T17:16:43.815Z 29 580-599 146 146 146 253 253 253 F F F
164692024-02-03T17:16:43.815ZMax gen: 151, Max flush: 265
164702024-02-03T17:16:43.815Z
164712024-02-03T17:16:43.815ZBegin loop 76
164722024-02-03T17:16:43.815ZDownstairs to restart: 0
164732024-02-03T17:16:43.826Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.394126429Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743}
164742024-02-03T17:16:43.827Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.394718242Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743}
164752024-02-03T17:16:44.253Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.394801699Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743}
164762024-02-03T17:16:44.253Z{"msg":"Crucible a5e166b9-3209-4049-9cae-853b674768bd has session id: e77e3ac6-9d86-4f13-a652-4a3daef6bc72","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.394878507Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
164772024-02-03T17:16:44.253Z{"msg":"Crucible runtime is spawned
164782024-02-03T17:16:44.253ZUpstairs opts: Upstairs UUID: a5e166b9-3209-4049-9cae-853b674768bd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.395023509Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
164792024-02-03T17:16:44.253Z{"msg":"Crucible stats registered with UUID: a5e166b9-3209-4049-9cae-853b674768bd","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.395160719Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
164802024-02-03T17:16:44.253Z{"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.395244182Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743}
164812024-02-03T17:16:44.253Z{"msg":"a5e166b9-3209-4049-9cae-853b674768bd active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.395296002Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
164822024-02-03T17:16:44.253Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.395359958Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
164832024-02-03T17:16:44.253Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.395387693Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
164842024-02-03T17:16:44.253Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.395413886Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
164852024-02-03T17:16:44.253Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.395477234Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
164862024-02-03T17:16:44.253Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.395504059Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
164872024-02-03T17:16:44.253Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.395527156Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
164882024-02-03T17:16:44.254Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.395714995Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"":"io task","client":"2","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
164892024-02-03T17:16:44.254Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.395973306Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"":"io task","client":"0","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
164902024-02-03T17:16:44.254Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.396158574Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"":"io task","client":"1","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
164912024-02-03T17:16:44.254Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.396199843Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"":"io task","client":"0","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
164922024-02-03T17:16:44.254Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.396241782Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"":"io task","client":"1","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
164932024-02-03T17:16:44.254Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.396285006Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"":"io task","client":"2","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
164942024-02-03T17:16:44.254Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.397212445Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
164952024-02-03T17:16:44.254Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.397290108Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
164962024-02-03T17:16:44.254Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.397335461Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
164972024-02-03T17:16:44.254Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.397413894Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
164982024-02-03T17:16:44.254Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.39817454Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
164992024-02-03T17:16:44.254Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.398286744Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165002024-02-03T17:16:44.254Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.398399804Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165012024-02-03T17:16:44.254Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.398445763Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165022024-02-03T17:16:44.254Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.399196855Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165032024-02-03T17:16:44.254Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.399270454Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165042024-02-03T17:16:44.254Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.399297419Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165052024-02-03T17:16:44.254Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.39932583Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165062024-02-03T17:16:44.254Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.399408917Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165072024-02-03T17:16:44.254Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.399491764Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165082024-02-03T17:16:44.254Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.399620998Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165092024-02-03T17:16:44.254Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.400354748Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165102024-02-03T17:16:44.254Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.400413499Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165112024-02-03T17:16:44.255Z{"msg":"[0]R flush_numbers[0..12]: [264, 254, 253, 265, 265, 263, 249, 253, 256, 260, 265, 253]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.400443257Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165122024-02-03T17:16:44.255Z{"msg":"[0]R generation[0..12]: [150, 148, 146, 151, 151, 150, 144, 146, 148, 150, 151, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.400498172Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165132024-02-03T17:16:44.255Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.400525883Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165142024-02-03T17:16:44.255Z{"msg":"[1]R flush_numbers[0..12]: [264, 254, 253, 265, 265, 263, 249, 253, 256, 260, 265, 253]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.400550389Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165152024-02-03T17:16:44.255Z{"msg":"[1]R generation[0..12]: [150, 148, 146, 151, 151, 150, 144, 146, 148, 150, 151, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.400599015Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165162024-02-03T17:16:44.255Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.400625288Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165172024-02-03T17:16:44.255ZWait for a query_work_queue command to finish before sending IO
165182024-02-03T17:16:44.255Z{"msg":"[2]R flush_numbers[0..12]: [264, 254, 253, 265, 265, 263, 249, 253, 256, 260, 265, 253]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.400678242Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165192024-02-03T17:16:44.255Z{"msg":"[2]R generation[0..12]: [150, 148, 146, 151, 151, 150, 144, 146, 148, 150, 151, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.400704282Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165202024-02-03T17:16:44.255Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.40075291Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165212024-02-03T17:16:44.255Z{"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.400777367Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165222024-02-03T17:16:44.255Z{"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.400825092Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165232024-02-03T17:16:44.255Z{"msg":"Next flush: 266","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.400848751Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165242024-02-03T17:16:44.255Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.400895526Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165252024-02-03T17:16:44.255Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.400920095Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165262024-02-03T17:16:44.255Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.400984243Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165272024-02-03T17:16:44.255Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.401008852Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165282024-02-03T17:16:44.255Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.40103072Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165292024-02-03T17:16:44.255Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.401053134Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165302024-02-03T17:16:44.255Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.401074575Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165312024-02-03T17:16:44.255Z{"msg":"a5e166b9-3209-4049-9cae-853b674768bd is now active with session: e77e3ac6-9d86-4f13-a652-4a3daef6bc72","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.401105608Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165322024-02-03T17:16:44.255Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.401126312Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743,"session_id":"e77e3ac6-9d86-4f13-a652-4a3daef6bc72"}
165332024-02-03T17:16:44.255Z{"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:43.40114651Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743}
165342024-02-03T17:16:44.255ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
165352024-02-03T17:16:44.256ZLoading write count information from file "/tmp/test_repair_verify.data"
165362024-02-03T17:16:44.256ZRun Repair workload
165372024-02-03T17:16:44.256Z01/30 Write block 326 len 32768 data: 10 15 14 14 13 13 11 12
165382024-02-03T17:16:44.256Z02/30 Read block 395 len 12288
165392024-02-03T17:16:44.267Z03/30 Read block 189 len 8192
165402024-02-03T17:16:44.267Z04/30 Read block 339 len 16384
165412024-02-03T17:16:44.267Z05/30 Write block 321 len 32768 data: 13 12 14 13 12 11 16 15
165422024-02-03T17:16:44.267Z06/30 Read block 353 len 32768
165432024-02-03T17:16:44.267Z07/30 Write block 248 len 8192 data: 13 13
165442024-02-03T17:16:44.267Z08/30 Flush
165452024-02-03T17:16:44.267Z09/30 Write block 289 len 12288 data: 15 13 12
165462024-02-03T17:16:44.267Z10/30 Read block 530 len 16384
165472024-02-03T17:16:44.267Z11/30 Flush
165482024-02-03T17:16:44.267Z12/30 Write block 308 len 24576 data: 9 11 13 11 13 12
165492024-02-03T17:16:44.267Z13/30 Read block 336 len 8192
165502024-02-03T17:16:44.267Z14/30 Write block 198 len 4096 data: 14
165512024-02-03T17:16:44.268Z15/30 Read block 545 len 28672
165522024-02-03T17:16:44.268Z16/30 Read block 366 len 36864
165532024-02-03T17:16:44.268Z17/30 Write block 389 len 36864 data: 6 13 14 19 18 15 13 13 11
165542024-02-03T17:16:44.268Z18/30 Write block 155 len 12288 data: 11 13 11
165552024-02-03T17:16:44.268Z19/30 Read block 155 len 32768
165562024-02-03T17:16:44.268Z20/30 Read block 136 len 20480
165572024-02-03T17:16:44.268Z21/30 Write block 432 len 36864 data: 14 12 10 14 12 13 14 15 16
165582024-02-03T17:16:44.268Z22/30 Read block 492 len 36864
165592024-02-03T17:16:44.268Z23/30 Read block 67 len 4096
165602024-02-03T17:16:44.268Z24/30 Write block 170 len 32768 data: 17 14 11 11 11 10 10 14
165612024-02-03T17:16:44.268Z25/30 Write block 68 len 16384 data: 8 6 7 4
165622024-02-03T17:16:44.268Z26/30 Write block 546 len 20480 data: 10 13 11 13 15
165632024-02-03T17:16:44.268Z27/30 Write block 482 len 24576 data: 11 10 9 10 10 7
165642024-02-03T17:16:44.268Z28/30 Write block 321 len 4096 data: 14
165652024-02-03T17:16:44.268Z29/30 Read block 326 len 40960
165662024-02-03T17:16:44.268Z30/30 Read block 550 len 12288
165672024-02-03T17:16:44.268Z----------------------------------------------------------------
165682024-02-03T17:16:44.268Z Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:30
165692024-02-03T17:16:44.268ZStates: Active Active Active
165702024-02-03T17:16:44.268ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
165712024-02-03T17:16:44.268Z 1 Acked 1000 Write 8 Sent Done Done false
165722024-02-03T17:16:44.269Z 2 Acked 1001 Read 3 Sent Done Done false
165732024-02-03T17:16:44.269Z 3 Acked 1002 Read 2 Sent Done Done false
165742024-02-03T17:16:44.269Z 4 Acked 1003 Read 4 Sent Done Done false
165752024-02-03T17:16:44.269Z 5 Acked 1004 Write 8 Sent Done Done false
165762024-02-03T17:16:44.269Z 6 Acked 1005 Read 8 Sent Done Done false
165772024-02-03T17:16:44.269Z 7 Acked 1006 Write 2 Sent Done Done false
165782024-02-03T17:16:44.269Z 8 Acked 1007 Flush 0 Sent Done Done false
165792024-02-03T17:16:44.269Z 9 Acked 1008 Write 3 Sent Done Done false
165802024-02-03T17:16:44.269Z 10 Acked 1009 Read 4 Sent Done Done false
165812024-02-03T17:16:44.269Z 11 Acked 1010 Flush 0 Sent Done Done false
165822024-02-03T17:16:44.269Z 12 Acked 1011 Write 6 Sent Done Done false
165832024-02-03T17:16:44.269Z 13 Acked 1012 Read 2 Sent Done Done false
165842024-02-03T17:16:44.269Z 14 Acked 1013 Write 1 Sent Done Done false
165852024-02-03T17:16:44.269Z 15 Acked 1014 Read 7 Sent Done Done false
165862024-02-03T17:16:44.269Z 16 Acked 1015 Read 9 Sent Done Done false
165872024-02-03T17:16:44.269Z 17 Acked 1016 Write 9 Sent Done Done false
165882024-02-03T17:16:44.269Z 18 Acked 1017 Write 3 Sent Done Done false
165892024-02-03T17:16:44.269Z 19 Acked 1018 Read 8 Sent Done Done false
165902024-02-03T17:16:44.269Z 20 Acked 1019 Read 5 Sent Done Done false
165912024-02-03T17:16:44.269Z 21 Acked 1020 Write 9 Sent Done Done false
165922024-02-03T17:16:44.269Z 22 Acked 1021 Read 9 Sent Done Done false
165932024-02-03T17:16:44.270Z 23 Acked 1022 Read 1 Sent Done Done false
165942024-02-03T17:16:44.270Z 24 Acked 1023 Write 8 Sent Done Done false
165952024-02-03T17:16:44.270Z 25 Acked 1024 Write 4 Sent Done Done false
165962024-02-03T17:16:44.270Z 26 Acked 1025 Write 5 Sent Done Done false
165972024-02-03T17:16:44.270Z 27 Acked 1026 Write 6 Sent Done Done false
165982024-02-03T17:16:44.270Z 28 Acked 1027 Write 1 Sent Done Done false
165992024-02-03T17:16:44.270Z 29 Acked 1028 Read 10 Sent Done Done false
166002024-02-03T17:16:44.270Z 30 Acked 1029 Read 3 Sent Sent Done false
166012024-02-03T17:16:44.270Z STATES DS:0 DS:1 DS:2 TOTAL
166022024-02-03T17:16:44.270Z New 0 0 0 0
166032024-02-03T17:16:44.270Z Sent 30 1 0 31
166042024-02-03T17:16:44.270Z Done 0 29 30 59
166052024-02-03T17:16:44.270Z Skipped 0 0 0 0
166062024-02-03T17:16:44.270Z Error 0 0 0 0
166072024-02-03T17:16:44.270ZLast Flush: 0 1010 1010
166082024-02-03T17:16:44.270ZDownstairs last five completed:
166092024-02-03T17:16:44.270ZUpstairs last five completed: 30 29 28 27 26
166102024-02-03T17:16:44.270Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:16:43.450471159Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1743}
166112024-02-03T17:16:44.270ZWrote out file "/tmp/test_repair_verify.data"
166122024-02-03T17:16:44.270Z
166132024-02-03T17:16:44.270ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
166142024-02-03T17:16:44.270ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
166152024-02-03T17:16:44.270Z 0 000-019 150 150 150 264 264 264 F F F
166162024-02-03T17:16:44.270Z 1 020-039 148 148 148 254 254 254 F F F
166172024-02-03T17:16:44.271Z 2 040-059 146 146 146 253 253 253 F F F
166182024-02-03T17:16:44.271Z 3 060-079 151 151 151 265 265 265 F T T <---
166192024-02-03T17:16:44.271Z 4 080-099 151 151 151 265 265 265 F F F
166202024-02-03T17:16:44.271Z 5 100-119 150 150 150 263 263 263 F F F
166212024-02-03T17:16:44.271Z 6 120-139 144 144 144 249 249 249 F F F
166222024-02-03T17:16:44.271Z 7 140-159 146 146 146 253 253 253 F T T <---
166232024-02-03T17:16:44.271Z 8 160-179 148 148 148 256 256 256 F T T <---
166242024-02-03T17:16:44.271Z 9 180-199 150 150 150 260 260 260 F T T <---
166252024-02-03T17:16:44.271Z 10 200-219 151 151 151 265 265 265 F F F
166262024-02-03T17:16:44.271Z 11 220-239 146 146 146 253 253 253 F F F
166272024-02-03T17:16:44.271Z 12 240-259 150 152 152 260 266 266 F F F <---
166282024-02-03T17:16:44.271Z 13 260-279 150 150 150 264 264 264 F F F
166292024-02-03T17:16:44.271Z 14 280-299 149 152 152 257 267 267 F F F <---
166302024-02-03T17:16:44.271Z 15 300-319 146 146 146 253 253 253 F T T <---
166312024-02-03T17:16:44.271Z 16 320-339 151 152 152 265 266 266 F T T <---
166322024-02-03T17:16:44.271Z 17 340-359 151 151 151 265 265 265 F F F
166332024-02-03T17:16:44.272Z 18 360-379 151 151 151 265 265 265 F F F
166342024-02-03T17:16:44.272Z 19 380-399 151 151 151 265 265 265 F T T <---
166352024-02-03T17:16:44.272Z 20 400-419 146 146 146 253 253 253 F F F
166362024-02-03T17:16:44.272Z 21 420-439 146 146 146 253 253 253 F T T <---
166372024-02-03T17:16:44.272Z 22 440-459 150 150 150 258 258 258 F T T <---
166382024-02-03T17:16:44.272Z 23 460-479 151 151 151 265 265 265 F F F
166392024-02-03T17:16:44.284Z 24 480-499 148 148 148 254 254 254 F T T <---
166402024-02-03T17:16:44.284Z 25 500-519 148 148 148 254 254 254 F F F
166412024-02-03T17:16:44.284Z 26 520-539 148 148 148 256 256 256 F F F
166422024-02-03T17:16:44.284Z 27 540-559 148 148 148 256 256 256 F T T <---
166432024-02-03T17:16:44.285Z 28 560-579 149 149 149 257 257 257 F F F
166442024-02-03T17:16:44.285Z 29 580-599 146 146 146 253 253 253 F F F
166452024-02-03T17:16:44.285ZMax gen: 152, Max flush: 267
166462024-02-03T17:16:44.285ZError: Difference in extent metadata found!
166472024-02-03T17:16:44.285ZA Difference in extent metadata is expected here
166482024-02-03T17:16:44.286ZOn loop 76
166492024-02-03T17:16:44.286Z
166502024-02-03T17:16:44.286ZVerifying data now
166512024-02-03T17:16:49.101ZNo change after verify
166522024-02-03T17:16:49.101ZLoop: 76 Downstairs dump after verify (and repair):
166532024-02-03T17:16:49.124ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
166542024-02-03T17:16:49.125Z 0 000-019 150 150 150 264 264 264 F F F
166552024-02-03T17:16:49.125Z 1 020-039 148 148 148 254 254 254 F F F
166562024-02-03T17:16:49.125Z 2 040-059 146 146 146 253 253 253 F F F
166572024-02-03T17:16:49.131Z 3 060-079 153 153 153 268 268 268 F F F
166582024-02-03T17:16:49.132Z 4 080-099 151 151 151 265 265 265 F F F
166592024-02-03T17:16:49.132Z 5 100-119 150 150 150 263 263 263 F F F
166602024-02-03T17:16:49.132Z 6 120-139 144 144 144 249 249 249 F F F
166612024-02-03T17:16:49.132Z 7 140-159 153 153 153 268 268 268 F F F
166622024-02-03T17:16:49.132Z 8 160-179 153 153 153 268 268 268 F F F
166632024-02-03T17:16:49.132Z 9 180-199 153 153 153 268 268 268 F F F
166642024-02-03T17:16:49.132Z 10 200-219 151 151 151 265 265 265 F F F
166652024-02-03T17:16:49.132Z 11 220-239 146 146 146 253 253 253 F F F
166662024-02-03T17:16:49.132Z 12 240-259 152 152 152 266 266 266 F F F
166672024-02-03T17:16:49.132Z 13 260-279 150 150 150 264 264 264 F F F
166682024-02-03T17:16:49.133Z 14 280-299 152 152 152 267 267 267 F F F
166692024-02-03T17:16:49.133Z 15 300-319 153 153 153 268 268 268 F F F
166702024-02-03T17:16:49.133Z 16 320-339 153 153 153 268 268 268 F F F
166712024-02-03T17:16:49.133Z 17 340-359 151 151 151 265 265 265 F F F
166722024-02-03T17:16:49.133Z 18 360-379 151 151 151 265 265 265 F F F
166732024-02-03T17:16:49.133Z 19 380-399 153 153 153 268 268 268 F F F
166742024-02-03T17:16:49.134Z 20 400-419 146 146 146 253 253 253 F F F
166752024-02-03T17:16:49.134Z 21 420-439 153 153 153 268 268 268 F F F
166762024-02-03T17:16:49.134Z 22 440-459 153 153 153 268 268 268 F F F
166772024-02-03T17:16:49.134Z 23 460-479 151 151 151 265 265 265 F F F
166782024-02-03T17:16:49.134Z 24 480-499 153 153 153 268 268 268 F F F
166792024-02-03T17:16:49.134Z 25 500-519 148 148 148 254 254 254 F F F
166802024-02-03T17:16:49.134Z 26 520-539 148 148 148 256 256 256 F F F
166812024-02-03T17:16:49.134Z 27 540-559 153 153 153 268 268 268 F F F
166822024-02-03T17:16:49.135Z 28 560-579 149 149 149 257 257 257 F F F
166832024-02-03T17:16:49.135Z 29 580-599 146 146 146 253 253 253 F F F
166842024-02-03T17:16:49.135ZMax gen: 153, Max flush: 268
166852024-02-03T17:16:49.135Z
166862024-02-03T17:16:49.135ZBegin loop 77
166872024-02-03T17:16:49.136ZDownstairs to restart: 2
166882024-02-03T17:16:49.166Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.733062038Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751}
166892024-02-03T17:16:49.174Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.733552607Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751}
166902024-02-03T17:16:49.174Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.733628245Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751}
166912024-02-03T17:16:49.174Z{"msg":"Crucible ee125a9d-cde6-430d-bb7f-11fa1ca3e5a9 has session id: 050a1369-4c17-45d4-bd58-64b436cd19e7","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.733724558Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
166922024-02-03T17:16:49.174Z{"msg":"Upstairs opts: Upstairs UUID: ee125a9d-cde6-430d-bb7f-11fa1ca3e5a9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time"Crucible runtime is spawned
166932024-02-03T17:16:49.174Z:"2024-02-03T17:16:48.733806438Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
166942024-02-03T17:16:49.174Z{"msg":"Crucible stats registered with UUID: ee125a9d-cde6-430d-bb7f-11fa1ca3e5a9","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.733953231Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
166952024-02-03T17:16:49.174Z{"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.733992738Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751}
166962024-02-03T17:16:49.174Z{"msg":"ee125a9d-cde6-430d-bb7f-11fa1ca3e5a9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.734177273Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
166972024-02-03T17:16:49.174Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.734236852Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
166982024-02-03T17:16:49.174Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.734264174Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
166992024-02-03T17:16:49.174Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.734335186Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167002024-02-03T17:16:49.174Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.73447481Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167012024-02-03T17:16:49.174Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.734544541Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167022024-02-03T17:16:49.174Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.734688007Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167032024-02-03T17:16:49.174Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.734737182Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"":"io task","client":"0","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167042024-02-03T17:16:49.175Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.734827532Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"":"io task","client":"2","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167052024-02-03T17:16:49.175Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.734876951Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"":"io task","client":"1","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167062024-02-03T17:16:49.175Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.734954953Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"":"io task","client":"2","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167072024-02-03T17:16:49.175Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.734999139Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"":"io task","client":"0","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167082024-02-03T17:16:49.175Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.73504618Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"":"io task","client":"1","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167092024-02-03T17:16:49.175Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.736393439Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167102024-02-03T17:16:49.175Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.736582268Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167112024-02-03T17:16:49.175Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.736659908Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167122024-02-03T17:16:49.175Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.736688082Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167132024-02-03T17:16:49.175Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.73726509Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167142024-02-03T17:16:49.175Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.73744414Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167152024-02-03T17:16:49.175Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.737582657Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167162024-02-03T17:16:49.175Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.73797993Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167172024-02-03T17:16:49.175Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.738304126Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167182024-02-03T17:16:49.175Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.738377233Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167192024-02-03T17:16:49.175Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.738404782Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167202024-02-03T17:16:49.175Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.73848951Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167212024-02-03T17:16:49.175Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.738855897Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167222024-02-03T17:16:49.175Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.738921447Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167232024-02-03T17:16:49.175Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.738948044Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167242024-02-03T17:16:49.175Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.73910645Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167252024-02-03T17:16:49.175Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.739191814Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167262024-02-03T17:16:49.175Z{"msg":"[0]R flush_numbers[0..12]: [264, 254, 253, 268, 265, 263, 249, 268, 268, 268, 265, 253]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.739233391Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167272024-02-03T17:16:49.175Z{"msg":"[0]R generation[0..12]: [150, 148, 146, 153, 151, 150, 144, 153, 153, 153, 151, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.73930303Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167282024-02-03T17:16:49.175Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.739332905Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167292024-02-03T17:16:49.176Z{"msg":"[1]R flush_numbers[0..12]: [264, 254, 253, 268, 265, 263, 249, 268, 268, 268, 265, 253]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.739383318Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167302024-02-03T17:16:49.176Z{"msg":"[1]R generation[0..12]: [150, 148, 146, 153, 151, 150, 144, 153, 153, 153, 151, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.739409796Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167312024-02-03T17:16:49.176Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.739433607Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167322024-02-03T17:16:49.176Z{"Wait for a query_work_queue command to finish before sending IO
167332024-02-03T17:16:49.176Zmsg":"[2]R flush_numbers[0..12]: [264, 254, 253, 268, 265, 263, 249, 268, 268, 268, 265, 253]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.739485578Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167342024-02-03T17:16:49.176Z{"msg":"[2]R generation[0..12]: [150, 148, 146, 153, 151, 150, 144, 153, 153, 153, 151, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.739510935Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167352024-02-03T17:16:49.176Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.739534356Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167362024-02-03T17:16:49.176Z{"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.739589232Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167372024-02-03T17:16:49.176Z{"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.739616405Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167382024-02-03T17:16:49.176Z{"msg":"Next flush: 269","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.739638046Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167392024-02-03T17:16:49.176Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.739685525Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167402024-02-03T17:16:49.176Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.739709917Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167412024-02-03T17:16:49.176Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.739729922Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167422024-02-03T17:16:49.176Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.739751513Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167432024-02-03T17:16:49.176Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.73979782Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167442024-02-03T17:16:49.176Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.739822345Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167452024-02-03T17:16:49.176Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.739842771Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167462024-02-03T17:16:49.177Z{"msg":"ee125a9d-cde6-430d-bb7f-11fa1ca3e5a9 is now active with session: 050a1369-4c17-45d4-bd58-64b436cd19e7","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.739864173Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167472024-02-03T17:16:49.177Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.739908438Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751,"session_id":"050a1369-4c17-45d4-bd58-64b436cd19e7"}
167482024-02-03T17:16:49.177Z{"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:48.739930916Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751}
167492024-02-03T17:16:49.177ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
167502024-02-03T17:16:49.177ZLoading write count information from file "/tmp/test_repair_verify.data"
167512024-02-03T17:16:49.177ZRun Repair workload
167522024-02-03T17:16:49.177Z01/30 Write block 206 len 24576 data: 11 12 13 12 15 12
167532024-02-03T17:16:49.177Z02/30 Write block 581 len 4096 data: 9
167542024-02-03T17:16:49.182Z03/30 Read block 528 len 16384
167552024-02-03T17:16:49.182Z04/30 Read block 160 len 20480
167562024-02-03T17:16:49.183Z05/30 Write block 586 len 20480 data: 10 10 13 11 12
167572024-02-03T17:16:49.183Z06/30 Read block 38 len 8192
167582024-02-03T17:16:49.183Z07/30 Read block 40 len 36864
167592024-02-03T17:16:49.183Z08/30 Write block 226 len 4096 data: 6
167602024-02-03T17:16:49.183Z09/30 Read block 253 len 40960
167612024-02-03T17:16:49.187Z10/30 Read block 471 len 28672
167622024-02-03T17:16:49.187Z11/30 Write block 236 len 16384 data: 10 9 8 8
167632024-02-03T17:16:49.187Z12/30 Write block 139 len 28672 data: 12 13 16 16 14 10 9
167642024-02-03T17:16:49.187Z13/30 Write block 19 len 36864 data: 21 17 20 16 15 14 13 11 12
167652024-02-03T17:16:49.191Z14/30 Read block 109 len 24576
167662024-02-03T17:16:49.191Z15/30 Write block 28 len 40960 data: 13 11 11 15 13 12 13 12 14 12
167672024-02-03T17:16:49.199Z16/30 Write block 111 len 36864 data: 8 10 8 9 10 11 15 13 12
167682024-02-03T17:16:49.199Z17/30 Read block 342 len 4096
167692024-02-03T17:16:49.199Z18/30 Write block 324 len 4096 data: 14
167702024-02-03T17:16:49.203Z19/30 Write block 245 len 32768 data: 9 11 9 14 14 12 13 12
167712024-02-03T17:16:49.203Z20/30 Read block 471 len 24576
167722024-02-03T17:16:49.203Z21/30 Read block 395 len 36864
167732024-02-03T17:16:49.203Z22/30 Write block 119 len 20480 data: 13 11 10 11 10
167742024-02-03T17:16:49.203Z23/30 Read block 320 len 12288
167752024-02-03T17:16:49.207Z24/30 Read block 306 len 12288
167762024-02-03T17:16:49.207Z25/30 Read block 348 len 36864
167772024-02-03T17:16:49.207Z26/30 Read block 237 len 20480
167782024-02-03T17:16:49.212Z27/30 Read block 218 len 36864
167792024-02-03T17:16:49.212Z28/30 Write block 71 len 8192 data: 5 4
167802024-02-03T17:16:49.212Z29/30 Read block 154 len 28672
167812024-02-03T17:16:49.218Z30/30 Read block 82 len 8192
167822024-02-03T17:16:49.218Z----------------------------------------------------------------
167832024-02-03T17:16:49.218Z Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30
167842024-02-03T17:16:49.218ZStates: Active Active Active
167852024-02-03T17:16:49.218ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
167862024-02-03T17:16:49.218Z 1 Acked 1000 Write 6 Done Done Done false
167872024-02-03T17:16:49.218Z 2 Acked 1001 Write 1 Done Done Done false
167882024-02-03T17:16:49.218Z 3 Acked 1002 Read 4 Done Done Done false
167892024-02-03T17:16:49.218Z 4 Acked 1003 Read 5 Done Done Done false
167902024-02-03T17:16:49.218Z 5 Acked 1004 Write 5 Done Done Sent false
167912024-02-03T17:16:49.218Z 6 Acked 1005 Read 2 Done Done Sent false
167922024-02-03T17:16:49.218Z 7 Acked 1006 Read 9 Done Done Sent false
167932024-02-03T17:16:49.219Z 8 Acked 1007 Write 1 Done Done Sent false
167942024-02-03T17:16:49.219Z 9 Acked 1008 Read 10 Done Done Sent false
167952024-02-03T17:16:49.219Z 10 Acked 1009 Read 7 Done Done Sent false
167962024-02-03T17:16:49.219Z 11 Acked 1010 Write 4 Done Done Sent false
167972024-02-03T17:16:49.219Z 12 Acked 1011 Write 7 Done Done Sent false
167982024-02-03T17:16:49.219Z 13 Acked 1012 Write 9 Done Done Sent false
167992024-02-03T17:16:49.219Z 14 Acked 1013 Read 6 Done Done Sent false
168002024-02-03T17:16:49.219Z 15 Acked 1014 Write 10 Done Done Sent false
168012024-02-03T17:16:49.219Z 16 Acked 1015 Write 9 Done Done Sent false
168022024-02-03T17:16:49.219Z 17 Acked 1016 Read 1 Done Done Sent false
168032024-02-03T17:16:49.219Z 18 Acked 1017 Write 1 Done Done Sent false
168042024-02-03T17:16:49.219Z 19 Acked 1018 Write 8 Done Done Sent false
168052024-02-03T17:16:49.219Z 20 Acked 1019 Read 6 Done Done Sent false
168062024-02-03T17:16:49.219Z 21 Acked 1020 Read 9 Done Done Sent false
168072024-02-03T17:16:49.219Z 22 Acked 1021 Write 5 Done Done Sent false
168082024-02-03T17:16:49.219Z 23 Acked 1022 Read 3 Done Done Sent false
168092024-02-03T17:16:49.219Z 24 Acked 1023 Read 3 Done Done Sent false
168102024-02-03T17:16:49.219Z 25 Acked 1024 Read 9 Done Done Sent false
168112024-02-03T17:16:49.219Z 26 Acked 1025 Read 5 Done Done Sent false
168122024-02-03T17:16:49.219Z 27 Acked 1026 Read 9 Done Done Sent false
168132024-02-03T17:16:49.219Z 28 Acked 1027 Write 2 Done Done Sent false
168142024-02-03T17:16:49.219Z 29 Acked 1028 Read 7 Done Done Sent false
168152024-02-03T17:16:49.219Z 30 Acked 1029 Read 2 Done Sent Sent false
168162024-02-03T17:16:49.219Z STATES DS:0 DS:1 DS:2 TOTAL
168172024-02-03T17:16:49.219Z New 0 0 0 0
168182024-02-03T17:16:49.220Z Sent 0 1 26 27
168192024-02-03T17:16:49.220Z Done 30 29 4 63
168202024-02-03T17:16:49.220Z Skipped 0 0 0 0
168212024-02-03T17:16:49.220Z Error 0 0 0 0
168222024-02-03T17:16:49.220ZLast Flush: 0 0 0
168232024-02-03T17:16:49.220ZDownstairs last five completed:
168242024-02-03T17:16:49.220ZUpstairs last five completed: 30 29 28 27 26
168252024-02-03T17:16:49.220Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:16:48.78791315Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1751}
168262024-02-03T17:16:49.220ZWrote out file "/tmp/test_repair_verify.data"
168272024-02-03T17:16:49.224Z
168282024-02-03T17:16:49.229ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
168292024-02-03T17:16:49.261ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
168302024-02-03T17:16:49.261Z 0 000-019 150 150 150 264 264 264 T T F <---
168312024-02-03T17:16:49.261Z 1 020-039 148 148 148 254 254 254 T T F <---
168322024-02-03T17:16:49.261Z 2 040-059 146 146 146 253 253 253 F F F
168332024-02-03T17:16:49.262Z 3 060-079 153 153 153 268 268 268 T T F <---
168342024-02-03T17:16:49.262Z 4 080-099 151 151 151 265 265 265 F F F
168352024-02-03T17:16:49.262Z 5 100-119 150 150 150 263 263 263 T T F <---
168362024-02-03T17:16:49.266Z 6 120-139 144 144 144 249 249 249 T T F <---
168372024-02-03T17:16:49.266Z 7 140-159 153 153 153 268 268 268 T T F <---
168382024-02-03T17:16:49.266Z 8 160-179 153 153 153 268 268 268 F F F
168392024-02-03T17:16:49.266Z 9 180-199 153 153 153 268 268 268 F F F
168402024-02-03T17:16:49.266Z 10 200-219 151 151 151 265 265 265 T T T
168412024-02-03T17:16:49.266Z 11 220-239 146 146 146 253 253 253 T T F <---
168422024-02-03T17:16:49.266Z 12 240-259 152 152 152 266 266 266 T T F <---
168432024-02-03T17:16:49.267Z 13 260-279 150 150 150 264 264 264 F F F
168442024-02-03T17:16:49.267Z 14 280-299 152 152 152 267 267 267 F F F
168452024-02-03T17:16:49.267Z 15 300-319 153 153 153 268 268 268 F F F
168462024-02-03T17:16:49.267Z 16 320-339 153 153 153 268 268 268 T T F <---
168472024-02-03T17:16:49.267Z 17 340-359 151 151 151 265 265 265 F F F
168482024-02-03T17:16:49.267Z 18 360-379 151 151 151 265 265 265 F F F
168492024-02-03T17:16:49.267Z 19 380-399 153 153 153 268 268 268 F F F
168502024-02-03T17:16:49.267Z 20 400-419 146 146 146 253 253 253 F F F
168512024-02-03T17:16:49.267Z 21 420-439 153 153 153 268 268 268 F F F
168522024-02-03T17:16:49.267Z 22 440-459 153 153 153 268 268 268 F F F
168532024-02-03T17:16:49.267Z 23 460-479 151 151 151 265 265 265 F F F
168542024-02-03T17:16:49.267Z 24 480-499 153 153 153 268 268 268 F F F
168552024-02-03T17:16:49.267Z 25 500-519 148 148 148 254 254 254 F F F
168562024-02-03T17:16:49.268Z 26 520-539 148 148 148 256 256 256 F F F
168572024-02-03T17:16:49.268Z 27 540-559 153 153 153 268 268 268 F F F
168582024-02-03T17:16:49.268Z 28 560-579 149 149 149 257 257 257 F F F
168592024-02-03T17:16:49.268Z 29 580-599 146 146 146 253 253 253 T T T
168602024-02-03T17:16:49.268ZMax gen: 153, Max flush: 268
168612024-02-03T17:16:49.268ZError: Difference in extent metadata found!
168622024-02-03T17:16:49.268ZA Difference in extent metadata is expected here
168632024-02-03T17:16:49.268ZOn loop 77
168642024-02-03T17:16:49.268Z
168652024-02-03T17:16:49.275ZVerifying data now
168662024-02-03T17:16:55.691ZNo change after verify
168672024-02-03T17:16:55.691ZLoop: 77 Downstairs dump after verify (and repair):
168682024-02-03T17:16:55.698ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
168692024-02-03T17:16:55.698Z 0 000-019 154 154 154 269 269 269 F F F
168702024-02-03T17:16:55.698Z 1 020-039 154 154 154 269 269 269 F F F
168712024-02-03T17:16:55.698Z 2 040-059 146 146 146 253 253 253 F F F
168722024-02-03T17:16:55.698Z 3 060-079 154 154 154 269 269 269 F F F
168732024-02-03T17:16:55.698Z 4 080-099 151 151 151 265 265 265 F F F
168742024-02-03T17:16:55.698Z 5 100-119 154 154 154 269 269 269 F F F
168752024-02-03T17:16:55.698Z 6 120-139 154 154 154 269 269 269 F F F
168762024-02-03T17:16:55.698Z 7 140-159 154 154 154 269 269 269 F F F
168772024-02-03T17:16:55.698Z 8 160-179 153 153 153 268 268 268 F F F
168782024-02-03T17:16:55.698Z 9 180-199 153 153 153 268 268 268 F F F
168792024-02-03T17:16:55.698Z 10 200-219 154 154 154 269 269 269 F F F
168802024-02-03T17:16:55.698Z 11 220-239 154 154 154 269 269 269 F F F
168812024-02-03T17:16:55.698Z 12 240-259 154 154 154 269 269 269 F F F
168822024-02-03T17:16:55.698Z 13 260-279 150 150 150 264 264 264 F F F
168832024-02-03T17:16:55.698Z 14 280-299 152 152 152 267 267 267 F F F
168842024-02-03T17:16:55.699Z 15 300-319 153 153 153 268 268 268 F F F
168852024-02-03T17:16:55.699Z 16 320-339 154 154 154 269 269 269 F F F
168862024-02-03T17:16:55.699Z 17 340-359 151 151 151 265 265 265 F F F
168872024-02-03T17:16:55.699Z 18 360-379 151 151 151 265 265 265 F F F
168882024-02-03T17:16:55.699Z 19 380-399 153 153 153 268 268 268 F F F
168892024-02-03T17:16:55.699Z 20 400-419 146 146 146 253 253 253 F F F
168902024-02-03T17:16:55.699Z 21 420-439 153 153 153 268 268 268 F F F
168912024-02-03T17:16:55.699Z 22 440-459 153 153 153 268 268 268 F F F
168922024-02-03T17:16:55.699Z 23 460-479 151 151 151 265 265 265 F F F
168932024-02-03T17:16:55.699Z 24 480-499 153 153 153 268 268 268 F F F
168942024-02-03T17:16:55.699Z 25 500-519 148 148 148 254 254 254 F F F
168952024-02-03T17:16:55.699Z 26 520-539 148 148 148 256 256 256 F F F
168962024-02-03T17:16:55.699Z 27 540-559 153 153 153 268 268 268 F F F
168972024-02-03T17:16:55.699Z 28 560-579 149 149 149 257 257 257 F F F
168982024-02-03T17:16:55.699Z 29 580-599 154 154 154 269 269 269 F F F
168992024-02-03T17:16:55.699ZMax gen: 154, Max flush: 269
169002024-02-03T17:16:55.699Z
169012024-02-03T17:16:55.699ZBegin loop 78
169022024-02-03T17:16:55.699ZDownstairs to restart: 2
169032024-02-03T17:16:55.699Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.918946978Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759}
169042024-02-03T17:16:55.699Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.91949431Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759}
169052024-02-03T17:16:55.699Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.919524552Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759}
169062024-02-03T17:16:55.700Z{"msg":"Crucible 02fd3296-cd05-46e6-b67b-addd8e65a602 has session id: c0270aec-28f4-4f12-bf63-3da46e241730","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.919595104Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169072024-02-03T17:16:55.700Z{"msg":"Crucible runtime is spawned
169082024-02-03T17:16:55.700ZUpstairs opts: Upstairs UUID: 02fd3296-cd05-46e6-b67b-addd8e65a602, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.91968627Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169092024-02-03T17:16:55.700Z{"msg":"Crucible stats registered with UUID: 02fd3296-cd05-46e6-b67b-addd8e65a602","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.919800157Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169102024-02-03T17:16:55.700Z{"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.919842229Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759}
169112024-02-03T17:16:55.700Z{"msg":"02fd3296-cd05-46e6-b67b-addd8e65a602 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.920104253Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169122024-02-03T17:16:55.700Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.920168641Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169132024-02-03T17:16:55.700Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.920194477Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169142024-02-03T17:16:55.700Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.920290563Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169152024-02-03T17:16:55.700Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.920458995Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169162024-02-03T17:16:55.700Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.92049933Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169172024-02-03T17:16:55.700Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.920524379Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169182024-02-03T17:16:55.700Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.920558895Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"":"io task","client":"2","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169192024-02-03T17:16:55.700Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.920676797Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"":"io task","client":"0","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169202024-02-03T17:16:55.700Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.920793613Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"":"io task","client":"1","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169212024-02-03T17:16:55.700Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.920930771Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"":"io task","client":"1","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169222024-02-03T17:16:55.700Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.920960396Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"":"io task","client":"2","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169232024-02-03T17:16:55.700Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.920984334Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"":"io task","client":"0","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169242024-02-03T17:16:55.700Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.922631922Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169252024-02-03T17:16:55.700Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.922724945Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169262024-02-03T17:16:55.700Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.92286734Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169272024-02-03T17:16:55.700Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.923026275Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169282024-02-03T17:16:55.700Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.923538422Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169292024-02-03T17:16:55.700Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.923645616Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169302024-02-03T17:16:55.701Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.92392948Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169312024-02-03T17:16:55.701Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.924000898Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169322024-02-03T17:16:55.701Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.924761222Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169332024-02-03T17:16:55.701Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.924975944Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169342024-02-03T17:16:55.701Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.925105077Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169352024-02-03T17:16:55.701Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.925131709Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169362024-02-03T17:16:55.701Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.925152821Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169372024-02-03T17:16:55.701Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.925175106Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169382024-02-03T17:16:55.701Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.92522235Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169392024-02-03T17:16:55.701Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.925561582Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169402024-02-03T17:16:55.701Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.92566497Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169412024-02-03T17:16:55.701Z{"msg":"[0]R flush_numbers[0..12]: [269, 269, 253, 269, 265, 269, 269, 269, 268, 268, 269, 269]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.92569233Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169422024-02-03T17:16:55.701Z{"msg":"[0]R generation[0..12]: [154, 154, 146, 154, 151, 154, 154, 154, 153, 153, 154, 154]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.925716337Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169432024-02-03T17:16:55.701Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.925765887Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169442024-02-03T17:16:55.701Z{"msg":"[1]R flush_numbers[0..12]: [269, 269, 253, 269, 265, 269, 269, 269, 268, 268, 269, 269]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.925792296Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169452024-02-03T17:16:55.701Z{"msg":"[1]R generation[0..12]: [154, 154, 146, 154, 151, 154, 154, 154, 153, 153, 154, 154]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.925840256Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169462024-02-03T17:16:55.701Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.925866262Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169472024-02-03T17:16:55.701ZWait for a query_work_queue command to finish before sending IO
169482024-02-03T17:16:55.701Z{"msg":"[2]R flush_numbers[0..12]: [269, 269, 253, 269, 265, 269, 269, 269, 268, 268, 269, 269]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.925919668Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169492024-02-03T17:16:55.701Z{"msg":"[2]R generation[0..12]: [154, 154, 146, 154, 151, 154, 154, 154, 153, 153, 154, 154]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.925945436Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169502024-02-03T17:16:55.701Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.925994023Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169512024-02-03T17:16:55.701Z{"msg":"Max found gen is 155","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.926018351Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169522024-02-03T17:16:55.701Z{"msg":"Generation requested: 156 >= found:155","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.926067569Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169532024-02-03T17:16:55.701Z{"msg":"Next flush: 270","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.926091588Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169542024-02-03T17:16:55.702Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.926139961Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169552024-02-03T17:16:55.702Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.92616447Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169562024-02-03T17:16:55.702Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.926210554Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169572024-02-03T17:16:55.702Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.926234913Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169582024-02-03T17:16:55.702Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.926258307Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169592024-02-03T17:16:55.702Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.926305067Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169602024-02-03T17:16:55.702Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.926327759Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169612024-02-03T17:16:55.702Z{"msg":"02fd3296-cd05-46e6-b67b-addd8e65a602 is now active with session: c0270aec-28f4-4f12-bf63-3da46e241730","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.926350836Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169622024-02-03T17:16:55.702Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.926397786Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759,"session_id":"c0270aec-28f4-4f12-bf63-3da46e241730"}
169632024-02-03T17:16:55.702Z{"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:16:54.926420464Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759}
169642024-02-03T17:16:55.702ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
169652024-02-03T17:16:55.702ZLoading write count information from file "/tmp/test_repair_verify.data"
169662024-02-03T17:16:55.702ZRun Repair workload
169672024-02-03T17:16:55.702Z01/30 Write block 450 len 8192 data: 8 6
169682024-02-03T17:16:55.707Z02/30 Read block 552 len 24576
169692024-02-03T17:16:55.707Z03/30 Read block 461 len 36864
169702024-02-03T17:16:55.707Z04/30 Read block 377 len 36864
169712024-02-03T17:16:55.707Z05/30 Write block 307 len 16384 data: 10 10 12 14
169722024-02-03T17:16:55.708Z06/30 Write block 78 len 20480 data: 11 12 10 13 12
169732024-02-03T17:16:55.708Z07/30 Read block 127 len 32768
169742024-02-03T17:16:55.708Z08/30 Flush
169752024-02-03T17:16:55.708Z09/30 Write block 501 len 16384 data: 7 11 14 13
169762024-02-03T17:16:55.708Z10/30 Read block 408 len 32768
169772024-02-03T17:16:55.708Z11/30 Write block 118 len 32768 data: 14 14 12 11 12 11 9 7
169782024-02-03T17:16:55.708Z12/30 Read block 511 len 12288
169792024-02-03T17:16:55.708Z13/30 Write block 595 len 16384 data: 9 10 7 5
169802024-02-03T17:16:55.708Z14/30 Write block 294 len 24576 data: 13 12 13 12 12 11
169812024-02-03T17:16:55.708Z15/30 Write block 354 len 16384 data: 10 11 9 10
169822024-02-03T17:16:55.708Z16/30 Read block 386 len 20480
169832024-02-03T17:16:55.708Z17/30 Flush
169842024-02-03T17:16:55.708Z18/30 Flush
169852024-02-03T17:16:55.708Z19/30 Flush
169862024-02-03T17:16:55.708Z20/30 Write block 405 len 16384 data: 10 13 14 12
169872024-02-03T17:16:55.708Z21/30 Read block 356 len 24576
169882024-02-03T17:16:55.708Z22/30 Write block 358 len 32768 data: 9 12 11 11 9 9 10 10
169892024-02-03T17:16:55.708Z23/30 Write block 159 len 24576 data: 11 16 16 18 18 18
169902024-02-03T17:16:55.708Z24/30 Write block 512 len 20480 data: 16 13 13 13 14
169912024-02-03T17:16:55.708Z25/30 Write block 300 len 16384 data: 9 10 10 10
169922024-02-03T17:16:55.708Z26/30 Read block 82 len 40960
169932024-02-03T17:16:55.708Z27/30 Read block 294 len 32768
169942024-02-03T17:16:55.708Z28/30 Read block 356 len 4096
169952024-02-03T17:16:55.708Z29/30 Read block 197 len 20480
169962024-02-03T17:16:55.709Z30/30 Write block 488 len 12288 data: 9 9 7
169972024-02-03T17:16:55.709Z----------------------------------------------------------------
169982024-02-03T17:16:55.709Z Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:22
169992024-02-03T17:16:55.709ZStates: Active Active Active
170002024-02-03T17:16:55.709ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
170012024-02-03T17:16:55.709Z 9 Acked 1008 Write 4 Done Done Done false
170022024-02-03T17:16:55.709Z 10 Acked 1009 Read 8 Done Done Done false
170032024-02-03T17:16:55.709Z 11 Acked 1010 Write 8 Done Done Sent false
170042024-02-03T17:16:55.709Z 12 Acked 1011 Read 3 Done Done Sent false
170052024-02-03T17:16:55.709Z 13 Acked 1012 Write 4 Done Done Sent false
170062024-02-03T17:16:55.709Z 14 Acked 1013 Write 6 Done Done Sent false
170072024-02-03T17:16:55.709Z 15 Acked 1014 Write 4 Done Done Sent false
170082024-02-03T17:16:55.709Z 16 Acked 1015 Read 5 Done Done Sent false
170092024-02-03T17:16:55.709Z 17 Acked 1016 Flush 0 Done Done Sent false
170102024-02-03T17:16:55.709Z 18 Acked 1017 Flush 0 Done Done Sent false
170112024-02-03T17:16:55.709Z 19 Acked 1018 Flush 0 Done Done Sent false
170122024-02-03T17:16:55.709Z 20 Acked 1019 Write 4 Done Done Sent false
170132024-02-03T17:16:55.709Z 21 Acked 1020 Read 6 Done Done Sent false
170142024-02-03T17:16:55.709Z 22 Acked 1021 Write 8 Done Done Sent false
170152024-02-03T17:16:55.709Z 23 Acked 1022 Write 6 Done Done Sent false
170162024-02-03T17:16:55.709Z 24 Acked 1023 Write 5 Done Done Sent false
170172024-02-03T17:16:55.709Z 25 Acked 1024 Write 4 Done Done Sent false
170182024-02-03T17:16:55.709Z 26 Acked 1025 Read 10 Done Done Sent false
170192024-02-03T17:16:55.709Z 27 Acked 1026 Read 8 Done Done Sent false
170202024-02-03T17:16:55.710Z 28 Acked 1027 Read 1 Done Done Sent false
170212024-02-03T17:16:55.710Z 29 Acked 1028 Read 5 Done Done Sent false
170222024-02-03T17:16:55.710Z 30 Acked 1029 Write 3 Sent Sent Sent false
170232024-02-03T17:16:55.710Z STATES DS:0 DS:1 DS:2 TOTAL
170242024-02-03T17:16:55.710Z New 0 0 0 0
170252024-02-03T17:16:55.710Z Sent 1 1 20 22
170262024-02-03T17:16:55.710Z Done 21 21 2 44
170272024-02-03T17:16:55.710Z Skipped 0 0 0 0
170282024-02-03T17:16:55.710Z Error 0 0 0 0
170292024-02-03T17:16:55.710ZLast Flush: 1018 1018 1007
170302024-02-03T17:16:55.710ZDownstairs last five completed: 1007 1006 1005 1004 1003
170312024-02-03T17:16:55.710ZUpstairs last five completed: 30 29 28 27 26
170322024-02-03T17:16:55.710Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:16:54.976689709Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1759}
170332024-02-03T17:16:55.710ZWrote out file "/tmp/test_repair_verify.data"
170342024-02-03T17:16:55.710Z
170352024-02-03T17:16:55.710ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
170362024-02-03T17:16:55.710ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
170372024-02-03T17:16:55.711Z 0 000-019 154 154 154 269 269 269 F F F
170382024-02-03T17:16:55.711Z 1 020-039 154 154 154 269 269 269 F F F
170392024-02-03T17:16:55.711Z 2 040-059 146 146 146 253 253 253 F F F
170402024-02-03T17:16:55.711Z 3 060-079 156 156 156 270 270 270 F F F
170412024-02-03T17:16:55.711Z 4 080-099 156 156 156 270 270 270 F F F
170422024-02-03T17:16:55.711Z 5 100-119 156 156 154 271 271 269 F F F <---
170432024-02-03T17:16:55.711Z 6 120-139 156 156 154 271 271 269 F F F <---
170442024-02-03T17:16:55.711Z 7 140-159 154 154 154 269 269 269 T T F <---
170452024-02-03T17:16:55.711Z 8 160-179 153 153 153 268 268 268 T T F <---
170462024-02-03T17:16:55.711Z 9 180-199 153 153 153 268 268 268 F F F
170472024-02-03T17:16:55.711Z 10 200-219 154 154 154 269 269 269 F F F
170482024-02-03T17:16:55.711Z 11 220-239 154 154 154 269 269 269 F F F
170492024-02-03T17:16:55.711Z 12 240-259 154 154 154 269 269 269 F F F
170502024-02-03T17:16:55.712Z 13 260-279 150 150 150 264 264 264 F F F
170512024-02-03T17:16:55.712Z 14 280-299 156 156 152 271 271 267 F F F <---
170522024-02-03T17:16:55.712Z 15 300-319 156 156 156 270 270 270 T T F <---
170532024-02-03T17:16:55.712Z 16 320-339 154 154 154 269 269 269 F F F
170542024-02-03T17:16:55.712Z 17 340-359 156 156 151 271 271 265 T T F <---
170552024-02-03T17:16:55.712Z 18 360-379 151 151 151 265 265 265 T T F <---
170562024-02-03T17:16:55.712Z 19 380-399 153 153 153 268 268 268 F F F
170572024-02-03T17:16:55.713Z 20 400-419 146 146 146 253 253 253 T T F <---
170582024-02-03T17:16:55.713Z 21 420-439 153 153 153 268 268 268 F F F
170592024-02-03T17:16:55.713Z 22 440-459 156 156 156 270 270 270 F F F
170602024-02-03T17:16:55.713Z 23 460-479 151 151 151 265 265 265 F F F
170612024-02-03T17:16:55.713Z 24 480-499 153 153 153 268 268 268 T T F <---
170622024-02-03T17:16:55.713Z 25 500-519 156 156 148 271 271 254 T T T <---
170632024-02-03T17:16:55.713Z 26 520-539 148 148 148 256 256 256 F F F
170642024-02-03T17:16:55.713Z 27 540-559 153 153 153 268 268 268 F F F
170652024-02-03T17:16:55.714Z 28 560-579 149 149 149 257 257 257 F F F
170662024-02-03T17:16:55.714Z 29 580-599 156 156 154 271 271 269 F F F <---
170672024-02-03T17:16:55.714ZMax gen: 156, Max flush: 271
170682024-02-03T17:16:55.729ZError: Difference in extent metadata found!
170692024-02-03T17:16:55.730ZA Difference in extent metadata is expected here
170702024-02-03T17:16:55.730ZOn loop 78
170712024-02-03T17:16:55.730Z
170722024-02-03T17:16:55.730ZVerifying data now
170732024-02-03T17:17:00.541ZNo change after verify
170742024-02-03T17:17:00.542ZLoop: 78 Downstairs dump after verify (and repair):
170752024-02-03T17:17:00.553ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
170762024-02-03T17:17:00.553Z 0 000-019 154 154 154 269 269 269 F F F
170772024-02-03T17:17:00.553Z 1 020-039 154 154 154 269 269 269 F F F
170782024-02-03T17:17:00.553Z 2 040-059 146 146 146 253 253 253 F F F
170792024-02-03T17:17:00.553Z 3 060-079 156 156 156 270 270 270 F F F
170802024-02-03T17:17:00.559Z 4 080-099 156 156 156 270 270 270 F F F
170812024-02-03T17:17:00.559Z 5 100-119 156 156 156 271 271 271 F F F
170822024-02-03T17:17:00.559Z 6 120-139 156 156 156 271 271 271 F F F
170832024-02-03T17:17:00.559Z 7 140-159 157 157 157 272 272 272 F F F
170842024-02-03T17:17:00.559Z 8 160-179 157 157 157 272 272 272 F F F
170852024-02-03T17:17:00.559Z 9 180-199 153 153 153 268 268 268 F F F
170862024-02-03T17:17:00.560Z 10 200-219 154 154 154 269 269 269 F F F
170872024-02-03T17:17:00.560Z 11 220-239 154 154 154 269 269 269 F F F
170882024-02-03T17:17:00.560Z 12 240-259 154 154 154 269 269 269 F F F
170892024-02-03T17:17:00.560Z 13 260-279 150 150 150 264 264 264 F F F
170902024-02-03T17:17:00.560Z 14 280-299 156 156 156 271 271 271 F F F
170912024-02-03T17:17:00.561Z 15 300-319 157 157 157 272 272 272 F F F
170922024-02-03T17:17:00.561Z 16 320-339 154 154 154 269 269 269 F F F
170932024-02-03T17:17:00.561Z 17 340-359 157 157 157 272 272 272 F F F
170942024-02-03T17:17:00.561Z 18 360-379 157 157 157 272 272 272 F F F
170952024-02-03T17:17:00.561Z 19 380-399 153 153 153 268 268 268 F F F
170962024-02-03T17:17:00.561Z 20 400-419 157 157 157 272 272 272 F F F
170972024-02-03T17:17:00.561Z 21 420-439 153 153 153 268 268 268 F F F
170982024-02-03T17:17:00.562Z 22 440-459 156 156 156 270 270 270 F F F
170992024-02-03T17:17:00.562Z 23 460-479 151 151 151 265 265 265 F F F
171002024-02-03T17:17:00.562Z 24 480-499 157 157 157 272 272 272 F F F
171012024-02-03T17:17:00.562Z 25 500-519 157 157 157 272 272 272 F F F
171022024-02-03T17:17:00.562Z 26 520-539 148 148 148 256 256 256 F F F
171032024-02-03T17:17:00.563Z 27 540-559 153 153 153 268 268 268 F F F
171042024-02-03T17:17:00.563Z 28 560-579 149 149 149 257 257 257 F F F
171052024-02-03T17:17:00.563Z 29 580-599 156 156 156 271 271 271 F F F
171062024-02-03T17:17:00.563ZMax gen: 157, Max flush: 272
171072024-02-03T17:17:00.563Z
171082024-02-03T17:17:00.563ZBegin loop 79
171092024-02-03T17:17:00.563ZDownstairs to restart: 1
171102024-02-03T17:17:00.589Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.160908417Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767}
171112024-02-03T17:17:00.589Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.161423178Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767}
171122024-02-03T17:17:00.602Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
171132024-02-03T17:17:00.602Z,"time":"2024-02-03T17:17:00.161522889Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767}
171142024-02-03T17:17:00.602Z{"msg":"Crucible 7cff6bcd-5305-40b6-a589-03770bcd4053 has session id: 66e52c0b-97c7-4076-abfb-917947eb397c","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.161791332Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171152024-02-03T17:17:00.603Z{"msg":"Upstairs opts: Upstairs UUID: 7cff6bcd-5305-40b6-a589-03770bcd4053, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.161837923Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171162024-02-03T17:17:00.603Z{"msg":"Crucible stats registered with UUID: 7cff6bcd-5305-40b6-a589-03770bcd4053","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.161892086Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171172024-02-03T17:17:00.603Z{"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.161916696Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767}
171182024-02-03T17:17:00.603Z{"msg":"7cff6bcd-5305-40b6-a589-03770bcd4053 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.161937864Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171192024-02-03T17:17:00.603Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.161984159Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171202024-02-03T17:17:00.603Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.162010531Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171212024-02-03T17:17:00.604Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.162058253Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171222024-02-03T17:17:00.604Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.162091147Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171232024-02-03T17:17:00.604Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.162172309Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171242024-02-03T17:17:00.604Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.162220808Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171252024-02-03T17:17:00.604Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.162354203Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"":"io task","client":"2","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171262024-02-03T17:17:00.604Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.162493545Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"":"io task","client":"1","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171272024-02-03T17:17:00.604Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.162578164Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"":"io task","client":"0","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171282024-02-03T17:17:00.605Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.162649956Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"":"io task","client":"0","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171292024-02-03T17:17:00.605Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.162696163Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"":"io task","client":"2","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171302024-02-03T17:17:00.605Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.162939273Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"":"io task","client":"1","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171312024-02-03T17:17:00.605Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.16401438Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171322024-02-03T17:17:00.605Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.164089515Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171332024-02-03T17:17:00.605Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.164116973Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171342024-02-03T17:17:00.606Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.164223759Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171352024-02-03T17:17:00.606Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.165122042Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171362024-02-03T17:17:00.606Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.165223391Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171372024-02-03T17:17:00.606Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.16577059Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171382024-02-03T17:17:00.606Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.165864728Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171392024-02-03T17:17:00.606Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.166080202Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171402024-02-03T17:17:00.606Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.166159059Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171412024-02-03T17:17:00.606Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.166187199Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171422024-02-03T17:17:00.606Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.166208371Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171432024-02-03T17:17:00.607Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.166288978Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171442024-02-03T17:17:00.607Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.16633212Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171452024-02-03T17:17:00.607Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.166792102Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171462024-02-03T17:17:00.607Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.16758209Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171472024-02-03T17:17:00.607Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.167643707Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171482024-02-03T17:17:00.607Z{"msg":"[0]R flush_numbers[0..12]: [269, 269, 253, 270, 270, 271, 271, 272, 272, 268, 269, 269]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.167670746Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171492024-02-03T17:17:00.607Z{"msg":"[0]R generation[0..12]: [154, 154, 146, 156, 156, 156, 156, 157, 157, 153, 154, 154]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.167736592Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171502024-02-03T17:17:00.607Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.167771106Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171512024-02-03T17:17:00.607Z{"msg":"[1]R flush_numbers[0..12]: [269, 269, 253, 270, 270, 271, 271, 272, 272, 268, 269, 269]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.167835426Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171522024-02-03T17:17:00.608Z{"msg":"[1]R generation[0..12]: [154, 154, 146, 156, 156, 156, 156, 157, 157, 153, 154, 154]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.16786572Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171532024-02-03T17:17:00.608Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
171542024-02-03T17:17:00.608Z,"time":"2024-02-03T17:17:00.167930111Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171552024-02-03T17:17:00.608Z{"msg":"[2]R flush_numbers[0..12]: [269, 269, 253, 270, 270, 271, 271, 272, 272, 268, 269, 269]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.167963146Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171562024-02-03T17:17:00.608Z{"msg":"[2]R generation[0..12]: [154, 154, 146, 156, 156, 156, 156, 157, 157, 153, 154, 154]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.168042992Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171572024-02-03T17:17:00.608Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.168072548Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171582024-02-03T17:17:00.608Z{"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.168143367Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171592024-02-03T17:17:00.608Z{"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.16817168Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171602024-02-03T17:17:00.608Z{"msg":"Next flush: 273","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.168193314Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171612024-02-03T17:17:00.608Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.168229212Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171622024-02-03T17:17:00.609Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.168258674Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171632024-02-03T17:17:00.609Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.168339646Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171642024-02-03T17:17:00.609Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.16836742Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171652024-02-03T17:17:00.609Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.168432437Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171662024-02-03T17:17:00.609Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.168459865Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171672024-02-03T17:17:00.609Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.168497662Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171682024-02-03T17:17:00.609Z{"msg":"7cff6bcd-5305-40b6-a589-03770bcd4053 is now active with session: 66e52c0b-97c7-4076-abfb-917947eb397c","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.168530676Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171692024-02-03T17:17:00.610Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.168589092Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767,"session_id":"66e52c0b-97c7-4076-abfb-917947eb397c"}
171702024-02-03T17:17:00.610Z{"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:00.168613326Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767}
171712024-02-03T17:17:00.610ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
171722024-02-03T17:17:00.610ZLoading write count information from file "/tmp/test_repair_verify.data"
171732024-02-03T17:17:00.610ZRun Repair workload
171742024-02-03T17:17:00.620Z01/30 Write block 50 len 40960 data: 14 14 15 12 11 12 13 11 9 10
171752024-02-03T17:17:00.620Z02/30 Write block 79 len 24576 data: 13 11 14 13 10 10
171762024-02-03T17:17:00.620Z03/30 Write block 325 len 28672 data: 13 12 17 16 15 14 14
171772024-02-03T17:17:00.620Z04/30 Read block 41 len 32768
171782024-02-03T17:17:00.620Z05/30 Read block 407 len 24576
171792024-02-03T17:17:00.621Z06/30 Flush
171802024-02-03T17:17:00.621Z07/30 Write block 380 len 8192 data: 15 12
171812024-02-03T17:17:00.621Z08/30 Write block 550 len 20480 data: 16 14 14 16 15
171822024-02-03T17:17:00.621Z09/30 Flush
171832024-02-03T17:17:00.621Z10/30 Write block 381 len 12288 data: 13 13 13
171842024-02-03T17:17:00.621Z11/30 Write block 19 len 24576 data: 22 18 21 17 16 15
171852024-02-03T17:17:00.621Z12/30 Read block 467 len 36864
171862024-02-03T17:17:00.629Z13/30 Read block 155 len 16384
171872024-02-03T17:17:00.629Z14/30 Read block 205 len 8192
171882024-02-03T17:17:00.629Z15/30 Write block 119 len 12288 data: 15 13 12
171892024-02-03T17:17:00.629Z16/30 Write block 352 len 28672 data: 10 10 11 12 10 11 10
171902024-02-03T17:17:00.630Z17/30 Write block 146 len 40960 data: 9 6 11 12 11 13 14 12 11 12
171912024-02-03T17:17:00.630Z18/30 Write block 304 len 24576 data: 11 11 12 11 11 13
171922024-02-03T17:17:00.630Z19/30 Write block 12 len 20480 data: 12 13 15 12 11
171932024-02-03T17:17:00.630Z20/30 Read block 64 len 12288
171942024-02-03T17:17:00.639Z21/30 Read block 314 len 4096
171952024-02-03T17:17:00.640Z22/30 Write block 498 len 4096 data: 9
171962024-02-03T17:17:00.640Z23/30 Flush
171972024-02-03T17:17:00.650Z24/30 Write block 428 len 40960 data: 13 16 16 13 15 13 11 15 13 14
171982024-02-03T17:17:00.650Z25/30 Write block 31 len 8192 data: 16 14
171992024-02-03T17:17:00.650Z26/30 Write block 59 len 28672 data: 11 10 9 11 10 8 8
172002024-02-03T17:17:00.650Z27/30 Write block 227 len 36864 data: 7 11 8 8 10 11 11 13 12
172012024-02-03T17:17:00.651Z28/30 Write block 476 len 20480 data: 7 8 10 12 12
172022024-02-03T17:17:00.651Z29/30 Read block 63 len 12288
172032024-02-03T17:17:00.651Z30/30 Read block 526 len 20480
172042024-02-03T17:17:00.651Z----------------------------------------------------------------
172052024-02-03T17:17:00.651Z Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30
172062024-02-03T17:17:00.651ZStates: Active Active Active
172072024-02-03T17:17:00.651ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
172082024-02-03T17:17:00.651Z 1 Acked 1000 Write 10 Done Sent Done false
172092024-02-03T17:17:00.652Z 2 Acked 1001 Write 6 Done Sent Done false
172102024-02-03T17:17:00.652Z 3 Acked 1002 Write 7 Done Sent Done false
172112024-02-03T17:17:00.652Z 4 Acked 1003 Read 8 Done Sent Done false
172122024-02-03T17:17:00.652Z 5 Acked 1004 Read 6 Done Sent Done false
172132024-02-03T17:17:00.652Z 6 Acked 1005 Flush 0 Done Sent Done false
172142024-02-03T17:17:00.652Z 7 Acked 1006 Write 2 Done Sent Done false
172152024-02-03T17:17:00.652Z 8 Acked 1007 Write 5 Done Sent Done false
172162024-02-03T17:17:00.653Z 9 Acked 1008 Flush 0 Done Sent Done false
172172024-02-03T17:17:00.653Z 10 Acked 1009 Write 3 Done Sent Done false
172182024-02-03T17:17:00.653Z 11 Acked 1010 Write 6 Done Sent Done false
172192024-02-03T17:17:00.653Z 12 Acked 1011 Read 9 Done Sent Done false
172202024-02-03T17:17:00.653Z 13 Acked 1012 Read 4 Done Sent Done false
172212024-02-03T17:17:00.653Z 14 Acked 1013 Read 2 Done Sent Done false
172222024-02-03T17:17:00.653Z 15 Acked 1014 Write 3 Done Sent Done false
172232024-02-03T17:17:00.654Z 16 Acked 1015 Write 7 Done Sent Done false
172242024-02-03T17:17:00.654Z 17 Acked 1016 Write 10 Done Sent Done false
172252024-02-03T17:17:00.654Z 18 Acked 1017 Write 6 Done Sent Done false
172262024-02-03T17:17:00.654Z 19 Acked 1018 Write 5 Done Sent Done false
172272024-02-03T17:17:00.654Z 20 Acked 1019 Read 3 Done Sent Done false
172282024-02-03T17:17:00.654Z 21 Acked 1020 Read 1 Done Sent Done false
172292024-02-03T17:17:00.654Z 22 Acked 1021 Write 1 Done Sent Done false
172302024-02-03T17:17:00.654Z 23 Acked 1022 Flush 0 Done Sent Done false
172312024-02-03T17:17:00.654Z 24 Acked 1023 Write 10 Done Sent Done false
172322024-02-03T17:17:00.654Z 25 Acked 1024 Write 2 Done Sent Done false
172332024-02-03T17:17:00.654Z 26 Acked 1025 Write 7 Done Sent Done false
172342024-02-03T17:17:00.655Z 27 Acked 1026 Write 9 Done Sent Done false
172352024-02-03T17:17:00.655Z 28 Acked 1027 Write 5 Done Sent Done false
172362024-02-03T17:17:00.655Z 29 Acked 1028 Read 3 Done Sent Done false
172372024-02-03T17:17:00.655Z 30 Acked 1029 Read 5 Done Sent Done false
172382024-02-03T17:17:00.655Z STATES DS:0 DS:1 DS:2 TOTAL
172392024-02-03T17:17:00.655Z New 0 0 0 0
172402024-02-03T17:17:00.655Z Sent 0 30 0 30
172412024-02-03T17:17:00.655Z Done 30 0 30 60
172422024-02-03T17:17:00.655Z Skipped 0 0 0 0
172432024-02-03T17:17:00.656Z Error 0 0 0 0
172442024-02-03T17:17:00.656ZLast Flush: 1022 0 1022
172452024-02-03T17:17:00.656ZDownstairs last five completed:
172462024-02-03T17:17:00.656ZUpstairs last five completed: 30 29 28 27 26
172472024-02-03T17:17:00.656Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:17:00.218533671Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1767}
172482024-02-03T17:17:00.656ZWrote out file "/tmp/test_repair_verify.data"
172492024-02-03T17:17:00.671Z
172502024-02-03T17:17:00.671ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
172512024-02-03T17:17:00.687ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
172522024-02-03T17:17:00.687Z 0 000-019 158 154 158 275 269 275 F F F <---
172532024-02-03T17:17:00.687Z 1 020-039 158 154 158 275 269 275 T F T <---
172542024-02-03T17:17:00.687Z 2 040-059 158 146 158 273 253 273 T F T <---
172552024-02-03T17:17:00.688Z 3 060-079 158 156 158 273 270 273 T F T <---
172562024-02-03T17:17:00.702Z 4 080-099 158 156 158 273 270 273 F F F <---
172572024-02-03T17:17:00.702Z 5 100-119 158 156 158 275 271 275 F F F <---
172582024-02-03T17:17:00.702Z 6 120-139 158 156 158 275 271 275 F F F <---
172592024-02-03T17:17:00.703Z 7 140-159 158 157 158 275 272 275 F F F <---
172602024-02-03T17:17:00.703Z 8 160-179 157 157 157 272 272 272 F F F
172612024-02-03T17:17:00.703Z 9 180-199 153 153 153 268 268 268 F F F
172622024-02-03T17:17:00.703Z 10 200-219 154 154 154 269 269 269 F F F
172632024-02-03T17:17:00.703Z 11 220-239 154 154 154 269 269 269 T F T <---
172642024-02-03T17:17:00.703Z 12 240-259 154 154 154 269 269 269 F F F
172652024-02-03T17:17:00.703Z 13 260-279 150 150 150 264 264 264 F F F
172662024-02-03T17:17:00.703Z 14 280-299 156 156 156 271 271 271 F F F
172672024-02-03T17:17:00.704Z 15 300-319 158 157 158 275 272 275 F F F <---
172682024-02-03T17:17:00.704Z 16 320-339 158 154 158 273 269 273 F F F <---
172692024-02-03T17:17:00.704Z 17 340-359 158 157 158 275 272 275 F F F <---
172702024-02-03T17:17:00.704Z 18 360-379 157 157 157 272 272 272 F F F
172712024-02-03T17:17:00.704Z 19 380-399 158 153 158 275 268 275 F F F <---
172722024-02-03T17:17:00.704Z 20 400-419 157 157 157 272 272 272 F F F
172732024-02-03T17:17:00.704Z 21 420-439 153 153 153 268 268 268 T F T <---
172742024-02-03T17:17:00.704Z 22 440-459 156 156 156 270 270 270 F F F
172752024-02-03T17:17:00.704Z 23 460-479 151 151 151 265 265 265 T F T <---
172762024-02-03T17:17:00.705Z 24 480-499 158 157 158 275 272 275 T F T <---
172772024-02-03T17:17:00.705Z 25 500-519 157 157 157 272 272 272 F F F
172782024-02-03T17:17:00.705Z 26 520-539 148 148 148 256 256 256 F F F
172792024-02-03T17:17:00.705Z 27 540-559 158 153 158 274 268 274 F F F <---
172802024-02-03T17:17:00.705Z 28 560-579 149 149 149 257 257 257 F F F
172812024-02-03T17:17:00.705Z 29 580-599 156 156 156 271 271 271 F F F
172822024-02-03T17:17:00.705ZMax gen: 158, Max flush: 275
172832024-02-03T17:17:00.705ZError: Difference in extent metadata found!
172842024-02-03T17:17:00.705ZA Difference in extent metadata is expected here
172852024-02-03T17:17:00.705ZOn loop 79
172862024-02-03T17:17:00.706Z
172872024-02-03T17:17:01.128ZVerifying data now
172882024-02-03T17:17:06.149ZNo change after verify
172892024-02-03T17:17:06.150ZLoop: 79 Downstairs dump after verify (and repair):
172902024-02-03T17:17:06.190ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
172912024-02-03T17:17:06.190Z 0 000-019 158 158 158 275 275 275 F F F
172922024-02-03T17:17:06.191Z 1 020-039 159 159 159 276 276 276 F F F
172932024-02-03T17:17:06.191Z 2 040-059 159 159 159 276 276 276 F F F
172942024-02-03T17:17:06.191Z 3 060-079 159 159 159 276 276 276 F F F
172952024-02-03T17:17:06.192Z 4 080-099 158 158 158 273 273 273 F F F
172962024-02-03T17:17:06.192Z 5 100-119 158 158 158 275 275 275 F F F
172972024-02-03T17:17:06.192Z 6 120-139 158 158 158 275 275 275 F F F
172982024-02-03T17:17:06.192Z 7 140-159 158 158 158 275 275 275 F F F
172992024-02-03T17:17:06.192Z 8 160-179 157 157 157 272 272 272 F F F
173002024-02-03T17:17:06.192Z 9 180-199 153 153 153 268 268 268 F F F
173012024-02-03T17:17:06.193Z 10 200-219 154 154 154 269 269 269 F F F
173022024-02-03T17:17:06.193Z 11 220-239 159 159 159 276 276 276 F F F
173032024-02-03T17:17:06.193Z 12 240-259 154 154 154 269 269 269 F F F
173042024-02-03T17:17:06.193Z 13 260-279 150 150 150 264 264 264 F F F
173052024-02-03T17:17:06.193Z 14 280-299 156 156 156 271 271 271 F F F
173062024-02-03T17:17:06.193Z 15 300-319 158 158 158 275 275 275 F F F
173072024-02-03T17:17:06.194Z 16 320-339 158 158 158 273 273 273 F F F
173082024-02-03T17:17:06.194Z 17 340-359 158 158 158 275 275 275 F F F
173092024-02-03T17:17:06.195Z 18 360-379 157 157 157 272 272 272 F F F
173102024-02-03T17:17:06.195Z 19 380-399 158 158 158 275 275 275 F F F
173112024-02-03T17:17:06.195Z 20 400-419 157 157 157 272 272 272 F F F
173122024-02-03T17:17:06.195Z 21 420-439 159 159 159 276 276 276 F F F
173132024-02-03T17:17:06.196Z 22 440-459 156 156 156 270 270 270 F F F
173142024-02-03T17:17:06.196Z 23 460-479 159 159 159 276 276 276 F F F
173152024-02-03T17:17:06.196Z 24 480-499 159 159 159 276 276 276 F F F
173162024-02-03T17:17:06.196Z 25 500-519 157 157 157 272 272 272 F F F
173172024-02-03T17:17:06.197Z 26 520-539 148 148 148 256 256 256 F F F
173182024-02-03T17:17:06.198Z 27 540-559 158 158 158 274 274 274 F F F
173192024-02-03T17:17:06.198Z 28 560-579 149 149 149 257 257 257 F F F
173202024-02-03T17:17:06.198Z 29 580-599 156 156 156 271 271 271 F F F
173212024-02-03T17:17:06.198ZMax gen: 159, Max flush: 276
173222024-02-03T17:17:06.247Z
173232024-02-03T17:17:06.248ZBegin loop 80
173242024-02-03T17:17:06.248ZDownstairs to restart: 1
173252024-02-03T17:17:06.248Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.774073562Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775}
173262024-02-03T17:17:06.248Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.774667062Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775}
173272024-02-03T17:17:06.248Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
173282024-02-03T17:17:06.248Z,"time":"2024-02-03T17:17:05.774750377Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775}
173292024-02-03T17:17:06.248Z{"msg":"Crucible 9fd5ed2a-119e-4791-adc6-3f1aa31525eb has session id: 2b17ca54-fa70-47cf-8d61-5b05a88e028e","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.774988995Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173302024-02-03T17:17:06.249Z{"msg":"Upstairs opts: Upstairs UUID: 9fd5ed2a-119e-4791-adc6-3f1aa31525eb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.77503371Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173312024-02-03T17:17:06.249Z{"msg":"Crucible stats registered with UUID: 9fd5ed2a-119e-4791-adc6-3f1aa31525eb","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.775076711Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173322024-02-03T17:17:06.249Z{"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.775111829Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775}
173332024-02-03T17:17:06.249Z{"msg":"9fd5ed2a-119e-4791-adc6-3f1aa31525eb active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.775279408Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173342024-02-03T17:17:06.249Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.775341676Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173352024-02-03T17:17:06.249Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.775368912Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173362024-02-03T17:17:06.249Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.775391673Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173372024-02-03T17:17:06.249Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.775468678Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173382024-02-03T17:17:06.249Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.775515681Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173392024-02-03T17:17:06.249Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.775560468Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173402024-02-03T17:17:06.250Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.775605036Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"":"io task","client":"2","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173412024-02-03T17:17:06.250Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.775648055Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"":"io task","client":"0","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173422024-02-03T17:17:06.250Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.775752716Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"":"io task","client":"1","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173432024-02-03T17:17:06.250Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.775840874Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"":"io task","client":"2","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173442024-02-03T17:17:06.251Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.77604536Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"":"io task","client":"1","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173452024-02-03T17:17:06.251Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.776139389Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"":"io task","client":"0","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173462024-02-03T17:17:06.251Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.777452975Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173472024-02-03T17:17:06.252Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.777521083Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173482024-02-03T17:17:06.252Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.777547991Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173492024-02-03T17:17:06.252Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.777570191Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173502024-02-03T17:17:06.253Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.777794828Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173512024-02-03T17:17:06.255Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.777952217Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173522024-02-03T17:17:06.256Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.778707457Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173532024-02-03T17:17:06.256Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.77878171Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173542024-02-03T17:17:06.257Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.779145078Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173552024-02-03T17:17:06.257Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.779762333Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173562024-02-03T17:17:06.257Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.779813581Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173572024-02-03T17:17:06.258Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.779940964Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173582024-02-03T17:17:06.258Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.779998935Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173592024-02-03T17:17:06.258Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.780024603Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173602024-02-03T17:17:06.258Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.78004695Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173612024-02-03T17:17:06.258Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.780093207Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173622024-02-03T17:17:06.259Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.780118164Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173632024-02-03T17:17:06.259Z{"msg":"[0]R flush_numbers[0..12]: [275, 276, 276, 276, 273, 275, 275, 275, 272, 268, 269, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.780166771Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173642024-02-03T17:17:06.259Z{"msg":"[0]R generation[0..12]: [158, 159, 159, 159, 158, 158, 158, 158, 157, 153, 154, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.780192483Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173652024-02-03T17:17:06.260Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.780260719Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"Wait for a query_work_queue command to finish before sending IO
173662024-02-03T17:17:06.260Z":"downstairs","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173672024-02-03T17:17:06.260Z{"msg":"[1]R flush_numbers[0..12]: [275, 276, 276, 276, 273, 275, 275, 275, 272, 268, 269, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.78029059Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173682024-02-03T17:17:06.260Z{"msg":"[1]R generation[0..12]: [158, 159, 159, 159, 158, 158, 158, 158, 157, 153, 154, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.780314747Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173692024-02-03T17:17:06.260Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.78038715Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173702024-02-03T17:17:06.261Z{"msg":"[2]R flush_numbers[0..12]: [275, 276, 276, 276, 273, 275, 275, 275, 272, 268, 269, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.780414777Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173712024-02-03T17:17:06.261Z{"msg":"[2]R generation[0..12]: [158, 159, 159, 159, 158, 158, 158, 158, 157, 153, 154, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.780438069Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173722024-02-03T17:17:06.261Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.780486855Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173732024-02-03T17:17:06.261Z{"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.780511731Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173742024-02-03T17:17:06.262Z{"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.780535497Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173752024-02-03T17:17:06.262Z{"msg":"Next flush: 277","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.780556879Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173762024-02-03T17:17:06.263Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.780578257Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173772024-02-03T17:17:06.263Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.780600132Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173782024-02-03T17:17:06.263Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.780628194Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173792024-02-03T17:17:06.264Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.780694951Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173802024-02-03T17:17:06.264Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.780722944Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173812024-02-03T17:17:06.264Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.780753678Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173822024-02-03T17:17:06.264Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.780814729Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173832024-02-03T17:17:06.265Z{"msg":"9fd5ed2a-119e-4791-adc6-3f1aa31525eb is now active with session: 2b17ca54-fa70-47cf-8d61-5b05a88e028e","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.780841178Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173842024-02-03T17:17:06.265Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.780904915Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775,"session_id":"2b17ca54-fa70-47cf-8d61-5b05a88e028e"}
173852024-02-03T17:17:06.265Z{"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:05.780931823Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775}
173862024-02-03T17:17:06.266ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
173872024-02-03T17:17:06.266ZLoading write count information from file "/tmp/test_repair_verify.data"
173882024-02-03T17:17:06.266ZRun Repair workload
173892024-02-03T17:17:06.266Z01/30 Write block 571 len 24576 data: 11 10 10 10 9 9
173902024-02-03T17:17:06.267Z02/30 Write block 468 len 16384 data: 12 11 11 10
173912024-02-03T17:17:06.267Z03/30 Read block 85 len 28672
173922024-02-03T17:17:06.268Z04/30 Read block 54 len 36864
173932024-02-03T17:17:06.268Z05/30 Write block 232 len 40960 data: 12 12 14 13 11 10 9 9 6 8
173942024-02-03T17:17:06.269Z06/30 Read block 167 len 8192
173952024-02-03T17:17:06.269Z07/30 Read block 272 len 4096
173962024-02-03T17:17:06.269Z08/30 Write block 91 len 32768 data: 11 11 14 15 13 13 12 10
173972024-02-03T17:17:06.269Z09/30 Read block 92 len 32768
173982024-02-03T17:17:06.270Z10/30 Read block 578 len 28672
173992024-02-03T17:17:06.270Z11/30 Read block 46 len 32768
174002024-02-03T17:17:06.271Z12/30 Read block 16 len 32768
174012024-02-03T17:17:06.271Z13/30 Write block 15 len 40960 data: 13 12 15 17 23 19 22 18 17 16
174022024-02-03T17:17:06.271Z14/30 Read block 292 len 32768
174032024-02-03T17:17:06.271Z15/30 Read block 47 len 4096
174042024-02-03T17:17:06.271Z16/30 Write block 570 len 28672 data: 13 12 11 11 11 10 10
174052024-02-03T17:17:06.271Z17/30 Read block 385 len 20480
174062024-02-03T17:17:06.304Z18/30 Write block 239 len 24576 data: 10 7 9 8 7 8
174072024-02-03T17:17:06.304Z19/30 Read block 150 len 32768
174082024-02-03T17:17:06.304Z20/30 Flush
174092024-02-03T17:17:06.304Z21/30 Write block 595 len 8192 data: 10 11
174102024-02-03T17:17:06.304Z22/30 Write block 377 len 24576 data: 18 15 17 16 14 14
174112024-02-03T17:17:06.304Z23/30 Write block 477 len 24576 data: 9 11 13 13 11 12
174122024-02-03T17:17:06.304Z24/30 Read block 470 len 4096
174132024-02-03T17:17:06.304Z25/30 Write block 158 len 20480 data: 11 12 17 17 19
174142024-02-03T17:17:06.304Z26/30 Flush
174152024-02-03T17:17:06.305Z27/30 Write block 481 len 4096 data: 12
174162024-02-03T17:17:06.305Z28/30 Read block 60 len 8192
174172024-02-03T17:17:06.305Z29/30 Write block 434 len 40960 data: 12 16 14 15 15 16 17 17 19 16
174182024-02-03T17:17:06.305Z30/30 Read block 408 len 40960
174192024-02-03T17:17:06.305Z----------------------------------------------------------------
174202024-02-03T17:17:06.305Z Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30
174212024-02-03T17:17:06.305ZStates: Active Active Active
174222024-02-03T17:17:06.305ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
174232024-02-03T17:17:06.305Z 1 Acked 1000 Write 6 Done Done Done false
174242024-02-03T17:17:06.305Z 2 Acked 1001 Write 4 Done Sent Done false
174252024-02-03T17:17:06.305Z 3 Acked 1002 Read 7 Done Sent Done false
174262024-02-03T17:17:06.305Z 4 Acked 1003 Read 9 Done Sent Done false
174272024-02-03T17:17:06.305Z 5 Acked 1004 Write 10 Done Sent Done false
174282024-02-03T17:17:06.305Z 6 Acked 1005 Read 2 Done Sent Done false
174292024-02-03T17:17:06.305Z 7 Acked 1006 Read 1 Done Sent Done false
174302024-02-03T17:17:06.305Z 8 Acked 1007 Write 8 Done Sent Done false
174312024-02-03T17:17:06.306Z 9 Acked 1008 Read 8 Done Sent Done false
174322024-02-03T17:17:06.306Z 10 Acked 1009 Read 7 Done Sent Done false
174332024-02-03T17:17:06.306Z 11 Acked 1010 Read 8 Done Sent Done false
174342024-02-03T17:17:06.306Z 12 Acked 1011 Read 8 Done Sent Done false
174352024-02-03T17:17:06.306Z 13 Acked 1012 Write 10 Done Sent Done false
174362024-02-03T17:17:06.306Z 14 Acked 1013 Read 8 Done Sent Done false
174372024-02-03T17:17:06.306Z 15 Acked 1014 Read 1 Done Sent Done false
174382024-02-03T17:17:06.306Z 16 Acked 1015 Write 7 Done Sent Done false
174392024-02-03T17:17:06.307Z 17 Acked 1016 Read 5 Done Sent Done false
174402024-02-03T17:17:06.307Z 18 Acked 1017 Write 6 Done Sent Done false
174412024-02-03T17:17:06.307Z 19 Acked 1018 Read 8 Done Sent Done false
174422024-02-03T17:17:06.307Z 20 Acked 1019 Flush 0 Done Sent Done false
174432024-02-03T17:17:06.307Z 21 Acked 1020 Write 2 Done Sent Done false
174442024-02-03T17:17:06.307Z 22 Acked 1021 Write 6 Done Sent Done false
174452024-02-03T17:17:06.307Z 23 Acked 1022 Write 6 Done Sent Done false
174462024-02-03T17:17:06.307Z 24 Acked 1023 Read 1 Done Sent Done false
174472024-02-03T17:17:06.307Z 25 Acked 1024 Write 5 Done Sent Done false
174482024-02-03T17:17:06.307Z 26 Acked 1025 Flush 0 Done Sent Done false
174492024-02-03T17:17:06.307Z 27 Acked 1026 Write 1 Done Sent Done false
174502024-02-03T17:17:06.307Z 28 Acked 1027 Read 2 Done Sent Done false
174512024-02-03T17:17:06.307Z 29 Acked 1028 Write 10 Done Sent Done false
174522024-02-03T17:17:06.307Z 30 Acked 1029 Read 10 Done Sent Done false
174532024-02-03T17:17:06.307Z STATES DS:0 DS:1 DS:2 TOTAL
174542024-02-03T17:17:06.307Z New 0 0 0 0
174552024-02-03T17:17:06.307Z Sent 0 29 0 29
174562024-02-03T17:17:06.307Z Done 30 1 30 61
174572024-02-03T17:17:06.307Z Skipped 0 0 0 0
174582024-02-03T17:17:06.307Z Error 0 0 0 0
174592024-02-03T17:17:06.307ZLast Flush: 1025 0 1025
174602024-02-03T17:17:06.307ZDownstairs last five completed:
174612024-02-03T17:17:06.307ZUpstairs last five completed: 30 29 28 27 26
174622024-02-03T17:17:06.308Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:17:05.835838635Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1775}
174632024-02-03T17:17:06.308ZWrote out file "/tmp/test_repair_verify.data"
174642024-02-03T17:17:06.308Z
174652024-02-03T17:17:06.309ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
174662024-02-03T17:17:06.309ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
174672024-02-03T17:17:06.309Z 0 000-019 160 158 160 277 275 277 F F F <---
174682024-02-03T17:17:06.309Z 1 020-039 160 159 160 277 276 277 F F F <---
174692024-02-03T17:17:06.309Z 2 040-059 159 159 159 276 276 276 F F F
174702024-02-03T17:17:06.309Z 3 060-079 159 159 159 276 276 276 F F F
174712024-02-03T17:17:06.310Z 4 080-099 160 158 160 277 273 277 F F F <---
174722024-02-03T17:17:06.310Z 5 100-119 158 158 158 275 275 275 F F F
174732024-02-03T17:17:06.310Z 6 120-139 158 158 158 275 275 275 F F F
174742024-02-03T17:17:06.310Z 7 140-159 160 158 160 278 275 278 F F F <---
174752024-02-03T17:17:06.310Z 8 160-179 160 157 160 278 272 278 F F F <---
174762024-02-03T17:17:06.310Z 9 180-199 153 153 153 268 268 268 F F F
174772024-02-03T17:17:06.310Z 10 200-219 154 154 154 269 269 269 F F F
174782024-02-03T17:17:06.310Z 11 220-239 160 159 160 277 276 277 F F F <---
174792024-02-03T17:17:06.310Z 12 240-259 160 154 160 277 269 277 F F F <---
174802024-02-03T17:17:06.310Z 13 260-279 150 150 150 264 264 264 F F F
174812024-02-03T17:17:06.311Z 14 280-299 156 156 156 271 271 271 F F F
174822024-02-03T17:17:06.311Z 15 300-319 158 158 158 275 275 275 F F F
174832024-02-03T17:17:06.311Z 16 320-339 158 158 158 273 273 273 F F F
174842024-02-03T17:17:06.311Z 17 340-359 158 158 158 275 275 275 F F F
174852024-02-03T17:17:06.311Z 18 360-379 160 157 160 278 272 278 F F F <---
174862024-02-03T17:17:06.311Z 19 380-399 160 158 160 278 275 278 F F F <---
174872024-02-03T17:17:06.311Z 20 400-419 157 157 157 272 272 272 F F F
174882024-02-03T17:17:06.311Z 21 420-439 159 159 159 276 276 276 T F T <---
174892024-02-03T17:17:06.311Z 22 440-459 156 156 156 270 270 270 T F T <---
174902024-02-03T17:17:06.312Z 23 460-479 160 159 160 278 276 278 F F F <---
174912024-02-03T17:17:06.312Z 24 480-499 160 159 160 278 276 278 T F T <---
174922024-02-03T17:17:06.312Z 25 500-519 157 157 157 272 272 272 F F F
174932024-02-03T17:17:06.312Z 26 520-539 148 148 148 256 256 256 F F F
174942024-02-03T17:17:06.312Z 27 540-559 158 158 158 274 274 274 F F F
174952024-02-03T17:17:06.312Z 28 560-579 160 149 160 277 257 277 F T F <---
174962024-02-03T17:17:06.312Z 29 580-599 160 156 160 278 271 278 F F F <---
174972024-02-03T17:17:06.312ZMax gen: 160, Max flush: 278
174982024-02-03T17:17:06.312ZError: Difference in extent metadata found!
174992024-02-03T17:17:06.330ZA Difference in extent metadata is expected here
175002024-02-03T17:17:06.330ZOn loop 80
175012024-02-03T17:17:06.330Z
175022024-02-03T17:17:06.330ZVerifying data now
175032024-02-03T17:17:11.574ZNo change after verify
175042024-02-03T17:17:11.575ZLoop: 80 Downstairs dump after verify (and repair):
175052024-02-03T17:17:11.621ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
175062024-02-03T17:17:11.621Z 0 000-019 160 160 160 277 277 277 F F F
175072024-02-03T17:17:11.621Z 1 020-039 160 160 160 277 277 277 F F F
175082024-02-03T17:17:11.621Z 2 040-059 159 159 159 276 276 276 F F F
175092024-02-03T17:17:11.621Z 3 060-079 159 159 159 276 276 276 F F F
175102024-02-03T17:17:11.621Z 4 080-099 160 160 160 277 277 277 F F F
175112024-02-03T17:17:11.621Z 5 100-119 158 158 158 275 275 275 F F F
175122024-02-03T17:17:11.621Z 6 120-139 158 158 158 275 275 275 F F F
175132024-02-03T17:17:11.621Z 7 140-159 160 160 160 278 278 278 F F F
175142024-02-03T17:17:11.621Z 8 160-179 160 160 160 278 278 278 F F F
175152024-02-03T17:17:11.621Z 9 180-199 153 153 153 268 268 268 F F F
175162024-02-03T17:17:11.621Z 10 200-219 154 154 154 269 269 269 F F F
175172024-02-03T17:17:11.621Z 11 220-239 160 160 160 277 277 277 F F F
175182024-02-03T17:17:11.621Z 12 240-259 160 160 160 277 277 277 F F F
175192024-02-03T17:17:11.622Z 13 260-279 150 150 150 264 264 264 F F F
175202024-02-03T17:17:11.622Z 14 280-299 156 156 156 271 271 271 F F F
175212024-02-03T17:17:11.622Z 15 300-319 158 158 158 275 275 275 F F F
175222024-02-03T17:17:11.622Z 16 320-339 158 158 158 273 273 273 F F F
175232024-02-03T17:17:11.622Z 17 340-359 158 158 158 275 275 275 F F F
175242024-02-03T17:17:11.622Z 18 360-379 160 160 160 278 278 278 F F F
175252024-02-03T17:17:11.622Z 19 380-399 160 160 160 278 278 278 F F F
175262024-02-03T17:17:11.622Z 20 400-419 157 157 157 272 272 272 F F F
175272024-02-03T17:17:11.622Z 21 420-439 161 161 161 279 279 279 F F F
175282024-02-03T17:17:11.622Z 22 440-459 161 161 161 279 279 279 F F F
175292024-02-03T17:17:11.622Z 23 460-479 160 160 160 278 278 278 F F F
175302024-02-03T17:17:11.622Z 24 480-499 161 161 161 279 279 279 F F F
175312024-02-03T17:17:11.622Z 25 500-519 157 157 157 272 272 272 F F F
175322024-02-03T17:17:11.622Z 26 520-539 148 148 148 256 256 256 F F F
175332024-02-03T17:17:11.622Z 27 540-559 158 158 158 274 274 274 F F F
175342024-02-03T17:17:11.622Z 28 560-579 160 160 160 277 277 277 F F F
175352024-02-03T17:17:11.623Z 29 580-599 160 160 160 278 278 278 F F F
175362024-02-03T17:17:11.623ZMax gen: 161, Max flush: 279
175372024-02-03T17:17:11.623Z
175382024-02-03T17:17:11.623ZBegin loop 81
175392024-02-03T17:17:11.623ZDownstairs to restart: 0
175402024-02-03T17:17:12.208Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.208042949Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783}
175412024-02-03T17:17:12.208Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.208489001Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783}
175422024-02-03T17:17:12.208Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.208524505Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783}
175432024-02-03T17:17:12.209Z{"msg":"Crucible runtime is spawned
175442024-02-03T17:17:12.209ZCrucible bd972416-5542-4567-b771-844c26feaf2f has session id: 7d0ba128-8b56-456a-b5ad-69e3a683aaec","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.2088162Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175452024-02-03T17:17:12.209Z{"msg":"Upstairs opts: Upstairs UUID: bd972416-5542-4567-b771-844c26feaf2f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.208903381Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175462024-02-03T17:17:12.209Z{"msg":"Crucible stats registered with UUID: bd972416-5542-4567-b771-844c26feaf2f","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.208931707Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175472024-02-03T17:17:12.209Z{"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.208979119Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783}
175482024-02-03T17:17:12.209Z{"msg":"bd972416-5542-4567-b771-844c26feaf2f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.209071365Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175492024-02-03T17:17:12.209Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.209151385Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175502024-02-03T17:17:12.210Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.209209144Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175512024-02-03T17:17:12.210Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.209292118Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175522024-02-03T17:17:12.210Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.209319075Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175532024-02-03T17:17:12.210Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.20934145Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175542024-02-03T17:17:12.210Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.209364481Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175552024-02-03T17:17:12.210Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.209462595Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"":"io task","client":"2","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175562024-02-03T17:17:12.210Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.209494227Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"":"io task","client":"1","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175572024-02-03T17:17:12.210Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.209517299Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"":"io task","client":"0","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175582024-02-03T17:17:12.211Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.209550043Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"":"io task","client":"0","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175592024-02-03T17:17:12.211Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.209731073Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"":"io task","client":"2","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175602024-02-03T17:17:12.211Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.209925778Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"":"io task","client":"1","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175612024-02-03T17:17:12.211Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.210860507Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175622024-02-03T17:17:12.211Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.210958421Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175632024-02-03T17:17:12.211Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.211006854Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175642024-02-03T17:17:12.212Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.211097881Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175652024-02-03T17:17:12.212Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.211904504Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175662024-02-03T17:17:12.212Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.212028544Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175672024-02-03T17:17:12.213Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.212198974Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175682024-02-03T17:17:12.213Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.212500795Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175692024-02-03T17:17:12.213Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.212911148Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175702024-02-03T17:17:12.213Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.213003793Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175712024-02-03T17:17:12.213Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.213077191Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175722024-02-03T17:17:12.213Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.213104427Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175732024-02-03T17:17:12.213Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.213413533Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175742024-02-03T17:17:12.213Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.213508593Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175752024-02-03T17:17:12.213Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.213557261Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175762024-02-03T17:17:12.213Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.214015648Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175772024-02-03T17:17:12.213Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.21409757Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175782024-02-03T17:17:12.214Z{"msg":"[0]R flush_numbers[0..12]: [277, 277, 276, 276, 277, 275, 275, 278, 278, 268, 269, 277]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.214138657Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175792024-02-03T17:17:12.214Z{"msg":"[0]R generation[0..12]: [160, 160, 159, 159, 160, 158, 158, 160, 160, 153, 154, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.214165341Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175802024-02-03T17:17:12.214Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.214217916Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175812024-02-03T17:17:12.214Z{"msg":"[1]R flush_numbers[0..12]: [277, 277, 276, 276, 277, 275, 275, 278, 278, 268, 269, 277]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.214244863Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175822024-02-03T17:17:12.214Z{"msg":"[1]R generation[0..12]: [160, 160, 159, 159, 160, 158, 158, 160, 160, 153, 154, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.214268168Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175832024-02-03T17:17:12.214Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
175842024-02-03T17:17:12.214Z,"time":"2024-02-03T17:17:11.214317468Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175852024-02-03T17:17:12.214Z{"msg":"[2]R flush_numbers[0..12]: [277, 277, 276, 276, 277, 275, 275, 278, 278, 268, 269, 277]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.214346573Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175862024-02-03T17:17:12.214Z{"msg":"[2]R generation[0..12]: [160, 160, 159, 159, 160, 158, 158, 160, 160, 153, 154, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.21439539Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175872024-02-03T17:17:12.214Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.214421608Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175882024-02-03T17:17:12.214Z{"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.214470557Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175892024-02-03T17:17:12.215Z{"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.214495199Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175902024-02-03T17:17:12.215Z{"msg":"Next flush: 280","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.214518418Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175912024-02-03T17:17:12.215Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.214540443Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175922024-02-03T17:17:12.215Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.214587992Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175932024-02-03T17:17:12.215Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.214610819Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175942024-02-03T17:17:12.215Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.214659209Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175952024-02-03T17:17:12.215Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.214683486Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175962024-02-03T17:17:12.215Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.214732233Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175972024-02-03T17:17:12.216Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.214755245Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175982024-02-03T17:17:12.216Z{"msg":"bd972416-5542-4567-b771-844c26feaf2f is now active with session: 7d0ba128-8b56-456a-b5ad-69e3a683aaec","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.214802237Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
175992024-02-03T17:17:12.216Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.214824488Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783,"session_id":"7d0ba128-8b56-456a-b5ad-69e3a683aaec"}
176002024-02-03T17:17:12.216Z{"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:11.214856817Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783}
176012024-02-03T17:17:12.216ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
176022024-02-03T17:17:12.216ZLoading write count information from file "/tmp/test_repair_verify.data"
176032024-02-03T17:17:12.216ZRun Repair workload
176042024-02-03T17:17:12.216Z01/30 Write block 47 len 12288 data: 12 12 14
176052024-02-03T17:17:12.216Z02/30 Read block 244 len 40960
176062024-02-03T17:17:12.216Z03/30 Read block 283 len 36864
176072024-02-03T17:17:12.216Z04/30 Write block 181 len 40960 data: 15 14 15 14 18 15 20 21 23 22
176082024-02-03T17:17:12.216Z05/30 Write block 134 len 12288 data: 9 9 9
176092024-02-03T17:17:12.217Z06/30 Read block 487 len 24576
176102024-02-03T17:17:12.217Z07/30 Write block 555 len 28672 data: 12 13 13 10 11 11 12
176112024-02-03T17:17:12.217Z08/30 Write block 81 len 12288 data: 15 14 11
176122024-02-03T17:17:12.217Z09/30 Read block 44 len 32768
176132024-02-03T17:17:12.217Z10/30 Write block 477 len 12288 data: 10 12 14
176142024-02-03T17:17:12.217Z11/30 Write block 243 len 12288 data: 8 9 10
176152024-02-03T17:17:12.217Z12/30 Read block 245 len 36864
176162024-02-03T17:17:12.217Z13/30 Write block 123 len 28672 data: 12 10 8 9 10 11 12
176172024-02-03T17:17:12.217Z14/30 Write block 167 len 28672 data: 20 17 17 18 15 12 12
176182024-02-03T17:17:12.218Z15/30 Read block 296 len 4096
176192024-02-03T17:17:12.218Z16/30 Write block 547 len 12288 data: 14 12 14
176202024-02-03T17:17:12.218Z17/30 Read block 324 len 4096
176212024-02-03T17:17:12.218Z18/30 Write block 61 len 16384 data: 10 12 11 9
176222024-02-03T17:17:12.218Z19/30 Write block 521 len 24576 data: 8 10 11 10 12 11
176232024-02-03T17:17:12.218Z20/30 Read block 303 len 32768
176242024-02-03T17:17:12.218Z21/30 Read block 236 len 20480
176252024-02-03T17:17:12.218Z22/30 Write block 320 len 4096 data: 15
176262024-02-03T17:17:12.218Z23/30 Read block 191 len 20480
176272024-02-03T17:17:12.218Z24/30 Write block 89 len 12288 data: 12 12 12
176282024-02-03T17:17:12.218Z25/30 Write block 541 len 40960 data: 9 12 11 9 10 11 15 13 15 17
176292024-02-03T17:17:12.219Z26/30 Read block 126 len 8192
176302024-02-03T17:17:12.219Z27/30 Read block 285 len 20480
176312024-02-03T17:17:12.219Z28/30 Write block 171 len 40960 data: 16 13 13 12 11 11 15 12 12 12
176322024-02-03T17:17:12.219Z29/30 Write block 74 len 16384 data: 8 11 13 12
176332024-02-03T17:17:12.219Z30/30 Read block 563 len 4096
176342024-02-03T17:17:12.219Z----------------------------------------------------------------
176352024-02-03T17:17:12.219Z Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:30
176362024-02-03T17:17:12.219ZStates: Active Active Active
176372024-02-03T17:17:12.219ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
176382024-02-03T17:17:12.219Z 1 Acked 1000 Write 3 Done Done Done false
176392024-02-03T17:17:12.220Z 2 Acked 1001 Read 10 Done Done Done false
176402024-02-03T17:17:12.296Z 3 Acked 1002 Read 9 Sent Done Done false
176412024-02-03T17:17:12.296Z 4 Acked 1003 Write 10 Sent Done Done false
176422024-02-03T17:17:12.296Z 5 Acked 1004 Write 3 Sent Done Done false
176432024-02-03T17:17:12.296Z 6 Acked 1005 Read 6 Sent Done Done false
176442024-02-03T17:17:12.296Z 7 Acked 1006 Write 7 Sent Done Done false
176452024-02-03T17:17:12.297Z 8 Acked 1007 Write 3 Sent Done Done false
176462024-02-03T17:17:12.297Z 9 Acked 1008 Read 8 Sent Done Done false
176472024-02-03T17:17:12.297Z 10 Acked 1009 Write 3 Sent Done Done false
176482024-02-03T17:17:12.297Z 11 Acked 1010 Write 3 Sent Done Done false
176492024-02-03T17:17:12.297Z 12 Acked 1011 Read 9 Sent Done Done false
176502024-02-03T17:17:12.297Z 13 Acked 1012 Write 7 Sent Done Done false
176512024-02-03T17:17:12.298Z 14 Acked 1013 Write 7 Sent Done Done false
176522024-02-03T17:17:12.298Z 15 Acked 1014 Read 1 Sent Done Done false
176532024-02-03T17:17:12.298Z 16 Acked 1015 Write 3 Sent Done Done false
176542024-02-03T17:17:12.298Z 17 Acked 1016 Read 1 Sent Done Done false
176552024-02-03T17:17:12.298Z 18 Acked 1017 Write 4 Sent Done Done false
176562024-02-03T17:17:12.298Z 19 Acked 1018 Write 6 Sent Done Done false
176572024-02-03T17:17:12.298Z 20 Acked 1019 Read 8 Sent Done Done false
176582024-02-03T17:17:12.299Z 21 Acked 1020 Read 5 Sent Done Done false
176592024-02-03T17:17:12.299Z 22 Acked 1021 Write 1 Sent Done Done false
176602024-02-03T17:17:12.299Z 23 Acked 1022 Read 5 Sent Done Done false
176612024-02-03T17:17:12.299Z 24 Acked 1023 Write 3 Sent Done Done false
176622024-02-03T17:17:12.299Z 25 Acked 1024 Write 10 Sent Done Done false
176632024-02-03T17:17:12.299Z 26 Acked 1025 Read 2 Sent Done Done false
176642024-02-03T17:17:12.299Z 27 Acked 1026 Read 5 Sent Done Done false
176652024-02-03T17:17:12.299Z 28 Acked 1027 Write 10 Sent Done Done false
176662024-02-03T17:17:12.299Z 29 Acked 1028 Write 4 Sent Done Done false
176672024-02-03T17:17:12.299Z 30 Acked 1029 Read 1 Sent Done Done false
176682024-02-03T17:17:12.300Z STATES DS:0 DS:1 DS:2 TOTAL
176692024-02-03T17:17:12.300Z New 0 0 0 0
176702024-02-03T17:17:12.300Z Sent 28 0 0 28
176712024-02-03T17:17:12.300Z Done 2 30 30 62
176722024-02-03T17:17:12.300Z Skipped 0 0 0 0
176732024-02-03T17:17:12.300Z Error 0 0 0 0
176742024-02-03T17:17:12.300ZLast Flush: 0 0 0
176752024-02-03T17:17:12.300ZDownstairs last five completed:
176762024-02-03T17:17:12.301ZUpstairs last five completed: 30 29 28 27 26
176772024-02-03T17:17:12.301Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:17:11.260698942Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1783}
176782024-02-03T17:17:12.301ZWrote out file "/tmp/test_repair_verify.data"
176792024-02-03T17:17:12.302Z
176802024-02-03T17:17:12.302ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
176812024-02-03T17:17:12.302ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
176822024-02-03T17:17:12.302Z 0 000-019 160 160 160 277 277 277 F F F
176832024-02-03T17:17:12.302Z 1 020-039 160 160 160 277 277 277 F F F
176842024-02-03T17:17:12.302Z 2 040-059 159 159 159 276 276 276 T T T
176852024-02-03T17:17:12.303Z 3 060-079 159 159 159 276 276 276 F T T <---
176862024-02-03T17:17:12.303Z 4 080-099 160 160 160 277 277 277 F T T <---
176872024-02-03T17:17:12.303Z 5 100-119 158 158 158 275 275 275 F F F
176882024-02-03T17:17:12.303Z 6 120-139 158 158 158 275 275 275 F T T <---
176892024-02-03T17:17:12.303Z 7 140-159 160 160 160 278 278 278 F F F
176902024-02-03T17:17:12.303Z 8 160-179 160 160 160 278 278 278 F T T <---
176912024-02-03T17:17:12.303Z 9 180-199 153 153 153 268 268 268 F T T <---
176922024-02-03T17:17:12.304Z 10 200-219 154 154 154 269 269 269 F F F
176932024-02-03T17:17:12.304Z 11 220-239 160 160 160 277 277 277 F F F
176942024-02-03T17:17:12.304Z 12 240-259 160 160 160 277 277 277 F T T <---
176952024-02-03T17:17:12.304Z 13 260-279 150 150 150 264 264 264 F F F
176962024-02-03T17:17:12.304Z 14 280-299 156 156 156 271 271 271 F F F
176972024-02-03T17:17:12.304Z 15 300-319 158 158 158 275 275 275 F F F
176982024-02-03T17:17:12.304Z 16 320-339 158 158 158 273 273 273 F T T <---
176992024-02-03T17:17:12.305Z 17 340-359 158 158 158 275 275 275 F F F
177002024-02-03T17:17:12.305Z 18 360-379 160 160 160 278 278 278 F F F
177012024-02-03T17:17:12.305Z 19 380-399 160 160 160 278 278 278 F F F
177022024-02-03T17:17:12.306Z 20 400-419 157 157 157 272 272 272 F F F
177032024-02-03T17:17:12.306Z 21 420-439 161 161 161 279 279 279 F F F
177042024-02-03T17:17:12.306Z 22 440-459 161 161 161 279 279 279 F F F
177052024-02-03T17:17:12.306Z 23 460-479 160 160 160 278 278 278 F T T <---
177062024-02-03T17:17:12.306Z 24 480-499 161 161 161 279 279 279 F F F
177072024-02-03T17:17:12.307Z 25 500-519 157 157 157 272 272 272 F F F
177082024-02-03T17:17:12.307Z 26 520-539 148 148 148 256 256 256 F T T <---
177092024-02-03T17:17:12.307Z 27 540-559 158 158 158 274 274 274 F T T <---
177102024-02-03T17:17:12.307Z 28 560-579 160 160 160 277 277 277 F T T <---
177112024-02-03T17:17:12.307Z 29 580-599 160 160 160 278 278 278 F F F
177122024-02-03T17:17:12.307ZMax gen: 161, Max flush: 279
177132024-02-03T17:17:12.307ZError: Difference in extent metadata found!
177142024-02-03T17:17:12.308ZA Difference in extent metadata is expected here
177152024-02-03T17:17:12.308ZOn loop 81
177162024-02-03T17:17:12.308Z
177172024-02-03T17:17:12.308ZVerifying data now
177182024-02-03T17:17:31.871ZNo change after verify
177192024-02-03T17:17:31.909ZLoop: 81 Downstairs dump after verify (and repair):
177202024-02-03T17:17:31.909ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
177212024-02-03T17:17:31.909Z 0 000-019 160 160 160 277 277 277 F F F
177222024-02-03T17:17:31.910Z 1 020-039 160 160 160 277 277 277 F F F
177232024-02-03T17:17:31.910Z 2 040-059 162 162 162 280 280 280 F F F
177242024-02-03T17:17:31.910Z 3 060-079 162 162 162 280 280 280 F F F
177252024-02-03T17:17:31.910Z 4 080-099 162 162 162 280 280 280 F F F
177262024-02-03T17:17:31.910Z 5 100-119 158 158 158 275 275 275 F F F
177272024-02-03T17:17:31.910Z 6 120-139 162 162 162 280 280 280 F F F
177282024-02-03T17:17:31.910Z 7 140-159 160 160 160 278 278 278 F F F
177292024-02-03T17:17:31.910Z 8 160-179 162 162 162 280 280 280 F F F
177302024-02-03T17:17:31.910Z 9 180-199 162 162 162 280 280 280 F F F
177312024-02-03T17:17:31.911Z 10 200-219 154 154 154 269 269 269 F F F
177322024-02-03T17:17:31.911Z 11 220-239 160 160 160 277 277 277 F F F
177332024-02-03T17:17:31.911Z 12 240-259 162 162 162 280 280 280 F F F
177342024-02-03T17:17:31.911Z 13 260-279 150 150 150 264 264 264 F F F
177352024-02-03T17:17:31.911Z 14 280-299 156 156 156 271 271 271 F F F
177362024-02-03T17:17:31.911Z 15 300-319 158 158 158 275 275 275 F F F
177372024-02-03T17:17:31.911Z 16 320-339 162 162 162 280 280 280 F F F
177382024-02-03T17:17:31.911Z 17 340-359 158 158 158 275 275 275 F F F
177392024-02-03T17:17:31.911Z 18 360-379 160 160 160 278 278 278 F F F
177402024-02-03T17:17:31.911Z 19 380-399 160 160 160 278 278 278 F F F
177412024-02-03T17:17:31.911Z 20 400-419 157 157 157 272 272 272 F F F
177422024-02-03T17:17:31.911Z 21 420-439 161 161 161 279 279 279 F F F
177432024-02-03T17:17:31.911Z 22 440-459 161 161 161 279 279 279 F F F
177442024-02-03T17:17:31.911Z 23 460-479 162 162 162 280 280 280 F F F
177452024-02-03T17:17:31.911Z 24 480-499 161 161 161 279 279 279 F F F
177462024-02-03T17:17:31.911Z 25 500-519 157 157 157 272 272 272 F F F
177472024-02-03T17:17:31.911Z 26 520-539 162 162 162 280 280 280 F F F
177482024-02-03T17:17:31.911Z 27 540-559 162 162 162 280 280 280 F F F
177492024-02-03T17:17:31.911Z 28 560-579 162 162 162 280 280 280 F F F
177502024-02-03T17:17:31.911Z 29 580-599 160 160 160 278 278 278 F F F
177512024-02-03T17:17:31.911ZMax gen: 162, Max flush: 280
177522024-02-03T17:17:31.911Z
177532024-02-03T17:17:31.911ZBegin loop 82
177542024-02-03T17:17:31.911ZDownstairs to restart: 1
177552024-02-03T17:17:31.912Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.431373925Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791}
177562024-02-03T17:17:31.912Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.431903611Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791}
177572024-02-03T17:17:31.912Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.431990297Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791}
177582024-02-03T17:17:31.912ZCrucible runtime is spawned
177592024-02-03T17:17:31.912Z{"msg":"Crucible 75a919f3-70de-4fef-9def-6a1a435175e7 has session id: b4aa1ffb-a66c-4b77-a563-5d02a478655d","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.432117138Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177602024-02-03T17:17:31.912Z{"msg":"Upstairs opts: Upstairs UUID: 75a919f3-70de-4fef-9def-6a1a435175e7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.43222606Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177612024-02-03T17:17:31.912Z{"msg":"Crucible stats registered with UUID: 75a919f3-70de-4fef-9def-6a1a435175e7","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.432301687Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177622024-02-03T17:17:31.912Z{"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.432353664Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791}
177632024-02-03T17:17:31.912Z{"msg":"75a919f3-70de-4fef-9def-6a1a435175e7 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.432404132Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177642024-02-03T17:17:31.912Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.432452254Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177652024-02-03T17:17:31.912Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.432503139Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177662024-02-03T17:17:31.912Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.432555443Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177672024-02-03T17:17:31.912Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.432607331Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177682024-02-03T17:17:31.912Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.432679388Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177692024-02-03T17:17:31.912Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.432737471Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177702024-02-03T17:17:31.912Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.432801518Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"":"io task","client":"2","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177712024-02-03T17:17:31.912Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.432846706Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"":"io task","client":"0","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177722024-02-03T17:17:31.912Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.432955246Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"":"io task","client":"1","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177732024-02-03T17:17:31.912Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.433069035Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"":"io task","client":"2","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177742024-02-03T17:17:31.912Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.433116281Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"":"io task","client":"0","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177752024-02-03T17:17:31.912Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.433158441Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"":"io task","client":"1","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177762024-02-03T17:17:31.912Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.434608582Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177772024-02-03T17:17:31.912Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.434688354Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177782024-02-03T17:17:31.913Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.434780208Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177792024-02-03T17:17:31.913Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.434894188Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177802024-02-03T17:17:31.913Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.435402954Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177812024-02-03T17:17:31.913Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.435527921Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177822024-02-03T17:17:31.913Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.435874135Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177832024-02-03T17:17:31.913Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.435947146Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177842024-02-03T17:17:31.913Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.436601109Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177852024-02-03T17:17:31.913Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.436889962Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177862024-02-03T17:17:31.913Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.436969564Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177872024-02-03T17:17:31.913Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.437044453Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177882024-02-03T17:17:31.913Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.437109629Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177892024-02-03T17:17:31.913Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.437175237Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177902024-02-03T17:17:31.913Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.437241828Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177912024-02-03T17:17:31.913Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.43748047Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177922024-02-03T17:17:31.913Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.437548192Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177932024-02-03T17:17:31.913Z{"msg":"[0]R flush_numbers[0..12]: [277, 277, 280, 280, 280, 275, 280, 278, 280, 280, 269, 277]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.437618228Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177942024-02-03T17:17:31.913Z{"msg":"[0]R generation[0..12]: [160, 160, 162, 162, 162, 158, 162, 160, 162, 162, 154, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.437685634Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177952024-02-03T17:17:31.913Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.437753316Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177962024-02-03T17:17:31.913Z{"msg":"[1]R flush_numbers[0..12]: [277, 277, 280, 280, 280, 275, 280, 278, 280, 280, 269, 277]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
177972024-02-03T17:17:31.913Z,"time":"2024-02-03T17:17:16.437820283Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177982024-02-03T17:17:31.913Z{"msg":"[1]R generation[0..12]: [160, 160, 162, 162, 162, 158, 162, 160, 162, 162, 154, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.437892677Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
177992024-02-03T17:17:31.913Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.437959704Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
178002024-02-03T17:17:31.913Z{"msg":"[2]R flush_numbers[0..12]: [277, 277, 280, 280, 280, 275, 280, 278, 280, 280, 269, 277]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.438026454Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
178012024-02-03T17:17:31.914Z{"msg":"[2]R generation[0..12]: [160, 160, 162, 162, 162, 158, 162, 160, 162, 162, 154, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.438094959Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
178022024-02-03T17:17:31.914Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.438162229Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
178032024-02-03T17:17:31.914Z{"msg":"Max found gen is 163","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.43823962Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
178042024-02-03T17:17:31.914Z{"msg":"Generation requested: 164 >= found:163","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.438306283Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
178052024-02-03T17:17:31.914Z{"msg":"Next flush: 281","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.438371357Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
178062024-02-03T17:17:31.914Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.438439818Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
178072024-02-03T17:17:31.914Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.438514103Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
178082024-02-03T17:17:31.914Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.438577149Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal",Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
178092024-02-03T17:17:31.914Z"pid":1791,"client":"0","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
178102024-02-03T17:17:31.914Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.438716612Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
178112024-02-03T17:17:31.914Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.438784371Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
178122024-02-03T17:17:31.914Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.438850099Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
178132024-02-03T17:17:31.914Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.438913826Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
178142024-02-03T17:17:31.914Z{"msg":"75a919f3-70de-4fef-9def-6a1a435175e7 is now active with session: b4aa1ffb-a66c-4b77-a563-5d02a478655d","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.438978829Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
178152024-02-03T17:17:31.914Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.439040835Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791,"session_id":"b4aa1ffb-a66c-4b77-a563-5d02a478655d"}
178162024-02-03T17:17:31.914Z{"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:16.43910437Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791}
178172024-02-03T17:17:31.914ZLoading write count information from file "/tmp/test_repair_verify.data"
178182024-02-03T17:17:31.914ZRun Repair workload
178192024-02-03T17:17:31.953ZError: Difference in extent metadata found!
178202024-02-03T17:17:31.953Z01/30 Write block 90 len 12288 data: 13 13 12
178212024-02-03T17:17:31.954Z02/30 Write block 202 len 32768 data: 11 10 10 13 12 13 14 13
178222024-02-03T17:17:31.954Z03/30 Read block 133 len 8192
178232024-02-03T17:17:31.954Z04/30 Read block 73 len 32768
178242024-02-03T17:17:31.954Z05/30 Flush
178252024-02-03T17:17:31.954Z06/30 Write block 404 len 36864 data: 7 11 14 15 13 12 13 13 14
178262024-02-03T17:17:31.954Z07/30 Read block 104 len 16384
178272024-02-03T17:17:31.954Z08/30 Read block 203 len 32768
178282024-02-03T17:17:31.954Z09/30 Write block 126 len 36864 data: 10 11 12 13 12 13 12 10 10
178292024-02-03T17:17:31.954Z10/30 Read block 290 len 24576
178302024-02-03T17:17:31.954Z11/30 Write block 261 len 20480 data: 19 18 19 20 19
178312024-02-03T17:17:31.954Z12/30 Write block 98 len 20480 data: 11 12 12 12 8
178322024-02-03T17:17:31.954Z13/30 Write block 80 len 36864 data: 12 16 15 12 11 9 10 11 10
178332024-02-03T17:17:31.955Z14/30 Read block 85 len 20480
178342024-02-03T17:17:31.955Z15/30 Read block 351 len 20480
178352024-02-03T17:17:31.955Z16/30 Read block 516 len 40960
178362024-02-03T17:17:31.955Z17/30 Write block 118 len 32768 data: 15 16 14 13 13 13 11 9
178372024-02-03T17:17:31.955Z18/30 Write block 378 len 36864 data: 16 18 17 15 15 14 12 10 9
178382024-02-03T17:17:31.955Z19/30 Read block 314 len 12288
178392024-02-03T17:17:31.955Z20/30 Write block 310 len 32768 data: 15 12 14 13 14 16 16 17
178402024-02-03T17:17:31.955Z21/30 Read block 472 len 20480
178412024-02-03T17:17:31.955Z22/30 Read block 341 len 12288
178422024-02-03T17:17:31.955Z23/30 Write block 145 len 40960 data: 10 10 7 12 13 12 14 15 13 12
178432024-02-03T17:17:31.955Z24/30 Read block 297 len 8192
178442024-02-03T17:17:31.955Z25/30 Write block 486 len 8192 data: 11 8
178452024-02-03T17:17:31.955Z26/30 Write block 114 len 20480 data: 10 11 12 16 16
178462024-02-03T17:17:31.955Z27/30 Write block 574 len 28672 data: 12 11 11 8 8 9 8
178472024-02-03T17:17:31.955Z28/30 Read block 271 len 20480
178482024-02-03T17:17:31.955Z29/30 Write block 503 len 28672 data: 15 14 13 14 15 15 15
178492024-02-03T17:17:31.955Z30/30 Write block 511 len 16384 data: 16 17 14 14
178502024-02-03T17:17:31.955Z----------------------------------------------------------------
178512024-02-03T17:17:31.955Z Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:25
178522024-02-03T17:17:31.955ZStates: Active Active Active
178532024-02-03T17:17:31.955ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
178542024-02-03T17:17:31.956Z 6 Acked 1005 Write 9 Done Done Done false
178552024-02-03T17:17:31.956Z 7 Acked 1006 Read 4 Done Done Done false
178562024-02-03T17:17:31.956Z 8 Acked 1007 Read 8 Done Sent Done false
178572024-02-03T17:17:31.956Z 9 Acked 1008 Write 9 Done Sent Done false
178582024-02-03T17:17:31.956Z 10 Acked 1009 Read 6 Done Sent Done false
178592024-02-03T17:17:31.956Z 11 Acked 1010 Write 5 Done Sent Done false
178602024-02-03T17:17:31.956Z 12 Acked 1011 Write 5 Done Sent Done false
178612024-02-03T17:17:31.956Z 13 Acked 1012 Write 9 Done Sent Done false
178622024-02-03T17:17:31.956Z 14 Acked 1013 Read 5 Done Sent Done false
178632024-02-03T17:17:31.956Z 15 Acked 1014 Read 5 Done Sent Done false
178642024-02-03T17:17:31.956Z 16 Acked 1015 Read 10 Done Sent Done false
178652024-02-03T17:17:31.956Z 17 Acked 1016 Write 8 Done Sent Done false
178662024-02-03T17:17:31.956Z 18 Acked 1017 Write 9 Done Sent Done false
178672024-02-03T17:17:31.956Z 19 Acked 1018 Read 3 Done Sent Done false
178682024-02-03T17:17:31.956Z 20 Acked 1019 Write 8 Done Sent Done false
178692024-02-03T17:17:31.956Z 21 Acked 1020 Read 5 Done Sent Done false
178702024-02-03T17:17:31.956Z 22 Acked 1021 Read 3 Done Sent Done false
178712024-02-03T17:17:31.956Z 23 Acked 1022 Write 10 Done Sent Done false
178722024-02-03T17:17:31.956Z 24 Acked 1023 Read 2 Done Sent Done false
178732024-02-03T17:17:31.956Z 25 Acked 1024 Write 2 Done Sent Done false
178742024-02-03T17:17:31.956Z 26 Acked 1025 Write 5 Done Sent Done false
178752024-02-03T17:17:31.956Z 27 Acked 1026 Write 7 Done Sent Done false
178762024-02-03T17:17:31.956Z 28 Acked 1027 Read 5 Done Sent Done false
178772024-02-03T17:17:31.956Z 29 Acked 1028 Write 7 Sent Sent Sent false
178782024-02-03T17:17:31.957Z 30 Acked 1029 Write 4 Sent Sent Sent false
178792024-02-03T17:17:31.957Z STATES DS:0 DS:1 DS:2 TOTAL
178802024-02-03T17:17:31.957Z New 0 0 0 0
178812024-02-03T17:17:31.957Z Sent 2 23 2 27
178822024-02-03T17:17:31.957Z Done 23 2 23 48
178832024-02-03T17:17:31.957Z Skipped 0 0 0 0
178842024-02-03T17:17:31.957Z Error 0 0 0 0
178852024-02-03T17:17:31.957ZLast Flush: 1004 1004 1004
178862024-02-03T17:17:31.957ZDownstairs last five completed: 1004 1003 1002 1001 1000
178872024-02-03T17:17:31.957ZUpstairs last five completed: 30 29 28 27 26
178882024-02-03T17:17:31.957Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:17:16.488987008Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1791}
178892024-02-03T17:17:31.957ZWrote out file "/tmp/test_repair_verify.data"
178902024-02-03T17:17:31.957Z
178912024-02-03T17:17:31.957ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
178922024-02-03T17:17:31.957ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
178932024-02-03T17:17:31.957Z 0 000-019 160 160 160 277 277 277 F F F
178942024-02-03T17:17:31.957Z 1 020-039 160 160 160 277 277 277 F F F
178952024-02-03T17:17:31.957Z 2 040-059 162 162 162 280 280 280 F F F
178962024-02-03T17:17:31.957Z 3 060-079 162 162 162 280 280 280 F F F
178972024-02-03T17:17:31.957Z 4 080-099 164 164 164 281 281 281 T F T <---
178982024-02-03T17:17:31.957Z 5 100-119 158 158 158 275 275 275 T F T <---
178992024-02-03T17:17:31.957Z 6 120-139 162 162 162 280 280 280 T F T <---
179002024-02-03T17:17:31.957Z 7 140-159 160 160 160 278 278 278 T F T <---
179012024-02-03T17:17:31.958Z 8 160-179 162 162 162 280 280 280 F F F
179022024-02-03T17:17:31.958Z 9 180-199 162 162 162 280 280 280 F F F
179032024-02-03T17:17:31.958Z 10 200-219 164 164 164 281 281 281 F F F
179042024-02-03T17:17:31.958Z 11 220-239 160 160 160 277 277 277 F F F
179052024-02-03T17:17:31.958Z 12 240-259 162 162 162 280 280 280 F F F
179062024-02-03T17:17:31.958Z 13 260-279 150 150 150 264 264 264 T F T <---
179072024-02-03T17:17:31.958Z 14 280-299 156 156 156 271 271 271 F F F
179082024-02-03T17:17:31.958Z 15 300-319 158 158 158 275 275 275 T F T <---
179092024-02-03T17:17:31.958Z 16 320-339 162 162 162 280 280 280 F F F
179102024-02-03T17:17:31.958Z 17 340-359 158 158 158 275 275 275 F F F
179112024-02-03T17:17:31.958Z 18 360-379 160 160 160 278 278 278 T F T <---
179122024-02-03T17:17:31.958Z 19 380-399 160 160 160 278 278 278 T F T <---
179132024-02-03T17:17:31.958Z 20 400-419 157 157 157 272 272 272 T T T
179142024-02-03T17:17:31.958Z 21 420-439 161 161 161 279 279 279 F F F
179152024-02-03T17:17:31.958Z 22 440-459 161 161 161 279 279 279 F F F
179162024-02-03T17:17:31.958Z 23 460-479 162 162 162 280 280 280 F F F
179172024-02-03T17:17:31.958Z 24 480-499 161 161 161 279 279 279 T F T <---
179182024-02-03T17:17:31.958Z 25 500-519 157 157 157 272 272 272 T F T <---
179192024-02-03T17:17:32.099Z 26 520-539 162 162 162 280 280 280 F F F
179202024-02-03T17:17:32.099Z 27 540-559 162 162 162 280 280 280 F F F
179212024-02-03T17:17:32.100Z 28 560-579 162 162 162 280 280 280 T F T <---
179222024-02-03T17:17:32.100Z 29 580-599 160 160 160 278 278 278 T F T <---
179232024-02-03T17:17:32.100ZMax gen: 164, Max flush: 281
179242024-02-03T17:17:32.100ZA Difference in extent metadata is expected here
179252024-02-03T17:17:32.100ZOn loop 82
179262024-02-03T17:17:32.100Z
179272024-02-03T17:17:32.101ZVerifying data now
179282024-02-03T17:17:32.101ZNo change after verify
179292024-02-03T17:17:32.101ZLoop: 82 Downstairs dump after verify (and repair):
179302024-02-03T17:17:32.101ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
179312024-02-03T17:17:32.101Z 0 000-019 160 160 160 277 277 277 F F F
179322024-02-03T17:17:32.101Z 1 020-039 160 160 160 277 277 277 F F F
179332024-02-03T17:17:32.101Z 2 040-059 162 162 162 280 280 280 F F F
179342024-02-03T17:17:32.101Z 3 060-079 162 162 162 280 280 280 F F F
179352024-02-03T17:17:32.101Z 4 080-099 165 165 165 282 282 282 F F F
179362024-02-03T17:17:32.101Z 5 100-119 165 165 165 282 282 282 F F F
179372024-02-03T17:17:32.101Z 6 120-139 165 165 165 282 282 282 F F F
179382024-02-03T17:17:32.101Z 7 140-159 165 165 165 282 282 282 F F F
179392024-02-03T17:17:32.102Z 8 160-179 162 162 162 280 280 280 F F F
179402024-02-03T17:17:32.102Z 9 180-199 162 162 162 280 280 280 F F F
179412024-02-03T17:17:32.102Z 10 200-219 164 164 164 281 281 281 F F F
179422024-02-03T17:17:32.102Z 11 220-239 160 160 160 277 277 277 F F F
179432024-02-03T17:17:32.102Z 12 240-259 162 162 162 280 280 280 F F F
179442024-02-03T17:17:32.102Z 13 260-279 165 165 165 282 282 282 F F F
179452024-02-03T17:17:32.102Z 14 280-299 156 156 156 271 271 271 F F F
179462024-02-03T17:17:32.102Z 15 300-319 165 165 165 282 282 282 F F F
179472024-02-03T17:17:32.102Z 16 320-339 162 162 162 280 280 280 F F F
179482024-02-03T17:17:32.102Z 17 340-359 158 158 158 275 275 275 F F F
179492024-02-03T17:17:32.103Z 18 360-379 165 165 165 282 282 282 F F F
179502024-02-03T17:17:32.103Z 19 380-399 165 165 165 282 282 282 F F F
179512024-02-03T17:17:32.103Z 20 400-419 165 165 165 282 282 282 F F F
179522024-02-03T17:17:32.103Z 21 420-439 161 161 161 279 279 279 F F F
179532024-02-03T17:17:32.103Z 22 440-459 161 161 161 279 279 279 F F F
179542024-02-03T17:17:32.103Z 23 460-479 162 162 162 280 280 280 F F F
179552024-02-03T17:17:32.103Z 24 480-499 165 165 165 282 282 282 F F F
179562024-02-03T17:17:32.103Z 25 500-519 165 165 165 282 282 282 F F F
179572024-02-03T17:17:32.103Z 26 520-539 162 162 162 280 280 280 F F F
179582024-02-03T17:17:32.103Z 27 540-559 162 162 162 280 280 280 F F F
179592024-02-03T17:17:32.103Z 28 560-579 165 165 165 282 282 282 F F F
179602024-02-03T17:17:32.103Z 29 580-599 165 165 165 282 282 282 F F F
179612024-02-03T17:17:32.104ZMax gen: 165, Max flush: 282
179622024-02-03T17:17:32.104Z
179632024-02-03T17:17:32.104ZBegin loop 83
179642024-02-03T17:17:32.104ZDownstairs to restart: 1
179652024-02-03T17:17:32.104Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.759506467Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799}
179662024-02-03T17:17:32.104Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.760010475Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799}
179672024-02-03T17:17:32.104Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.760039Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799}
179682024-02-03T17:17:32.104Z{"msg":"Crucible e9583df6-867a-486f-b8d6-eeab8c2e87f5 has session id: 4049b976-2cea-4841-ad2a-982aece332cd","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.760060843Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
179692024-02-03T17:17:32.104Z{"msg":"Upstairs opts: Upstairs UUID: e9583df6-867a-486f-b8d6-eeab8c2e87f5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":Crucible runtime is spawned
179702024-02-03T17:17:32.104Z0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.76012474Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
179712024-02-03T17:17:32.104Z{"msg":"Crucible stats registered with UUID: e9583df6-867a-486f-b8d6-eeab8c2e87f5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.760167056Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
179722024-02-03T17:17:32.104Z{"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.760222403Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799}
179732024-02-03T17:17:32.105Z{"msg":"e9583df6-867a-486f-b8d6-eeab8c2e87f5 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.760436075Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
179742024-02-03T17:17:32.105Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.760465229Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
179752024-02-03T17:17:32.105Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.760489616Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
179762024-02-03T17:17:32.105Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.760511901Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
179772024-02-03T17:17:32.105Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.76053491Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
179782024-02-03T17:17:32.105Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.760557147Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
179792024-02-03T17:17:32.105Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.760579524Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
179802024-02-03T17:17:32.105Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.760601867Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"":"io task","client":"0","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
179812024-02-03T17:17:32.105Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.760649546Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"":"io task","client":"1","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
179822024-02-03T17:17:32.105Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.760771549Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"":"io task","client":"2","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
179832024-02-03T17:17:32.105Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.760908313Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"":"io task","client":"1","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
179842024-02-03T17:17:32.106Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.760960249Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"":"io task","client":"0","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
179852024-02-03T17:17:32.106Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.760999311Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"":"io task","client":"2","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
179862024-02-03T17:17:32.106Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.762500564Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
179872024-02-03T17:17:32.106Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.762557776Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
179882024-02-03T17:17:32.106Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.762600695Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
179892024-02-03T17:17:32.106Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.762703769Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
179902024-02-03T17:17:32.106Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.763036862Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
179912024-02-03T17:17:32.106Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.76309407Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
179922024-02-03T17:17:32.106Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.763904104Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
179932024-02-03T17:17:32.106Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.764210764Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
179942024-02-03T17:17:32.106Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.764448593Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
179952024-02-03T17:17:32.106Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.764864629Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
179962024-02-03T17:17:32.107Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.764904638Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
179972024-02-03T17:17:32.107Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:21.764928746Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
179982024-02-03T17:17:32.107Z{Wait for a query_work_queue command to finish before sending IO
179992024-02-03T17:17:32.107Z"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:31.35544955Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
180002024-02-03T17:17:32.107Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:31.355511963Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
180012024-02-03T17:17:32.107Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:31.355536779Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
180022024-02-03T17:17:32.107Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:31.355557761Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
180032024-02-03T17:17:32.107Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:31.355581465Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
180042024-02-03T17:17:32.107Z{"msg":"[0]R flush_numbers[0..12]: [277, 277, 280, 280, 282, 282, 282, 282, 280, 280, 281, 277]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:31.355614977Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
180052024-02-03T17:17:32.107Z{"msg":"[0]R generation[0..12]: [160, 160, 162, 162, 165, 165, 165, 165, 162, 162, 164, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:31.355638556Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
180062024-02-03T17:17:32.108Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:31.35566242Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
180072024-02-03T17:17:32.108Z{"msg":"[1]R flush_numbers[0..12]: [277, 277, 280, 280, 282, 282, 282, 282, 280, 280, 281, 277]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:31.35568632Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
180082024-02-03T17:17:32.108Z{"msg":"[1]R generation[0..12]: [160, 160, 162, 162, 165, 165, 165, 165, 162, 162, 164, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:31.355718571Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
180092024-02-03T17:17:32.108Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:31.355748451Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
180102024-02-03T17:17:32.108Z{"msg":"[2]R flush_numbers[0..12]: [277, 277, 280, 280, 282, 282, 282, 282, 280, 280, 281, 277]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:31.355773041Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
180112024-02-03T17:17:32.108Z{"msg":"[2]R generation[0..12]: [160, 160, 162, 162, 165, 165, 165, 165, 162, 162, 164, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:31.35582701Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
180122024-02-03T17:17:32.108Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:31.35585413Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
180132024-02-03T17:17:32.108Z{"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:31.355877912Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
180142024-02-03T17:17:32.108Z{"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:31.355928406Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
180152024-02-03T17:17:32.108Z{"msg":"Next flush: 283","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:31.355953748Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
180162024-02-03T17:17:32.108Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:31.356002859Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
180172024-02-03T17:17:32.108Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:31.356057766Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
180182024-02-03T17:17:32.109Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:31.35608026Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
180192024-02-03T17:17:32.131Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:31.35613354Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
180202024-02-03T17:17:32.131Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:31.356159427Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
180212024-02-03T17:17:32.131Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:31.356209687Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
180222024-02-03T17:17:32.132Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:31.356261996Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
180232024-02-03T17:17:32.132Z{"msg":"e9583df6-867a-486f-b8d6-eeab8c2e87f5 is now active with session: 4049b976-2cea-4841-ad2a-982aece332cd","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:31.356286957Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
180242024-02-03T17:17:32.132Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
180252024-02-03T17:17:32.132Z2024-02-03T17:17:31.356336355Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799,"session_id":"4049b976-2cea-4841-ad2a-982aece332cd"}
180262024-02-03T17:17:32.132Z{"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:31.356366957Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799}
180272024-02-03T17:17:32.132ZLoading write count information from file "/tmp/test_repair_verify.data"
180282024-02-03T17:17:32.132ZRun Repair workload
180292024-02-03T17:17:32.132Z01/30 Flush
180302024-02-03T17:17:32.132Z02/30 Write block 394 len 20480 data: 16 14 14 12 14
180312024-02-03T17:17:32.132Z03/30 Write block 484 len 4096 data: 10
180322024-02-03T17:17:32.132Z04/30 Write block 234 len 28672 data: 15 14 12 11 10 11 8
180332024-02-03T17:17:32.132Z05/30 Write block 52 len 4096 data: 16
180342024-02-03T17:17:32.132Z06/30 Read block 418 len 28672
180352024-02-03T17:17:32.133Z07/30 Read block 328 len 32768
180362024-02-03T17:17:32.133Z08/30 Flush
180372024-02-03T17:17:32.133Z09/30 Write block 523 len 16384 data: 12 11 13 12
180382024-02-03T17:17:32.133Z10/30 Read block 240 len 32768
180392024-02-03T17:17:32.133Z11/30 Write block 412 len 40960 data: 15 13 14 14 13 14 13 12 12 14
180402024-02-03T17:17:32.133Z12/30 Read block 489 len 24576
180412024-02-03T17:17:32.133Z13/30 Read block 355 len 16384
180422024-02-03T17:17:32.133Z14/30 Flush
180432024-02-03T17:17:32.133Z15/30 Write block 232 len 40960 data: 13 13 16 15 13 12 11 12 9 10
180442024-02-03T17:17:32.133Z16/30 Write block 422 len 40960 data: 12 9 9 9 10 13 14 17 17 14
180452024-02-03T17:17:32.133Z17/30 Read block 7 len 36864
180462024-02-03T17:17:32.133Z18/30 Read block 297 len 40960
180472024-02-03T17:17:32.133Z19/30 Write block 266 len 12288 data: 16 16 20
180482024-02-03T17:17:32.134Z20/30 Read block 273 len 12288
180492024-02-03T17:17:32.134Z21/30 Read block 179 len 28672
180502024-02-03T17:17:32.134Z22/30 Read block 0 len 36864
180512024-02-03T17:17:32.134Z23/30 Read block 525 len 4096
180522024-02-03T17:17:32.134Z24/30 Write block 472 len 12288 data: 10 7 8
180532024-02-03T17:17:32.134Z25/30 Read block 20 len 20480
180542024-02-03T17:17:32.134Z26/30 Flush
180552024-02-03T17:17:32.134Z27/30 Write block 308 len 36864 data: 12 14 16 13 15 14 15 17 17
180562024-02-03T17:17:32.134Z28/30 Write block 80 len 20480 data: 13 17 16 13 12
180572024-02-03T17:17:32.134Z29/30 Write block 516 len 20480 data: 15 12 9 9 10
180582024-02-03T17:17:32.134Z30/30 Write block 304 len 8192 data: 12 12
180592024-02-03T17:17:32.134Z----------------------------------------------------------------
180602024-02-03T17:17:32.134Z Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:29
180612024-02-03T17:17:32.135ZStates: Active Active Active
180622024-02-03T17:17:32.135ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
180632024-02-03T17:17:32.135Z 2 Acked 1001 Write 5 Done Sent Done false
180642024-02-03T17:17:32.135Z 3 Acked 1002 Write 1 Done Sent Done false
180652024-02-03T17:17:32.135Z 4 Acked 1003 Write 7 Done Sent Done false
180662024-02-03T17:17:32.135Z 5 Acked 1004 Write 1 Done Sent Done false
180672024-02-03T17:17:32.135Z 6 Acked 1005 Read 7 Done Sent Done false
180682024-02-03T17:17:32.135Z 7 Acked 1006 Read 8 Done Sent Done false
180692024-02-03T17:17:32.135Z 8 Acked 1007 Flush 0 Done Sent Done false
180702024-02-03T17:17:32.135Z 9 Acked 1008 Write 4 Done Sent Done false
180712024-02-03T17:17:32.135Z 10 Acked 1009 Read 8 Done Sent Done false
180722024-02-03T17:17:32.135Z 11 Acked 1010 Write 10 Done Sent Done false
180732024-02-03T17:17:32.135Z 12 Acked 1011 Read 6 Done Sent Done false
180742024-02-03T17:17:32.136Z 13 Acked 1012 Read 4 Done Sent Done false
180752024-02-03T17:17:32.136Z 14 Acked 1013 Flush 0 Done Sent Done false
180762024-02-03T17:17:32.136Z 15 Acked 1014 Write 10 Done Sent Done false
180772024-02-03T17:17:32.136Z 16 Acked 1015 Write 10 Done Sent Done false
180782024-02-03T17:17:32.136Z 17 Acked 1016 Read 9 Done Sent Done false
180792024-02-03T17:17:32.136Z 18 Acked 1017 Read 10 Done Sent Done false
180802024-02-03T17:17:32.136Z 19 Acked 1018 Write 3 Done Sent Done false
180812024-02-03T17:17:32.136Z 20 Acked 1019 Read 3 Done Sent Done false
180822024-02-03T17:17:32.136Z 21 Acked 1020 Read 7 Done Sent Done false
180832024-02-03T17:17:32.136Z 22 Acked 1021 Read 9 Done Sent Done false
180842024-02-03T17:17:32.136Z 23 Acked 1022 Read 1 Done Sent Done false
180852024-02-03T17:17:32.136ZError: Difference in extent metadata found!
180862024-02-03T17:17:32.136Z 24 Acked 1023 Write 3 Done Sent Done false
180872024-02-03T17:17:32.137Z 25 Acked 1024 Read 5 Done Sent Done false
180882024-02-03T17:17:32.137Z 26 Acked 1025 Flush 0 Done Sent Done false
180892024-02-03T17:17:32.137Z 27 Acked 1026 Write 9 Done Sent Done false
180902024-02-03T17:17:32.137Z 28 Acked 1027 Write 5 Done Sent Done false
180912024-02-03T17:17:32.137Z 29 Acked 1028 Write 5 Done Sent Sent false
180922024-02-03T17:17:32.137Z 30 Acked 1029 Write 2 Sent Sent Sent false
180932024-02-03T17:17:32.137Z STATES DS:0 DS:1 DS:2 TOTAL
180942024-02-03T17:17:32.137Z New 0 0 0 0
180952024-02-03T17:17:32.137Z Sent 1 29 2 32
180962024-02-03T17:17:32.137Z Done 28 0 27 55
180972024-02-03T17:17:32.137Z Skipped 0 0 0 0
180982024-02-03T17:17:32.137Z Error 0 0 0 0
180992024-02-03T17:17:32.138ZLast Flush: 1025 1000 1025
181002024-02-03T17:17:32.138ZDownstairs last five completed: 1000
181012024-02-03T17:17:32.138ZUpstairs last five completed: 30 29 28 27 26
181022024-02-03T17:17:32.138Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:17:31.407145908Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1799}
181032024-02-03T17:17:32.138ZWrote out file "/tmp/test_repair_verify.data"
181042024-02-03T17:17:32.138Z
181052024-02-03T17:17:32.138ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
181062024-02-03T17:17:32.138ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
181072024-02-03T17:17:32.138Z 0 000-019 160 160 160 277 277 277 F F F
181082024-02-03T17:17:32.138Z 1 020-039 160 160 160 277 277 277 F F F
181092024-02-03T17:17:32.138Z 2 040-059 166 162 166 284 280 284 F F F <---
181102024-02-03T17:17:32.138Z 3 060-079 162 162 162 280 280 280 F F F
181112024-02-03T17:17:32.138Z 4 080-099 165 165 165 282 282 282 T F T <---
181122024-02-03T17:17:32.139Z 5 100-119 165 165 165 282 282 282 F F F
181132024-02-03T17:17:32.139Z 6 120-139 165 165 165 282 282 282 F F F
181142024-02-03T17:17:32.139Z 7 140-159 165 165 165 282 282 282 F F F
181152024-02-03T17:17:32.139Z 8 160-179 162 162 162 280 280 280 F F F
181162024-02-03T17:17:32.139Z 9 180-199 162 162 162 280 280 280 F F F
181172024-02-03T17:17:32.139Z 10 200-219 164 164 164 281 281 281 F F F
181182024-02-03T17:17:32.139Z 11 220-239 166 160 166 286 277 286 F F F <---
181192024-02-03T17:17:32.150Z 12 240-259 166 162 166 286 280 286 F F F <---
181202024-02-03T17:17:32.150Z 13 260-279 166 165 166 286 282 286 F F F <---
181212024-02-03T17:17:32.150Z 14 280-299 156 156 156 271 271 271 F F F
181222024-02-03T17:17:32.150Z 15 300-319 165 165 165 282 282 282 T F T <---
181232024-02-03T17:17:32.150Z 16 320-339 162 162 162 280 280 280 F F F
181242024-02-03T17:17:32.151Z 17 340-359 158 158 158 275 275 275 F F F
181252024-02-03T17:17:32.151Z 18 360-379 165 165 165 282 282 282 F F F
181262024-02-03T17:17:32.151Z 19 380-399 166 165 166 284 282 284 F F F <---
181272024-02-03T17:17:32.151Z 20 400-419 166 165 166 285 282 285 F F F <---
181282024-02-03T17:17:32.151Z 21 420-439 166 161 166 286 279 286 F F F <---
181292024-02-03T17:17:32.151Z 22 440-459 161 161 161 279 279 279 F F F
181302024-02-03T17:17:32.151Z 23 460-479 166 162 166 286 280 286 F F F <---
181312024-02-03T17:17:32.151Z 24 480-499 166 165 166 284 282 284 F F F <---
181322024-02-03T17:17:32.151Z 25 500-519 165 165 165 282 282 282 T F T <---
181332024-02-03T17:17:32.151Z 26 520-539 166 162 166 285 280 285 T F T <---
181342024-02-03T17:17:32.151Z 27 540-559 162 162 162 280 280 280 F F F
181352024-02-03T17:17:32.151Z 28 560-579 165 165 165 282 282 282 F F F
181362024-02-03T17:17:32.151Z 29 580-599 165 165 165 282 282 282 F F F
181372024-02-03T17:17:32.151ZMax gen: 166, Max flush: 286
181382024-02-03T17:17:32.151ZA Difference in extent metadata is expected here
181392024-02-03T17:17:32.152ZOn loop 83
181402024-02-03T17:17:32.152Z
181412024-02-03T17:17:32.152ZVerifying data now
181422024-02-03T17:17:37.016ZNo change after verify
181432024-02-03T17:17:37.016ZLoop: 83 Downstairs dump after verify (and repair):
181442024-02-03T17:17:37.040ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
181452024-02-03T17:17:37.040Z 0 000-019 160 160 160 277 277 277 F F F
181462024-02-03T17:17:37.040Z 1 020-039 160 160 160 277 277 277 F F F
181472024-02-03T17:17:37.040Z 2 040-059 166 166 166 284 284 284 F F F
181482024-02-03T17:17:37.040Z 3 060-079 162 162 162 280 280 280 F F F
181492024-02-03T17:17:37.043Z 4 080-099 167 167 167 287 287 287 F F F
181502024-02-03T17:17:37.043Z 5 100-119 165 165 165 282 282 282 F F F
181512024-02-03T17:17:37.043Z 6 120-139 165 165 165 282 282 282 F F F
181522024-02-03T17:17:37.043Z 7 140-159 165 165 165 282 282 282 F F F
181532024-02-03T17:17:37.043Z 8 160-179 162 162 162 280 280 280 F F F
181542024-02-03T17:17:37.043Z 9 180-199 162 162 162 280 280 280 F F F
181552024-02-03T17:17:37.043Z 10 200-219 164 164 164 281 281 281 F F F
181562024-02-03T17:17:37.044Z 11 220-239 166 166 166 286 286 286 F F F
181572024-02-03T17:17:37.044Z 12 240-259 166 166 166 286 286 286 F F F
181582024-02-03T17:17:37.044Z 13 260-279 166 166 166 286 286 286 F F F
181592024-02-03T17:17:37.044Z 14 280-299 156 156 156 271 271 271 F F F
181602024-02-03T17:17:37.044Z 15 300-319 167 167 167 287 287 287 F F F
181612024-02-03T17:17:37.044Z 16 320-339 162 162 162 280 280 280 F F F
181622024-02-03T17:17:37.044Z 17 340-359 158 158 158 275 275 275 F F F
181632024-02-03T17:17:37.044Z 18 360-379 165 165 165 282 282 282 F F F
181642024-02-03T17:17:37.044Z 19 380-399 166 166 166 284 284 284 F F F
181652024-02-03T17:17:37.044Z 20 400-419 166 166 166 285 285 285 F F F
181662024-02-03T17:17:37.044Z 21 420-439 166 166 166 286 286 286 F F F
181672024-02-03T17:17:37.044Z 22 440-459 161 161 161 279 279 279 F F F
181682024-02-03T17:17:37.044Z 23 460-479 166 166 166 286 286 286 F F F
181692024-02-03T17:17:37.044Z 24 480-499 166 166 166 284 284 284 F F F
181702024-02-03T17:17:37.044Z 25 500-519 167 167 167 287 287 287 F F F
181712024-02-03T17:17:37.044Z 26 520-539 167 167 167 287 287 287 F F F
181722024-02-03T17:17:37.044Z 27 540-559 162 162 162 280 280 280 F F F
181732024-02-03T17:17:37.044Z 28 560-579 165 165 165 282 282 282 F F F
181742024-02-03T17:17:37.044Z 29 580-599 165 165 165 282 282 282 F F F
181752024-02-03T17:17:37.044ZMax gen: 167, Max flush: 287
181762024-02-03T17:17:37.048Z
181772024-02-03T17:17:37.048ZBegin loop 84
181782024-02-03T17:17:37.048ZDownstairs to restart: 2
181792024-02-03T17:17:37.077Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.654113591Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807}
181802024-02-03T17:17:37.077Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.65465735Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807}
181812024-02-03T17:17:37.081Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.654802957Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807}
181822024-02-03T17:17:37.081Z{"msg":"Crucible 9cf71b23-784b-4a6e-bb55-5dc49f40a6c8 has session id: 48386b17-3b75-4533-ab14-f0ca85e54f93","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.654872406Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
181832024-02-03T17:17:37.081Z{"msg":"Upstairs opts: Upstairs UUID: 9cf71b23-784b-4a6e-bb55-5dc49f40a6c8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.654923887Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
181842024-02-03T17:17:37.081Z{"msg":"Crucible stats registered with UUID: 9cf71b23-784b-4a6e-bb55-5dc49f40a6c8","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.654958227Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
181852024-02-03T17:17:37.081ZCrucible runtime is spawned
181862024-02-03T17:17:37.081Z{"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.655199424Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807}
181872024-02-03T17:17:37.081Z{"msg":"9cf71b23-784b-4a6e-bb55-5dc49f40a6c8 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.655343561Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
181882024-02-03T17:17:37.081Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.655403548Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
181892024-02-03T17:17:37.082Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.655430397Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
181902024-02-03T17:17:37.082Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.655460077Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
181912024-02-03T17:17:37.082Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.655525047Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
181922024-02-03T17:17:37.082Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.655636142Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
181932024-02-03T17:17:37.082Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.655724983Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
181942024-02-03T17:17:37.082Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.655993398Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"":"io task","client":"2","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
181952024-02-03T17:17:37.082Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.656125936Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"":"io task","client":"0","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
181962024-02-03T17:17:37.082Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.656161715Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"":"io task","client":"1","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
181972024-02-03T17:17:37.082Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.656233102Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"":"io task","client":"2","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
181982024-02-03T17:17:37.082Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.656261794Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"":"io task","client":"1","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
181992024-02-03T17:17:37.082Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.656349731Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"":"io task","client":"0","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182002024-02-03T17:17:37.082Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.657229131Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182012024-02-03T17:17:37.082Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.657322604Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182022024-02-03T17:17:37.082Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.657483761Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182032024-02-03T17:17:37.082Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.657592069Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182042024-02-03T17:17:37.082Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.657901205Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182052024-02-03T17:17:37.082Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.657996809Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182062024-02-03T17:17:37.082Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.658199969Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182072024-02-03T17:17:37.087Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.658577218Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182082024-02-03T17:17:37.087Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.65940475Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182092024-02-03T17:17:37.087Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.65956993Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182102024-02-03T17:17:37.087Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.659619738Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182112024-02-03T17:17:37.087Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.659729491Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182122024-02-03T17:17:37.087Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.65980948Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182132024-02-03T17:17:37.087Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.659840715Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182142024-02-03T17:17:37.087Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.659890787Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182152024-02-03T17:17:37.087Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.660322378Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182162024-02-03T17:17:37.087Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.660443298Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182172024-02-03T17:17:37.088Z{"msg":"[0]R flush_numbers[0..12]: [277, 277, 284, 280, 287, 282, 282, 282, 280, 280, 281, 286]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.660491915Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182182024-02-03T17:17:37.088Z{"msg":"[0]R generation[0..12]: [160, 160, 166, 162, 167, 165, 165, 165, 162, 162, 164, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.660558417Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182192024-02-03T17:17:37.088Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.660602773Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182202024-02-03T17:17:37.088Z{"msg":"[1]R flush_numbers[0..12]: [277, 277, 284, 280, 287, 282, 282, 282, 280, 280, 281, 286]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.660644348Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182212024-02-03T17:17:37.088ZWait for a query_work_queue command to finish before sending IO
182222024-02-03T17:17:37.088Z{"msg":"[1]R generation[0..12]: [160, 160, 166, 162, 167, 165, 165, 165, 162, 162, 164, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.660712421Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182232024-02-03T17:17:37.088Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.66075897Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182242024-02-03T17:17:37.088Z{"msg":"[2]R flush_numbers[0..12]: [277, 277, 284, 280, 287, 282, 282, 282, 280, 280, 281, 286]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.660827457Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182252024-02-03T17:17:37.088Z{"msg":"[2]R generation[0..12]: [160, 160, 166, 162, 167, 165, 165, 165, 162, 162, 164, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.660871973Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182262024-02-03T17:17:37.088Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.660936195Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182272024-02-03T17:17:37.088Z{"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.660978553Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182282024-02-03T17:17:37.088Z{"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.661047184Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182292024-02-03T17:17:37.088Z{"msg":"Next flush: 288","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.66107375Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182302024-02-03T17:17:37.088Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.661098139Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182312024-02-03T17:17:37.088Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.661177214Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182322024-02-03T17:17:37.088Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.661216793Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182332024-02-03T17:17:37.088Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.661281646Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182342024-02-03T17:17:37.088Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.661324428Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182352024-02-03T17:17:37.088Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.661388269Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182362024-02-03T17:17:37.088Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.661428346Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"session_id":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
182372024-02-03T17:17:37.088Z48386b17-3b75-4533-ab14-f0ca85e54f93"}
182382024-02-03T17:17:37.088Z{"msg":"9cf71b23-784b-4a6e-bb55-5dc49f40a6c8 is now active with session: 48386b17-3b75-4533-ab14-f0ca85e54f93","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.661576397Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182392024-02-03T17:17:37.088Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.661619433Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807,"session_id":"48386b17-3b75-4533-ab14-f0ca85e54f93"}
182402024-02-03T17:17:37.088Z{"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:36.661679117Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807}
182412024-02-03T17:17:37.089ZLoading write count information from file "/tmp/test_repair_verify.data"
182422024-02-03T17:17:37.089ZRun Repair workload
182432024-02-03T17:17:37.089Z01/30 Flush
182442024-02-03T17:17:37.092Z02/30 Write block 365 len 12288 data: 11 10 12
182452024-02-03T17:17:37.092Z03/30 Read block 263 len 20480
182462024-02-03T17:17:37.092Z04/30 Write block 535 len 20480 data: 9 6 8 9 8
182472024-02-03T17:17:37.092Z05/30 Read block 485 len 36864
182482024-02-03T17:17:37.095Z06/30 Read block 388 len 24576
182492024-02-03T17:17:37.098Z07/30 Write block 440 len 16384 data: 18 18 20 17
182502024-02-03T17:17:37.098Z08/30 Read block 504 len 36864
182512024-02-03T17:17:37.101Z09/30 Read block 485 len 12288
182522024-02-03T17:17:37.101Z10/30 Write block 533 len 28672 data: 10 8 10 7 9 10 9
182532024-02-03T17:17:37.103Z11/30 Read block 489 len 40960
182542024-02-03T17:17:37.104Z12/30 Read block 149 len 20480
182552024-02-03T17:17:37.108Z13/30 Flush
182562024-02-03T17:17:37.111Z14/30 Write block 127 len 40960 data: 12 13 14 13 14 13 11 11 10 10
182572024-02-03T17:17:37.111Z15/30 Write block 373 len 12288 data: 13 13 13
182582024-02-03T17:17:37.111Z16/30 Read block 322 len 36864
182592024-02-03T17:17:37.114Z17/30 Read block 254 len 4096
182602024-02-03T17:17:37.114Z18/30 Write block 216 len 20480 data: 10 9 9 8 7
182612024-02-03T17:17:37.117Z19/30 Read block 193 len 28672
182622024-02-03T17:17:37.117Z20/30 Read block 327 len 16384
182632024-02-03T17:17:37.120Z21/30 Read block 72 len 36864
182642024-02-03T17:17:37.120Z22/30 Flush
182652024-02-03T17:17:37.123Z23/30 Write block 373 len 8192 data: 14 14
182662024-02-03T17:17:37.127Z24/30 Read block 474 len 20480
182672024-02-03T17:17:37.127Z25/30 Write block 554 len 4096 data: 16
182682024-02-03T17:17:37.127Z26/30 Flush
182692024-02-03T17:17:37.130Z27/30 Write block 115 len 40960 data: 12 13 17 17 17 15 14 14 14 12
182702024-02-03T17:17:37.135Z28/30 Write block 493 len 12288 data: 8 9 10
182712024-02-03T17:17:37.135Z29/30 Read block 450 len 32768
182722024-02-03T17:17:37.139Z30/30 Read block 195 len 4096
182732024-02-03T17:17:37.139Z----------------------------------------------------------------
182742024-02-03T17:17:37.139Z Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:29
182752024-02-03T17:17:37.139ZStates: Active Active Active
182762024-02-03T17:17:37.139ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
182772024-02-03T17:17:37.139Z 2 Acked 1001 Write 3 Done Done Done false
182782024-02-03T17:17:37.139Z 3 Acked 1002 Read 5 Done Done Done false
182792024-02-03T17:17:37.139Z 4 Acked 1003 Write 5 Done Done Sent false
182802024-02-03T17:17:37.139Z 5 Acked 1004 Read 9 Done Done Sent false
182812024-02-03T17:17:37.139Z 6 Acked 1005 Read 6 Done Done Sent false
182822024-02-03T17:17:37.139Z 7 Acked 1006 Write 4 Done Done Sent false
182832024-02-03T17:17:37.139Z 8 Acked 1007 Read 9 Done Done Sent false
182842024-02-03T17:17:37.139Z 9 Acked 1008 Read 3 Done Done Sent false
182852024-02-03T17:17:37.139Z 10 Acked 1009 Write 7 Done Done Sent false
182862024-02-03T17:17:37.139Z 11 Acked 1010 Read 10 Done Done Sent false
182872024-02-03T17:17:37.139Z 12 Acked 1011 Read 5 Done Done Sent false
182882024-02-03T17:17:37.139Z 13 Acked 1012 Flush 0 Done Done Sent false
182892024-02-03T17:17:37.139Z 14 Acked 1013 Write 10 Done Done Sent false
182902024-02-03T17:17:37.139Z 15 Acked 1014 Write 3 Done Done Sent false
182912024-02-03T17:17:37.139Z 16 Acked 1015 Read 9 Done Done Sent false
182922024-02-03T17:17:37.139Z 17 Acked 1016 Read 1 Done Done Sent false
182932024-02-03T17:17:37.139Z 18 Acked 1017 Write 5 Done Done Sent false
182942024-02-03T17:17:37.140Z 19 Acked 1018 Read 7 Done Done Sent false
182952024-02-03T17:17:37.140Z 20 Acked 1019 Read 4 Done Done Sent false
182962024-02-03T17:17:37.140Z 21 Acked 1020 Read 9 Done Done Sent false
182972024-02-03T17:17:37.140Z 22 Acked 1021 Flush 0 Done Done Sent false
182982024-02-03T17:17:37.140Z 23 Acked 1022 Write 2 Done Done Sent false
182992024-02-03T17:17:37.140Z 24 Acked 1023 Read 5 Done Done Sent false
183002024-02-03T17:17:37.140Z 25 Acked 1024 Write 1 Done Done Sent false
183012024-02-03T17:17:37.140Z 26 Acked 1025 Flush 0 Done Done Sent false
183022024-02-03T17:17:37.140Z 27 Acked 1026 Write 10 Done Done Sent false
183032024-02-03T17:17:37.140Z 28 Acked 1027 Write 3 Done Done Sent false
183042024-02-03T17:17:37.140Z 29 Acked 1028 Read 8 Done Done Sent false
183052024-02-03T17:17:37.140Z 30 Acked 1029 Read 1 Done Done Sent false
183062024-02-03T17:17:37.140Z STATES DS:0 DS:1 DS:2 TOTAL
183072024-02-03T17:17:37.140Z New 0 0 0 0
183082024-02-03T17:17:37.140Z Sent 0 0 27 27
183092024-02-03T17:17:37.140Z Done 29 29 2 60
183102024-02-03T17:17:37.140Z Skipped 0 0 0 0
183112024-02-03T17:17:37.140Z Error 0 0 0 0
183122024-02-03T17:17:37.140ZLast Flush: 1025 1025 1000
183132024-02-03T17:17:37.140ZDownstairs last five completed: 1000
183142024-02-03T17:17:37.140ZUpstairs last five completed: 30 29 28 27 26
183152024-02-03T17:17:37.140Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:17:36.712639224Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1807}
183162024-02-03T17:17:37.140ZWrote out file "/tmp/test_repair_verify.data"
183172024-02-03T17:17:37.144Z
183182024-02-03T17:17:37.148ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
183192024-02-03T17:17:37.173ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
183202024-02-03T17:17:37.173Z 0 000-019 160 160 160 277 277 277 F F F
183212024-02-03T17:17:37.173Z 1 020-039 160 160 160 277 277 277 F F F
183222024-02-03T17:17:37.173Z 2 040-059 166 166 166 284 284 284 F F F
183232024-02-03T17:17:37.173Z 3 060-079 162 162 162 280 280 280 F F F
183242024-02-03T17:17:37.180Z 4 080-099 167 167 167 287 287 287 F F F
183252024-02-03T17:17:37.180Z 5 100-119 165 165 165 282 282 282 T T F <---
183262024-02-03T17:17:37.180Z 6 120-139 168 168 165 290 290 282 T T F <---
183272024-02-03T17:17:37.181Z 7 140-159 165 165 165 282 282 282 F F F
183282024-02-03T17:17:37.181Z 8 160-179 162 162 162 280 280 280 F F F
183292024-02-03T17:17:37.181Z 9 180-199 162 162 162 280 280 280 F F F
183302024-02-03T17:17:37.181Z 10 200-219 168 168 164 290 290 281 F F F <---
183312024-02-03T17:17:37.181Z 11 220-239 168 168 166 290 290 286 F F F <---
183322024-02-03T17:17:37.181Z 12 240-259 166 166 166 286 286 286 F F F
183332024-02-03T17:17:37.181Z 13 260-279 166 166 166 286 286 286 F F F
183342024-02-03T17:17:37.181Z 14 280-299 156 156 156 271 271 271 F F F
183352024-02-03T17:17:37.181Z 15 300-319 167 167 167 287 287 287 F F F
183362024-02-03T17:17:37.181Z 16 320-339 162 162 162 280 280 280 F F F
183372024-02-03T17:17:37.181Z 17 340-359 158 158 158 275 275 275 F F F
183382024-02-03T17:17:37.181Z 18 360-379 168 168 165 291 291 282 F F T <---
183392024-02-03T17:17:37.181Z 19 380-399 166 166 166 284 284 284 F F F
183402024-02-03T17:17:37.181Z 20 400-419 166 166 166 285 285 285 F F F
183412024-02-03T17:17:37.181Z 21 420-439 166 166 166 286 286 286 F F F
183422024-02-03T17:17:37.181Z 22 440-459 168 168 161 289 289 279 F F F <---
183432024-02-03T17:17:37.181Z 23 460-479 166 166 166 286 286 286 F F F
183442024-02-03T17:17:37.181Z 24 480-499 166 166 166 284 284 284 T T F <---
183452024-02-03T17:17:37.181Z 25 500-519 167 167 167 287 287 287 F F F
183462024-02-03T17:17:37.181Z 26 520-539 168 168 167 289 289 287 F F F <---
183472024-02-03T17:17:37.181Z 27 540-559 168 168 162 291 291 280 F F F <---
183482024-02-03T17:17:37.181Z 28 560-579 165 165 165 282 282 282 F F F
183492024-02-03T17:17:37.181Z 29 580-599 165 165 165 282 282 282 F F F
183502024-02-03T17:17:37.181ZMax gen: 168, Max flush: 291
183512024-02-03T17:17:37.182ZError: Difference in extent metadata found!
183522024-02-03T17:17:37.182ZA Difference in extent metadata is expected here
183532024-02-03T17:17:37.182ZOn loop 84
183542024-02-03T17:17:37.182Z
183552024-02-03T17:17:37.188ZVerifying data now
183562024-02-03T17:17:42.020ZNo change after verify
183572024-02-03T17:17:42.020ZLoop: 84 Downstairs dump after verify (and repair):
183582024-02-03T17:17:42.043ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
183592024-02-03T17:17:42.043Z 0 000-019 160 160 160 277 277 277 F F F
183602024-02-03T17:17:42.043Z 1 020-039 160 160 160 277 277 277 F F F
183612024-02-03T17:17:42.043Z 2 040-059 166 166 166 284 284 284 F F F
183622024-02-03T17:17:42.043Z 3 060-079 162 162 162 280 280 280 F F F
183632024-02-03T17:17:42.043Z 4 080-099 167 167 167 287 287 287 F F F
183642024-02-03T17:17:42.044Z 5 100-119 169 169 169 292 292 292 F F F
183652024-02-03T17:17:42.044Z 6 120-139 169 169 169 292 292 292 F F F
183662024-02-03T17:17:42.044Z 7 140-159 165 165 165 282 282 282 F F F
183672024-02-03T17:17:42.044Z 8 160-179 162 162 162 280 280 280 F F F
183682024-02-03T17:17:42.044Z 9 180-199 162 162 162 280 280 280 F F F
183692024-02-03T17:17:42.044Z 10 200-219 168 168 168 290 290 290 F F F
183702024-02-03T17:17:42.056Z 11 220-239 168 168 168 290 290 290 F F F
183712024-02-03T17:17:42.056Z 12 240-259 166 166 166 286 286 286 F F F
183722024-02-03T17:17:42.057Z 13 260-279 166 166 166 286 286 286 F F F
183732024-02-03T17:17:42.057Z 14 280-299 156 156 156 271 271 271 F F F
183742024-02-03T17:17:42.057Z 15 300-319 167 167 167 287 287 287 F F F
183752024-02-03T17:17:42.057Z 16 320-339 162 162 162 280 280 280 F F F
183762024-02-03T17:17:42.057Z 17 340-359 158 158 158 275 275 275 F F F
183772024-02-03T17:17:42.057Z 18 360-379 168 168 168 291 291 291 F F F
183782024-02-03T17:17:42.057Z 19 380-399 166 166 166 284 284 284 F F F
183792024-02-03T17:17:42.057Z 20 400-419 166 166 166 285 285 285 F F F
183802024-02-03T17:17:42.057Z 21 420-439 166 166 166 286 286 286 F F F
183812024-02-03T17:17:42.057Z 22 440-459 168 168 168 289 289 289 F F F
183822024-02-03T17:17:42.057Z 23 460-479 166 166 166 286 286 286 F F F
183832024-02-03T17:17:42.057Z 24 480-499 169 169 169 292 292 292 F F F
183842024-02-03T17:17:42.057Z 25 500-519 167 167 167 287 287 287 F F F
183852024-02-03T17:17:42.058Z 26 520-539 168 168 168 289 289 289 F F F
183862024-02-03T17:17:42.058Z 27 540-559 168 168 168 291 291 291 F F F
183872024-02-03T17:17:42.058Z 28 560-579 165 165 165 282 282 282 F F F
183882024-02-03T17:17:42.058Z 29 580-599 165 165 165 282 282 282 F F F
183892024-02-03T17:17:42.058ZMax gen: 169, Max flush: 292
183902024-02-03T17:17:42.058Z
183912024-02-03T17:17:42.058ZBegin loop 85
183922024-02-03T17:17:42.059ZDownstairs to restart: 1
183932024-02-03T17:17:42.086Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.656319792Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815}
183942024-02-03T17:17:42.086Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.656781381Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815}
183952024-02-03T17:17:42.100Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.656852065Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815}
183962024-02-03T17:17:42.100Z{"msg":"Crucible 5961d6ac-c6e1-4a5a-8bd0-c2b5b5d252f6 has session id: 516a4b97-a9ff-49c1-becf-c6d2de712312","v":0,"name":"crucible","level":Crucible runtime is spawned
183972024-02-03T17:17:42.101Z30,"time":"2024-02-03T17:17:41.657209428Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
183982024-02-03T17:17:42.101Z{"msg":"Upstairs opts: Upstairs UUID: 5961d6ac-c6e1-4a5a-8bd0-c2b5b5d252f6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.657296761Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
183992024-02-03T17:17:42.101Z{"msg":"Crucible stats registered with UUID: 5961d6ac-c6e1-4a5a-8bd0-c2b5b5d252f6","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.65736809Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184002024-02-03T17:17:42.101Z{"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.657406883Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815}
184012024-02-03T17:17:42.101Z{"msg":"5961d6ac-c6e1-4a5a-8bd0-c2b5b5d252f6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.657442094Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184022024-02-03T17:17:42.101Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.657501876Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184032024-02-03T17:17:42.101Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.657545686Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184042024-02-03T17:17:42.101Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.65761022Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184052024-02-03T17:17:42.101Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.657716093Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184062024-02-03T17:17:42.101Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.65775776Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184072024-02-03T17:17:42.101Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.657936698Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184082024-02-03T17:17:42.101Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.658023271Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"":"io task","client":"2","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184092024-02-03T17:17:42.101Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.658171532Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"":"io task","client":"1","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184102024-02-03T17:17:42.101Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.658222839Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"":"io task","client":"0","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184112024-02-03T17:17:42.101Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.658267639Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"":"io task","client":"2","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184122024-02-03T17:17:42.101Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.658307628Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"":"io task","client":"1","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184132024-02-03T17:17:42.101Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.658352593Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"":"io task","client":"0","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184142024-02-03T17:17:42.101Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.659346524Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184152024-02-03T17:17:42.101Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.659416955Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184162024-02-03T17:17:42.101Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.65944348Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184172024-02-03T17:17:42.101Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.65961474Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184182024-02-03T17:17:42.101Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.6598702Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184192024-02-03T17:17:42.101Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.659915567Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184202024-02-03T17:17:42.102Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.66055013Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184212024-02-03T17:17:42.102Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.660649148Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184222024-02-03T17:17:42.102Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.661011314Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184232024-02-03T17:17:42.102Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.661427318Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184242024-02-03T17:17:42.102Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.661515533Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184252024-02-03T17:17:42.102Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.661542377Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184262024-02-03T17:17:42.102Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.661563101Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184272024-02-03T17:17:42.102Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.661585092Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184282024-02-03T17:17:42.102Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.661606851Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184292024-02-03T17:17:42.102Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.661995113Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184302024-02-03T17:17:42.102Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.662058947Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184312024-02-03T17:17:42.102Z{"msg":"[0]R flush_numbers[0..12]: [277, 277, 284, 280, 287, 292, 292, 282, 280, 280, 290, 290]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.662151332Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184322024-02-03T17:17:42.102Z{"msg":"[0]R generation[0..12]: [160, 160, 166, 162, 167, 169, 169, 165, 162, 162, 168, 168]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.662204961Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184332024-02-03T17:17:42.102Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.662232601Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184342024-02-03T17:17:42.102Z{"msg":"[1]R flush_numbers[0..12]: [277, 277, 284, 280, 287, 292, 292, 282, 280, 280, 290, 290]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.662283539Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184352024-02-03T17:17:42.102Z{"msg":"[1]R generation[0..12]: [160, 160, 166, 162, 167, 169, 169, 165, 162, 162, 168, 168]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.662309175Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184362024-02-03T17:17:42.102Z{"msgWait for a query_work_queue command to finish before sending IO
184372024-02-03T17:17:42.102Z":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.662338023Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184382024-02-03T17:17:42.102Z{"msg":"[2]R flush_numbers[0..12]: [277, 277, 284, 280, 287, 292, 292, 282, 280, 280, 290, 290]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.66238872Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184392024-02-03T17:17:42.102Z{"msg":"[2]R generation[0..12]: [160, 160, 166, 162, 167, 169, 169, 165, 162, 162, 168, 168]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.662414206Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184402024-02-03T17:17:42.102Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.662437897Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184412024-02-03T17:17:42.102Z{"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.662503524Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184422024-02-03T17:17:42.102Z{"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.662527876Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184432024-02-03T17:17:42.102Z{"msg":"Next flush: 293","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.662549429Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184442024-02-03T17:17:42.103Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.662602344Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184452024-02-03T17:17:42.103Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.662629852Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184462024-02-03T17:17:42.103Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.662685705Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184472024-02-03T17:17:42.103Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.662711038Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184482024-02-03T17:17:42.103Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.662732805Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184492024-02-03T17:17:42.103Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.662780051Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184502024-02-03T17:17:42.103Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.662803033Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184512024-02-03T17:17:42.103Z{"msg":"5961d6ac-c6e1-4a5a-8bd0-c2b5b5d252f6 is now active with session: 516a4b97-a9ff-49c1-becf-c6d2de712312","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.662824342Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184522024-02-03T17:17:42.103Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.662843873Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815,"session_id":"516a4b97-a9ff-49c1-becf-c6d2de712312"}
184532024-02-03T17:17:42.103Z{"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:41.662864255Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815}
184542024-02-03T17:17:42.103ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
184552024-02-03T17:17:42.103ZLoading write count information from file "/tmp/test_repair_verify.data"
184562024-02-03T17:17:42.103ZRun Repair workload
184572024-02-03T17:17:42.103Z01/30 Write block 118 len 32768 data: 18 18 16 15 15 15 13 10
184582024-02-03T17:17:42.103Z02/30 Write block 172 len 8192 data: 14 14
184592024-02-03T17:17:42.103Z03/30 Write block 587 len 4096 data: 11
184602024-02-03T17:17:42.103Z04/30 Write block 497 len 28672 data: 10 10 8 6 8 12 16
184612024-02-03T17:17:42.109Z05/30 Read block 174 len 32768
184622024-02-03T17:17:42.109Z06/30 Write block 206 len 24576 data: 13 14 15 14 16 13
184632024-02-03T17:17:42.109Z07/30 Read block 45 len 32768
184642024-02-03T17:17:42.109Z08/30 Read block 289 len 8192
184652024-02-03T17:17:42.110Z09/30 Read block 139 len 20480
184662024-02-03T17:17:42.110Z10/30 Read block 492 len 8192
184672024-02-03T17:17:42.110Z11/30 Write block 100 len 28672 data: 13 13 9 9 8 9 11
184682024-02-03T17:17:42.110Z12/30 Read block 508 len 40960
184692024-02-03T17:17:42.110Z13/30 Read block 296 len 24576
184702024-02-03T17:17:42.116Z14/30 Write block 237 len 28672 data: 13 12 13 10 11 9 9
184712024-02-03T17:17:42.116Z15/30 Read block 135 len 40960
184722024-02-03T17:17:42.116Z16/30 Read block 218 len 4096
184732024-02-03T17:17:42.116Z17/30 Write block 395 len 8192 data: 15 15
184742024-02-03T17:17:42.116Z18/30 Read block 504 len 20480
184752024-02-03T17:17:42.122Z19/30 Write block 538 len 40960 data: 11 10 8 10 13 12 10 11 12 16
184762024-02-03T17:17:42.122Z20/30 Write block 175 len 8192 data: 12 12
184772024-02-03T17:17:42.122Z21/30 Read block 172 len 4096
184782024-02-03T17:17:42.122Z22/30 Read block 13 len 8192
184792024-02-03T17:17:42.122Z23/30 Read block 73 len 16384
184802024-02-03T17:17:42.131Z24/30 Write block 301 len 32768 data: 11 11 11 13 13 13 12 13
184812024-02-03T17:17:42.131Z25/30 Read block 95 len 20480
184822024-02-03T17:17:42.131Z26/30 Read block 343 len 40960
184832024-02-03T17:17:42.141Z27/30 Write block 546 len 28672 data: 13 17 14 16 18 15 15
184842024-02-03T17:17:42.141Z28/30 Write block 169 len 12288 data: 18 19 17
184852024-02-03T17:17:42.141Z29/30 Write block 574 len 36864 data: 13 12 12 9 9 10 9 10 8
184862024-02-03T17:17:42.141Z30/30 Write block 263 len 24576 data: 20 21 20 17 17 21
184872024-02-03T17:17:42.141Z----------------------------------------------------------------
184882024-02-03T17:17:42.141Z Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30
184892024-02-03T17:17:42.141ZStates: Active Active Active
184902024-02-03T17:17:42.141ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
184912024-02-03T17:17:42.141Z 1 Acked 1000 Write 8 Done Done Done false
184922024-02-03T17:17:42.142Z 2 Acked 1001 Write 2 Done Sent Done false
184932024-02-03T17:17:42.142Z 3 Acked 1002 Write 1 Done Sent Done false
184942024-02-03T17:17:42.142Z 4 Acked 1003 Write 7 Done Sent Done false
184952024-02-03T17:17:42.142Z 5 Acked 1004 Read 8 Done Sent Done false
184962024-02-03T17:17:42.142Z 6 Acked 1005 Write 6 Done Sent Done false
184972024-02-03T17:17:42.142Z 7 Acked 1006 Read 8 Done Sent Done false
184982024-02-03T17:17:42.142Z 8 Acked 1007 Read 2 Done Sent Done false
184992024-02-03T17:17:42.142Z 9 Acked 1008 Read 5 Done Sent Done false
185002024-02-03T17:17:42.142Z 10 Acked 1009 Read 2 Done Sent Done false
185012024-02-03T17:17:42.142Z 11 Acked 1010 Write 7 Done Sent Done false
185022024-02-03T17:17:42.142Z 12 Acked 1011 Read 10 Done Sent Done false
185032024-02-03T17:17:42.142Z 13 Acked 1012 Read 6 Done Sent Done false
185042024-02-03T17:17:42.142Z 14 Acked 1013 Write 7 Done Sent Done false
185052024-02-03T17:17:42.142Z 15 Acked 1014 Read 10 Done Sent Done false
185062024-02-03T17:17:42.142Z 16 Acked 1015 Read 1 Done Sent Done false
185072024-02-03T17:17:42.142Z 17 Acked 1016 Write 2 Done Sent Done false
185082024-02-03T17:17:42.142Z 18 Acked 1017 Read 5 Done Sent Done false
185092024-02-03T17:17:42.142Z 19 Acked 1018 Write 10 Done Sent Done false
185102024-02-03T17:17:42.142Z 20 Acked 1019 Write 2 Done Sent Done false
185112024-02-03T17:17:42.142Z 21 Acked 1020 Read 1 Done Sent Done false
185122024-02-03T17:17:42.142Z 22 Acked 1021 Read 2 Done Sent Done false
185132024-02-03T17:17:42.142Z 23 Acked 1022 Read 4 Done Sent Done false
185142024-02-03T17:17:42.142Z 24 Acked 1023 Write 8 Done Sent Done false
185152024-02-03T17:17:42.142Z 25 Acked 1024 Read 5 Done Sent Done false
185162024-02-03T17:17:42.142Z 26 Acked 1025 Read 10 Done Sent Done false
185172024-02-03T17:17:42.143Z 27 Acked 1026 Write 7 Done Sent Done false
185182024-02-03T17:17:42.143Z 28 Acked 1027 Write 3 Done Sent Done false
185192024-02-03T17:17:42.143Z 29 Acked 1028 Write 9 Done Sent Done false
185202024-02-03T17:17:42.143Z 30 Acked 1029 Write 6 Sent Sent Sent false
185212024-02-03T17:17:42.143Z STATES DS:0 DS:1 DS:2 TOTAL
185222024-02-03T17:17:42.143Z New 0 0 0 0
185232024-02-03T17:17:42.143Z Sent 1 29 1 31
185242024-02-03T17:17:42.143Z Done 29 1 29 59
185252024-02-03T17:17:42.143Z Skipped 0 0 0 0
185262024-02-03T17:17:42.143Z Error 0 0 0 0
185272024-02-03T17:17:42.143ZLast Flush: 0 0 0
185282024-02-03T17:17:42.143ZDownstairs last five completed:
185292024-02-03T17:17:42.143ZUpstairs last five completed: 30 29 28 27 26
185302024-02-03T17:17:42.143Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:17:41.70954506Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1815}
185312024-02-03T17:17:42.143ZWrote out file "/tmp/test_repair_verify.data"
185322024-02-03T17:17:42.152Z
185332024-02-03T17:17:42.152ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
185342024-02-03T17:17:42.186ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
185352024-02-03T17:17:42.186Z 0 000-019 160 160 160 277 277 277 F F F
185362024-02-03T17:17:42.186Z 1 020-039 160 160 160 277 277 277 F F F
185372024-02-03T17:17:42.186Z 2 040-059 166 166 166 284 284 284 F F F
185382024-02-03T17:17:42.186Z 3 060-079 162 162 162 280 280 280 F F F
185392024-02-03T17:17:42.186Z 4 080-099 167 167 167 287 287 287 F F F
185402024-02-03T17:17:42.186Z 5 100-119 169 169 169 292 292 292 T T T
185412024-02-03T17:17:42.206Z 6 120-139 169 169 169 292 292 292 T T T
185422024-02-03T17:17:42.206Z 7 140-159 165 165 165 282 282 282 F F F
185432024-02-03T17:17:42.206Z 8 160-179 162 162 162 280 280 280 T F T <---
185442024-02-03T17:17:42.206Z 9 180-199 162 162 162 280 280 280 F F F
185452024-02-03T17:17:42.206Z 10 200-219 168 168 168 290 290 290 T F T <---
185462024-02-03T17:17:42.206Z 11 220-239 168 168 168 290 290 290 T F T <---
185472024-02-03T17:17:42.206Z 12 240-259 166 166 166 286 286 286 T F T <---
185482024-02-03T17:17:42.206Z 13 260-279 166 166 166 286 286 286 T F T <---
185492024-02-03T17:17:42.206Z 14 280-299 156 156 156 271 271 271 F F F
185502024-02-03T17:17:42.206Z 15 300-319 167 167 167 287 287 287 T F T <---
185512024-02-03T17:17:42.206Z 16 320-339 162 162 162 280 280 280 F F F
185522024-02-03T17:17:42.206Z 17 340-359 158 158 158 275 275 275 F F F
185532024-02-03T17:17:42.207Z 18 360-379 168 168 168 291 291 291 F F F
185542024-02-03T17:17:42.207Z 19 380-399 166 166 166 284 284 284 T F T <---
185552024-02-03T17:17:42.207Z 20 400-419 166 166 166 285 285 285 F F F
185562024-02-03T17:17:42.207Z 21 420-439 166 166 166 286 286 286 F F F
185572024-02-03T17:17:42.207Z 22 440-459 168 168 168 289 289 289 F F F
185582024-02-03T17:17:42.207Z 23 460-479 166 166 166 286 286 286 F F F
185592024-02-03T17:17:42.207Z 24 480-499 169 169 169 292 292 292 T F T <---
185602024-02-03T17:17:42.207Z 25 500-519 167 167 167 287 287 287 T F T <---
185612024-02-03T17:17:42.207Z 26 520-539 168 168 168 289 289 289 T F T <---
185622024-02-03T17:17:42.207Z 27 540-559 168 168 168 291 291 291 T F T <---
185632024-02-03T17:17:42.207Z 28 560-579 165 165 165 282 282 282 T F T <---
185642024-02-03T17:17:42.207Z 29 580-599 165 165 165 282 282 282 T F T <---
185652024-02-03T17:17:42.207ZMax gen: 169, Max flush: 292
185662024-02-03T17:17:42.207ZError: Difference in extent metadata found!
185672024-02-03T17:17:42.207ZA Difference in extent metadata is expected here
185682024-02-03T17:17:42.207ZOn loop 85
185692024-02-03T17:17:42.207Z
185702024-02-03T17:17:42.207ZVerifying data now
185712024-02-03T17:17:48.529ZNo change after verify
185722024-02-03T17:17:48.529ZLoop: 85 Downstairs dump after verify (and repair):
185732024-02-03T17:17:48.536ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
185742024-02-03T17:17:48.536Z 0 000-019 160 160 160 277 277 277 F F F
185752024-02-03T17:17:48.536Z 1 020-039 160 160 160 277 277 277 F F F
185762024-02-03T17:17:48.537Z 2 040-059 166 166 166 284 284 284 F F F
185772024-02-03T17:17:48.537Z 3 060-079 162 162 162 280 280 280 F F F
185782024-02-03T17:17:48.537Z 4 080-099 167 167 167 287 287 287 F F F
185792024-02-03T17:17:48.537Z 5 100-119 170 170 170 293 293 293 F F F
185802024-02-03T17:17:48.537Z 6 120-139 170 170 170 293 293 293 F F F
185812024-02-03T17:17:48.537Z 7 140-159 165 165 165 282 282 282 F F F
185822024-02-03T17:17:48.537Z 8 160-179 170 170 170 293 293 293 F F F
185832024-02-03T17:17:48.537Z 9 180-199 162 162 162 280 280 280 F F F
185842024-02-03T17:17:48.537Z 10 200-219 170 170 170 293 293 293 F F F
185852024-02-03T17:17:48.537Z 11 220-239 170 170 170 293 293 293 F F F
185862024-02-03T17:17:48.537Z 12 240-259 170 170 170 293 293 293 F F F
185872024-02-03T17:17:48.537Z 13 260-279 170 170 170 293 293 293 F F F
185882024-02-03T17:17:48.537Z 14 280-299 156 156 156 271 271 271 F F F
185892024-02-03T17:17:48.537Z 15 300-319 170 170 170 293 293 293 F F F
185902024-02-03T17:17:48.537Z 16 320-339 162 162 162 280 280 280 F F F
185912024-02-03T17:17:48.537Z 17 340-359 158 158 158 275 275 275 F F F
185922024-02-03T17:17:48.537Z 18 360-379 168 168 168 291 291 291 F F F
185932024-02-03T17:17:48.537Z 19 380-399 170 170 170 293 293 293 F F F
185942024-02-03T17:17:48.537Z 20 400-419 166 166 166 285 285 285 F F F
185952024-02-03T17:17:48.538Z 21 420-439 166 166 166 286 286 286 F F F
185962024-02-03T17:17:48.538Z 22 440-459 168 168 168 289 289 289 F F F
185972024-02-03T17:17:48.538Z 23 460-479 166 166 166 286 286 286 F F F
185982024-02-03T17:17:48.538Z 24 480-499 170 170 170 293 293 293 F F F
185992024-02-03T17:17:48.538Z 25 500-519 170 170 170 293 293 293 F F F
186002024-02-03T17:17:48.538Z 26 520-539 170 170 170 293 293 293 F F F
186012024-02-03T17:17:48.538Z 27 540-559 170 170 170 293 293 293 F F F
186022024-02-03T17:17:48.538Z 28 560-579 170 170 170 293 293 293 F F F
186032024-02-03T17:17:48.538Z 29 580-599 170 170 170 293 293 293 F F F
186042024-02-03T17:17:48.538ZMax gen: 170, Max flush: 293
186052024-02-03T17:17:48.538Z
186062024-02-03T17:17:48.538ZBegin loop 86
186072024-02-03T17:17:48.538ZDownstairs to restart: 1
186082024-02-03T17:17:48.559Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.136618176Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823}
186092024-02-03T17:17:48.559Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.137096685Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823}
186102024-02-03T17:17:48.559Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.137153362Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823}
186112024-02-03T17:17:48.563Z{"msg":"Crucible a617a16f-20e5-4d47-af83-07fa8ed9b20b has session id: c4c26654-53c7-4385-847d-40a9ff244c94","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.137185748Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186122024-02-03T17:17:48.563Z{"msg":"Crucible runtime is spawned
186132024-02-03T17:17:48.563ZUpstairs opts: Upstairs UUID: a617a16f-20e5-4d47-af83-07fa8ed9b20b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.137366617Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186142024-02-03T17:17:48.563Z{"msg":"Crucible stats registered with UUID: a617a16f-20e5-4d47-af83-07fa8ed9b20b","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.137409925Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186152024-02-03T17:17:48.563Z{"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.137463569Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823}
186162024-02-03T17:17:48.563Z{"msg":"a617a16f-20e5-4d47-af83-07fa8ed9b20b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.137648843Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186172024-02-03T17:17:48.563Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.137707911Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186182024-02-03T17:17:48.563Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.137751784Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186192024-02-03T17:17:48.563Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.137829277Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186202024-02-03T17:17:48.563Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.137875115Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186212024-02-03T17:17:48.563Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.137920011Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186222024-02-03T17:17:48.563Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.137999411Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186232024-02-03T17:17:48.563Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.138079694Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"":"io task","client":"2","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186242024-02-03T17:17:48.563Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.138124733Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"":"io task","client":"0","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186252024-02-03T17:17:48.563Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.138169667Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"":"io task","client":"1","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186262024-02-03T17:17:48.563Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.138247353Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"":"io task","client":"0","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186272024-02-03T17:17:48.563Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.138358193Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"":"io task","client":"1","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186282024-02-03T17:17:48.564Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.138407052Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"":"io task","client":"2","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186292024-02-03T17:17:48.564Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.139541215Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186302024-02-03T17:17:48.564Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.13964916Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186312024-02-03T17:17:48.564Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.139793819Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186322024-02-03T17:17:48.564Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.139824702Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186332024-02-03T17:17:48.564Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.139953353Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186342024-02-03T17:17:48.564Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.139980623Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186352024-02-03T17:17:48.564Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.140422524Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186362024-02-03T17:17:48.569Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.140999745Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186372024-02-03T17:17:48.569Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.141129376Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186382024-02-03T17:17:48.569Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.141239452Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186392024-02-03T17:17:48.569Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.141342309Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186402024-02-03T17:17:48.569Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.141434385Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186412024-02-03T17:17:48.569Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.141869089Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186422024-02-03T17:17:48.569Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.141966673Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186432024-02-03T17:17:48.569Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.142002878Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186442024-02-03T17:17:48.569Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.142073484Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186452024-02-03T17:17:48.569Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.142118487Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186462024-02-03T17:17:48.570Z{"msg":"[0]R flush_numbers[0..12]: [277, 277, 284, 280, 287, 293, 293, 282, 293, 280, 293, 293]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.142181309Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186472024-02-03T17:17:48.570Z{"msg":"[0]R generation[0..12]: [160, 160, 166, 162, 167, 170, 170, 165, 170, 162, 170, 170]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.142209527Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186482024-02-03T17:17:48.570Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.142258892Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186492024-02-03T17:17:48.570Z{"msg":"[1]R flush_numbers[0..12]: [277, 277, 284, 280, 287, 293, 293, 282, 293, 280, 293, 293]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.142284453Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186502024-02-03T17:17:48.570Z{"msg":"[1]R generation[0..12]: [160, 160, 166, 162, 167, 170, 170, 165, 170, 162, 170, 170]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.142307772Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186512024-02-03T17:17:48.570Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
186522024-02-03T17:17:48.570Z2024-02-03T17:17:48.142374924Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186532024-02-03T17:17:48.570Z{"msg":"[2]R flush_numbers[0..12]: [277, 277, 284, 280, 287, 293, 293, 282, 293, 280, 293, 293]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.142407447Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186542024-02-03T17:17:48.570Z{"msg":"[2]R generation[0..12]: [160, 160, 166, 162, 167, 170, 170, 165, 170, 162, 170, 170]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.14245705Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186552024-02-03T17:17:48.570Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.142483654Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186562024-02-03T17:17:48.570Z{"msg":"Max found gen is 171","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.142508322Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186572024-02-03T17:17:48.570Z{"msg":"Generation requested: 172 >= found:171","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.142578759Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186582024-02-03T17:17:48.570Z{"msg":"Next flush: 294","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.142606816Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186592024-02-03T17:17:48.570Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.142628497Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186602024-02-03T17:17:48.570Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.142685704Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186612024-02-03T17:17:48.570Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.142724103Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186622024-02-03T17:17:48.570Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.142795493Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186632024-02-03T17:17:48.570Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.142825426Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186642024-02-03T17:17:48.570Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.142847691Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186652024-02-03T17:17:48.570Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.14291607Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186662024-02-03T17:17:48.570Z{"msg":"a617a16f-20e5-4d47-af83-07fa8ed9b20b is now active with session: c4c26654-53c7-4385-847d-40a9ff244c94","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.14294505Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186672024-02-03T17:17:48.570Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.142965015Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823,"session_id":"c4c26654-53c7-4385-847d-40a9ff244c94"}
186682024-02-03T17:17:48.570Z{"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:48.143033545Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823}
186692024-02-03T17:17:48.570ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
186702024-02-03T17:17:48.571ZLoading write count information from file "/tmp/test_repair_verify.data"
186712024-02-03T17:17:48.571ZRun Repair workload
186722024-02-03T17:17:48.571Z01/30 Write block 443 len 12288 data: 18 13 15
186732024-02-03T17:17:48.574Z02/30 Read block 390 len 16384
186742024-02-03T17:17:48.574Z03/30 Read block 271 len 4096
186752024-02-03T17:17:48.574Z04/30 Read block 454 len 32768
186762024-02-03T17:17:48.574Z05/30 Flush
186772024-02-03T17:17:48.577Z06/30 Write block 348 len 32768 data: 11 13 10 9 11 11 12 13
186782024-02-03T17:17:48.577Z07/30 Write block 145 len 40960 data: 11 11 8 13 14 13 15 16 14 13
186792024-02-03T17:17:48.580Z08/30 Flush
186802024-02-03T17:17:48.584Z09/30 Write block 525 len 28672 data: 14 13 13 12 12 13 11
186812024-02-03T17:17:48.584Z10/30 Read block 554 len 40960
186822024-02-03T17:17:48.587Z11/30 Read block 464 len 20480
186832024-02-03T17:17:48.590Z12/30 Read block 402 len 24576
186842024-02-03T17:17:48.590Z13/30 Read block 581 len 36864
186852024-02-03T17:17:48.592Z14/30 Read block 402 len 36864
186862024-02-03T17:17:48.593Z15/30 Write block 588 len 20480 data: 14 12 13 13 13
186872024-02-03T17:17:48.728Z16/30 Write block 555 len 8192 data: 13 14
186882024-02-03T17:17:48.728Z17/30 Write block 330 len 32768 data: 15 15 12 13 14 14 16 15
186892024-02-03T17:17:48.733Z18/30 Read block 286 len 20480
186902024-02-03T17:17:48.733Z19/30 Write block 110 len 8192 data: 8 9
186912024-02-03T17:17:48.733Z20/30 Write block 458 len 40960 data: 11 13 17 15 12 14 13 11 14 13
186922024-02-03T17:17:48.733Z21/30 Write block 306 len 4096 data: 14
186932024-02-03T17:17:48.733Z22/30 Write block 365 len 4096 data: 12
186942024-02-03T17:17:48.733Z23/30 Read block 68 len 8192
186952024-02-03T17:17:48.734Z24/30 Read block 449 len 20480
186962024-02-03T17:17:48.734Z25/30 Write block 525 len 40960 data: 15 14 14 13 13 14 12 10 11 9
186972024-02-03T17:17:48.734Z26/30 Read block 336 len 16384
186982024-02-03T17:17:48.734Z27/30 Write block 111 len 24576 data: 10 11 9 11 13 14
186992024-02-03T17:17:48.734Z28/30 Read block 488 len 16384
187002024-02-03T17:17:48.734Z29/30 Write block 469 len 20480 data: 12 12 11 11 8
187012024-02-03T17:17:48.734Z30/30 Read block 226 len 36864
187022024-02-03T17:17:48.734Z----------------------------------------------------------------
187032024-02-03T17:17:48.734Z Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30
187042024-02-03T17:17:48.734ZStates: Active Active Active
187052024-02-03T17:17:48.734ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
187062024-02-03T17:17:48.734Z 1 Acked 1000 Write 3 Done Done Done false
187072024-02-03T17:17:48.734Z 2 Acked 1001 Read 4 Done Sent Done false
187082024-02-03T17:17:48.734Z 3 Acked 1002 Read 1 Done Sent Done false
187092024-02-03T17:17:48.734Z 4 Acked 1003 Read 8 Done Sent Done false
187102024-02-03T17:17:48.734Z 5 Acked 1004 Flush 0 Done Sent Done false
187112024-02-03T17:17:48.734Z 6 Acked 1005 Write 8 Done Sent Done false
187122024-02-03T17:17:48.734Z 7 Acked 1006 Write 10 Done Sent Done false
187132024-02-03T17:17:48.734Z 8 Acked 1007 Flush 0 Done Sent Done false
187142024-02-03T17:17:48.734Z 9 Acked 1008 Write 7 Done Sent Done false
187152024-02-03T17:17:48.734Z 10 Acked 1009 Read 10 Done Sent Done false
187162024-02-03T17:17:48.734Z 11 Acked 1010 Read 5 Done Sent Done false
187172024-02-03T17:17:48.734Z 12 Acked 1011 Read 6 Done Sent Done false
187182024-02-03T17:17:48.734Z 13 Acked 1012 Read 9 Done Sent Done false
187192024-02-03T17:17:48.735Z 14 Acked 1013 Read 9 Done Sent Done false
187202024-02-03T17:17:48.735Z 15 Acked 1014 Write 5 Done Sent Done false
187212024-02-03T17:17:48.735Z 16 Acked 1015 Write 2 Done Sent Done false
187222024-02-03T17:17:48.735Z 17 Acked 1016 Write 8 Done Sent Done false
187232024-02-03T17:17:48.735Z 18 Acked 1017 Read 5 Done Sent Done false
187242024-02-03T17:17:48.735Z 19 Acked 1018 Write 2 Done Sent Done false
187252024-02-03T17:17:48.735Z 20 Acked 1019 Write 10 Done Sent Done false
187262024-02-03T17:17:48.735Z 21 Acked 1020 Write 1 Done Sent Done false
187272024-02-03T17:17:48.735Z 22 Acked 1021 Write 1 Done Sent Done false
187282024-02-03T17:17:48.735Z 23 Acked 1022 Read 2 Done Sent Done false
187292024-02-03T17:17:48.735Z 24 Acked 1023 Read 5 Done Sent Done false
187302024-02-03T17:17:48.735Z 25 Acked 1024 Write 10 Done Sent Done false
187312024-02-03T17:17:48.735Z 26 Acked 1025 Read 4 Done Sent Done false
187322024-02-03T17:17:48.735Z 27 Acked 1026 Write 6 Done Sent Done false
187332024-02-03T17:17:48.735Z 28 Acked 1027 Read 4 Done Sent Done false
187342024-02-03T17:17:48.735Z 29 Acked 1028 Write 5 Done Sent Done false
187352024-02-03T17:17:48.735Z 30 Acked 1029 Read 9 Done Sent Done false
187362024-02-03T17:17:48.735Z STATES DS:0 DS:1 DS:2 TOTAL
187372024-02-03T17:17:48.735Z New 0 0 0 0
187382024-02-03T17:17:48.735Z Sent 0 29 0 29
187392024-02-03T17:17:48.735Z Done 30 1 30 61
187402024-02-03T17:17:48.735Z Skipped 0 0 0 0
187412024-02-03T17:17:48.735Z Error 0 0 0 0
187422024-02-03T17:17:48.735ZLast Flush: 1007 0 1007
187432024-02-03T17:17:48.736ZDownstairs last five completed:
187442024-02-03T17:17:48.736ZUpstairs last five completed: 30 29 28 27 26
187452024-02-03T17:17:48.736Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:17:48.193000399Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1823}
187462024-02-03T17:17:48.736ZWrote out file "/tmp/test_repair_verify.data"
187472024-02-03T17:17:48.736Z
187482024-02-03T17:17:48.736ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
187492024-02-03T17:17:48.736ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
187502024-02-03T17:17:48.736Z 0 000-019 160 160 160 277 277 277 F F F
187512024-02-03T17:17:48.736Z 1 020-039 160 160 160 277 277 277 F F F
187522024-02-03T17:17:48.736Z 2 040-059 166 166 166 284 284 284 F F F
187532024-02-03T17:17:48.736Z 3 060-079 162 162 162 280 280 280 F F F
187542024-02-03T17:17:48.736Z 4 080-099 167 167 167 287 287 287 F F F
187552024-02-03T17:17:48.736Z 5 100-119 170 170 170 293 293 293 T F T <---
187562024-02-03T17:17:48.736Z 6 120-139 170 170 170 293 293 293 F F F
187572024-02-03T17:17:48.736Z 7 140-159 172 165 172 295 282 295 F F F <---
187582024-02-03T17:17:48.736Z 8 160-179 170 170 170 293 293 293 F F F
187592024-02-03T17:17:48.736Z 9 180-199 162 162 162 280 280 280 F F F
187602024-02-03T17:17:48.736Z 10 200-219 170 170 170 293 293 293 F F F
187612024-02-03T17:17:48.736Z 11 220-239 170 170 170 293 293 293 F F F
187622024-02-03T17:17:48.736Z 12 240-259 170 170 170 293 293 293 F F F
187632024-02-03T17:17:48.736Z 13 260-279 170 170 170 293 293 293 F F F
187642024-02-03T17:17:48.736Z 14 280-299 156 156 156 271 271 271 F F F
187652024-02-03T17:17:48.736Z 15 300-319 170 170 170 293 293 293 T F T <---
187662024-02-03T17:17:48.736Z 16 320-339 162 162 162 280 280 280 T F T <---
187672024-02-03T17:17:48.737Z 17 340-359 172 158 172 295 275 295 F F F <---
187682024-02-03T17:17:48.737Z 18 360-379 168 168 168 291 291 291 T F T <---
187692024-02-03T17:17:48.737Z 19 380-399 170 170 170 293 293 293 F F F
187702024-02-03T17:17:48.737Z 20 400-419 166 166 166 285 285 285 F F F
187712024-02-03T17:17:48.737Z 21 420-439 166 166 166 286 286 286 F F F
187722024-02-03T17:17:48.737Z 22 440-459 172 168 172 294 289 294 T T T <---
187732024-02-03T17:17:48.737Z 23 460-479 166 166 166 286 286 286 T F T <---
187742024-02-03T17:17:48.737Z 24 480-499 170 170 170 293 293 293 F F F
187752024-02-03T17:17:48.737Z 25 500-519 170 170 170 293 293 293 F F F
187762024-02-03T17:17:48.737Z 26 520-539 170 170 170 293 293 293 T F T <---
187772024-02-03T17:17:48.737Z 27 540-559 170 170 170 293 293 293 T F T <---
187782024-02-03T17:17:48.737Z 28 560-579 170 170 170 293 293 293 F F F
187792024-02-03T17:17:48.737Z 29 580-599 170 170 170 293 293 293 T F T <---
187802024-02-03T17:17:48.737ZMax gen: 172, Max flush: 295
187812024-02-03T17:17:48.737ZError: Difference in extent metadata found!
187822024-02-03T17:17:48.737ZA Difference in extent metadata is expected here
187832024-02-03T17:17:48.737ZOn loop 86
187842024-02-03T17:17:48.737Z
187852024-02-03T17:17:48.737ZVerifying data now
187862024-02-03T17:17:53.662ZNo change after verify
187872024-02-03T17:17:53.662ZLoop: 86 Downstairs dump after verify (and repair):
187882024-02-03T17:17:53.685ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
187892024-02-03T17:17:53.685Z 0 000-019 160 160 160 277 277 277 F F F
187902024-02-03T17:17:53.685Z 1 020-039 160 160 160 277 277 277 F F F
187912024-02-03T17:17:53.685Z 2 040-059 166 166 166 284 284 284 F F F
187922024-02-03T17:17:53.685Z 3 060-079 162 162 162 280 280 280 F F F
187932024-02-03T17:17:53.689Z 4 080-099 167 167 167 287 287 287 F F F
187942024-02-03T17:17:53.689Z 5 100-119 173 173 173 296 296 296 F F F
187952024-02-03T17:17:53.689Z 6 120-139 170 170 170 293 293 293 F F F
187962024-02-03T17:17:53.689Z 7 140-159 172 172 172 295 295 295 F F F
187972024-02-03T17:17:53.689Z 8 160-179 170 170 170 293 293 293 F F F
187982024-02-03T17:17:53.689Z 9 180-199 162 162 162 280 280 280 F F F
187992024-02-03T17:17:53.689Z 10 200-219 170 170 170 293 293 293 F F F
188002024-02-03T17:17:53.689Z 11 220-239 170 170 170 293 293 293 F F F
188012024-02-03T17:17:53.689Z 12 240-259 170 170 170 293 293 293 F F F
188022024-02-03T17:17:53.689Z 13 260-279 170 170 170 293 293 293 F F F
188032024-02-03T17:17:53.689Z 14 280-299 156 156 156 271 271 271 F F F
188042024-02-03T17:17:53.689Z 15 300-319 173 173 173 296 296 296 F F F
188052024-02-03T17:17:53.690Z 16 320-339 173 173 173 296 296 296 F F F
188062024-02-03T17:17:53.690Z 17 340-359 172 172 172 295 295 295 F F F
188072024-02-03T17:17:53.690Z 18 360-379 173 173 173 296 296 296 F F F
188082024-02-03T17:17:53.690Z 19 380-399 170 170 170 293 293 293 F F F
188092024-02-03T17:17:53.690Z 20 400-419 166 166 166 285 285 285 F F F
188102024-02-03T17:17:53.690Z 21 420-439 166 166 166 286 286 286 F F F
188112024-02-03T17:17:53.690Z 22 440-459 173 173 173 296 296 296 F F F
188122024-02-03T17:17:53.690Z 23 460-479 173 173 173 296 296 296 F F F
188132024-02-03T17:17:53.690Z 24 480-499 170 170 170 293 293 293 F F F
188142024-02-03T17:17:53.690Z 25 500-519 170 170 170 293 293 293 F F F
188152024-02-03T17:17:53.690Z 26 520-539 173 173 173 296 296 296 F F F
188162024-02-03T17:17:53.690Z 27 540-559 173 173 173 296 296 296 F F F
188172024-02-03T17:17:53.690Z 28 560-579 170 170 170 293 293 293 F F F
188182024-02-03T17:17:53.690Z 29 580-599 173 173 173 296 296 296 F F F
188192024-02-03T17:17:53.691ZMax gen: 173, Max flush: 296
188202024-02-03T17:17:53.696Z
188212024-02-03T17:17:53.696ZBegin loop 87
188222024-02-03T17:17:53.696ZDownstairs to restart: 1
188232024-02-03T17:17:53.720Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.298318999Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831}
188242024-02-03T17:17:53.720Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.298786601Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831}
188252024-02-03T17:17:53.724Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
188262024-02-03T17:17:53.724Z,"time":"2024-02-03T17:17:53.29891123Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831}
188272024-02-03T17:17:53.724Z{"msg":"Crucible b1349e6e-f6ce-4cbb-9464-30563064e6c7 has session id: 80092fcb-587e-4ade-87b3-ae4d0a334fe2","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.299075808Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188282024-02-03T17:17:53.724Z{"msg":"Upstairs opts: Upstairs UUID: b1349e6e-f6ce-4cbb-9464-30563064e6c7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.299140876Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188292024-02-03T17:17:53.724Z{"msg":"Crucible stats registered with UUID: b1349e6e-f6ce-4cbb-9464-30563064e6c7","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.299206213Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188302024-02-03T17:17:53.724Z{"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.299231271Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831}
188312024-02-03T17:17:53.724Z{"msg":"b1349e6e-f6ce-4cbb-9464-30563064e6c7 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.299275188Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188322024-02-03T17:17:53.724Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.29929725Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188332024-02-03T17:17:53.724Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.299321737Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188342024-02-03T17:17:53.724Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.299381076Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188352024-02-03T17:17:53.725Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.299495121Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188362024-02-03T17:17:53.725Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.299545705Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188372024-02-03T17:17:53.725Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.299589104Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188382024-02-03T17:17:53.725Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.299706951Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"":"io task","client":"0","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188392024-02-03T17:17:53.725Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.299796789Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"":"io task","client":"2","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188402024-02-03T17:17:53.725Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.29988806Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"":"io task","client":"1","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188412024-02-03T17:17:53.725Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.300026361Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"":"io task","client":"0","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188422024-02-03T17:17:53.725Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.300104069Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"":"io task","client":"1","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188432024-02-03T17:17:53.725Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.300132569Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"":"io task","client":"2","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188442024-02-03T17:17:53.725Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.301164928Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188452024-02-03T17:17:53.725Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.301233427Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188462024-02-03T17:17:53.725Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.301256696Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188472024-02-03T17:17:53.725Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.301278736Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188482024-02-03T17:17:53.726Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.301703252Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188492024-02-03T17:17:53.726Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.3017791Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188502024-02-03T17:17:53.757Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.302432118Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188512024-02-03T17:17:53.757Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.302504974Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188522024-02-03T17:17:53.757Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.302653245Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188532024-02-03T17:17:53.757Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.303456026Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188542024-02-03T17:17:53.757Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.303587485Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188552024-02-03T17:17:53.757Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.303690107Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188562024-02-03T17:17:53.757Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.303775237Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188572024-02-03T17:17:53.757Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.30382002Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188582024-02-03T17:17:53.757Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.303861227Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188592024-02-03T17:17:53.757Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.303937584Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188602024-02-03T17:17:53.758Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.303979249Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188612024-02-03T17:17:53.758Z{"msg":"[0]R flush_numbers[0..12]: [277, 277, 284, 280, 287, 296, 293, 295, 293, 280, 293, 293]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.304059491Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188622024-02-03T17:17:53.758Z{"msg":"[0]R generation[0..12]: [160, 160, 166, 162, 167, 173, 170, 172, 170, 162, 170, 170]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
188632024-02-03T17:17:53.758Z,"time":"2024-02-03T17:17:53.30417292Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188642024-02-03T17:17:53.758Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.304261823Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188652024-02-03T17:17:53.758Z{"msg":"[1]R flush_numbers[0..12]: [277, 277, 284, 280, 287, 296, 293, 295, 293, 280, 293, 293]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.304345358Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188662024-02-03T17:17:53.758Z{"msg":"[1]R generation[0..12]: [160, 160, 166, 162, 167, 173, 170, 172, 170, 162, 170, 170]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.304435588Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188672024-02-03T17:17:53.759Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.304518998Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188682024-02-03T17:17:53.759Z{"msg":"[2]R flush_numbers[0..12]: [277, 277, 284, 280, 287, 296, 293, 295, 293, 280, 293, 293]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.304607023Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188692024-02-03T17:17:53.759Z{"msg":"[2]R generation[0..12]: [160, 160, 166, 162, 167, 173, 170, 172, 170, 162, 170, 170]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.304693288Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188702024-02-03T17:17:53.759Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
188712024-02-03T17:17:53.761Z,"time":"2024-02-03T17:17:53.304777439Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188722024-02-03T17:17:53.761Z{"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.304850363Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188732024-02-03T17:17:53.761Z{"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.304878511Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188742024-02-03T17:17:53.761Z{"msg":"Next flush: 297","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.304926502Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188752024-02-03T17:17:53.761Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.304950101Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188762024-02-03T17:17:53.761Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.305002828Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188772024-02-03T17:17:53.761Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.305025413Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188782024-02-03T17:17:53.761Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.305073311Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188792024-02-03T17:17:53.761Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.305097295Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188802024-02-03T17:17:53.761Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.305144941Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188812024-02-03T17:17:53.762Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.305167531Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188822024-02-03T17:17:53.762Z{"msg":"b1349e6e-f6ce-4cbb-9464-30563064e6c7 is now active with session: 80092fcb-587e-4ade-87b3-ae4d0a334fe2","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.305214473Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188832024-02-03T17:17:53.762Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.30523614Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831,"session_id":"80092fcb-587e-4ade-87b3-ae4d0a334fe2"}
188842024-02-03T17:17:53.762Z{"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:53.305282866Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831}
188852024-02-03T17:17:53.762ZLoading write count information from file "/tmp/test_repair_verify.data"
188862024-02-03T17:17:53.762ZRun Repair workload
188872024-02-03T17:17:53.762Z01/30 Write block 582 len 36864 data: 9 7 6 7 11 12 15 13 14
188882024-02-03T17:17:53.762Z02/30 Read block 8 len 24576
188892024-02-03T17:17:53.766Z03/30 Write block 275 len 28672 data: 12 12 12 12 14 13 15
188902024-02-03T17:17:53.766Z04/30 Write block 513 len 40960 data: 15 15 14 16 13 10 10 11 9 11
188912024-02-03T17:17:53.766Z05/30 Read block 455 len 16384
188922024-02-03T17:17:53.767Z06/30 Flush
188932024-02-03T17:17:53.767Z07/30 Write block 193 len 28672 data: 16 17 18 17 15 15 9
188942024-02-03T17:17:53.767Z08/30 Write block 326 len 28672 data: 13 18 17 16 16 16 13
188952024-02-03T17:17:53.767Z09/30 Write block 448 len 28672 data: 14 10 9 7 7 9 9
188962024-02-03T17:17:53.767Z10/30 Write block 106 len 16384 data: 12 12 7 7
188972024-02-03T17:17:53.767Z11/30 Read block 306 len 20480
188982024-02-03T17:17:53.767Z12/30 Read block 73 len 24576
188992024-02-03T17:17:53.767Z13/30 Read block 119 len 40960
189002024-02-03T17:17:53.767Z14/30 Write block 165 len 12288 data: 17 18 21
189012024-02-03T17:17:53.767Z15/30 Read block 108 len 36864
189022024-02-03T17:17:53.767Z16/30 Write block 277 len 8192 data: 13 13
189032024-02-03T17:17:53.767Z17/30 Flush
189042024-02-03T17:17:53.767Z18/30 Write block 169 len 24576 data: 19 20 18 15 15 13
189052024-02-03T17:17:53.767Z19/30 Write block 35 len 40960 data: 13 15 13 12 12 9 8 11 10 10
189062024-02-03T17:17:53.767Z20/30 Read block 329 len 20480
189072024-02-03T17:17:53.767Z21/30 Read block 181 len 20480
189082024-02-03T17:17:53.767Z22/30 Write block 167 len 4096 data: 22
189092024-02-03T17:17:53.774Z23/30 Write block 442 len 36864 data: 21 19 14 16 16 16 15 11 10
189102024-02-03T17:17:53.774Z24/30 Flush
189112024-02-03T17:17:53.782Z25/30 Write block 350 len 12288 data: 11 10 12
189122024-02-03T17:17:53.782Z26/30 Read block 242 len 28672
189132024-02-03T17:17:53.782Z27/30 Read block 449 len 4096
189142024-02-03T17:17:53.782Z28/30 Read block 253 len 24576
189152024-02-03T17:17:53.782Z29/30 Read block 263 len 32768
189162024-02-03T17:17:53.782Z30/30 Read block 541 len 4096
189172024-02-03T17:17:53.793Z----------------------------------------------------------------
189182024-02-03T17:17:53.794Z Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:30
189192024-02-03T17:17:53.794ZStates: Active Active Active
189202024-02-03T17:17:53.794ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
189212024-02-03T17:17:53.794Z 1 Acked 1000 Write 9 Done Done Done false
189222024-02-03T17:17:53.794Z 2 Acked 1001 Read 6 Done Done Done false
189232024-02-03T17:17:53.794Z 3 Acked 1002 Write 7 Done Done Done false
189242024-02-03T17:17:53.794Z 4 Acked 1003 Write 10 Done Done Done false
189252024-02-03T17:17:53.794Z 5 Acked 1004 Read 4 Done Sent Done false
189262024-02-03T17:17:53.794Z 6 Acked 1005 Flush 0 Done Sent Done false
189272024-02-03T17:17:53.794Z 7 Acked 1006 Write 7 Done Sent Done false
189282024-02-03T17:17:53.794Z 8 Acked 1007 Write 7 Done Sent Done false
189292024-02-03T17:17:53.794Z 9 Acked 1008 Write 7 Done Sent Done false
189302024-02-03T17:17:53.794Z 10 Acked 1009 Write 4 Done Sent Done false
189312024-02-03T17:17:53.794Z 11 Acked 1010 Read 5 Done Sent Done false
189322024-02-03T17:17:53.794Z 12 Acked 1011 Read 6 Done Sent Done false
189332024-02-03T17:17:53.794Z 13 Acked 1012 Read 10 Done Sent Done false
189342024-02-03T17:17:53.794Z 14 Acked 1013 Write 3 Done Sent Done false
189352024-02-03T17:17:53.794Z 15 Acked 1014 Read 9 Done Sent Done false
189362024-02-03T17:17:53.794Z 16 Acked 1015 Write 2 Done Sent Done false
189372024-02-03T17:17:53.794Z 17 Acked 1016 Flush 0 Done Sent Done false
189382024-02-03T17:17:53.794Z 18 Acked 1017 Write 6 Done Sent Done false
189392024-02-03T17:17:53.794Z 19 Acked 1018 Write 10 Done Sent Done false
189402024-02-03T17:17:53.794Z 20 Acked 1019 Read 5 Done Sent Done false
189412024-02-03T17:17:53.795Z 21 Acked 1020 Read 5 Done Sent Done false
189422024-02-03T17:17:53.795Z 22 Acked 1021 Write 1 Done Sent Done false
189432024-02-03T17:17:53.795Z 23 Acked 1022 Write 9 Done Sent Done false
189442024-02-03T17:17:53.795Z 24 Acked 1023 Flush 0 Done Sent Done false
189452024-02-03T17:17:53.795Z 25 Acked 1024 Write 3 Done Sent Done false
189462024-02-03T17:17:53.795Z 26 Acked 1025 Read 7 Done Sent Done false
189472024-02-03T17:17:53.795Z 27 Acked 1026 Read 1 Done Sent Done false
189482024-02-03T17:17:53.795Z 28 Acked 1027 Read 6 Done Sent Done false
189492024-02-03T17:17:53.795Z 29 Acked 1028 Read 8 Done Sent Done false
189502024-02-03T17:17:53.795Z 30 Acked 1029 Read 1 Done Sent Done false
189512024-02-03T17:17:53.795Z STATES DS:0 DS:1 DS:2 TOTAL
189522024-02-03T17:17:53.795Z New 0 0 0 0
189532024-02-03T17:17:53.795Z Sent 0 26 0 26
189542024-02-03T17:17:53.795Z Done 30 4 30 64
189552024-02-03T17:17:53.795Z Skipped 0 0 0 0
189562024-02-03T17:17:53.795Z Error 0 0 0 0
189572024-02-03T17:17:53.795ZLast Flush: 1023 0 1023
189582024-02-03T17:17:53.795ZDownstairs last five completed:
189592024-02-03T17:17:53.795ZUpstairs last five completed: 30 29 28 27 26
189602024-02-03T17:17:53.796Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:17:53.358667486Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1831}
189612024-02-03T17:17:53.796ZWrote out file "/tmp/test_repair_verify.data"
189622024-02-03T17:17:53.796Z
189632024-02-03T17:17:53.805ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
189642024-02-03T17:17:53.822ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
189652024-02-03T17:17:53.822Z 0 000-019 160 160 160 277 277 277 F F F
189662024-02-03T17:17:53.823Z 1 020-039 174 160 174 299 277 299 F F F <---
189672024-02-03T17:17:53.823Z 2 040-059 174 166 174 299 284 299 F F F <---
189682024-02-03T17:17:53.823Z 3 060-079 162 162 162 280 280 280 F F F
189692024-02-03T17:17:53.823Z 4 080-099 167 167 167 287 287 287 F F F
189702024-02-03T17:17:53.830Z 5 100-119 174 173 174 298 296 298 F F F <---
189712024-02-03T17:17:53.830Z 6 120-139 170 170 170 293 293 293 F F F
189722024-02-03T17:17:53.830Z 7 140-159 172 172 172 295 295 295 F F F
189732024-02-03T17:17:53.830Z 8 160-179 174 170 174 299 293 299 F F F <---
189742024-02-03T17:17:53.831Z 9 180-199 174 162 174 298 280 298 F F F <---
189752024-02-03T17:17:53.831Z 10 200-219 170 170 170 293 293 293 F F F
189762024-02-03T17:17:53.831Z 11 220-239 170 170 170 293 293 293 F F F
189772024-02-03T17:17:53.831Z 12 240-259 170 170 170 293 293 293 F F F
189782024-02-03T17:17:53.831Z 13 260-279 174 170 174 298 293 298 F T F <---
189792024-02-03T17:17:53.831Z 14 280-299 174 156 174 297 271 297 F T F <---
189802024-02-03T17:17:53.831Z 15 300-319 173 173 173 296 296 296 F F F
189812024-02-03T17:17:53.831Z 16 320-339 174 173 174 298 296 298 F F F <---
189822024-02-03T17:17:53.831Z 17 340-359 172 172 172 295 295 295 T F T <---
189832024-02-03T17:17:53.831Z 18 360-379 173 173 173 296 296 296 F F F
189842024-02-03T17:17:53.831Z 19 380-399 170 170 170 293 293 293 F F F
189852024-02-03T17:17:53.831Z 20 400-419 166 166 166 285 285 285 F F F
189862024-02-03T17:17:53.831Z 21 420-439 166 166 166 286 286 286 F F F
189872024-02-03T17:17:53.832Z 22 440-459 174 173 174 299 296 299 F F F <---
189882024-02-03T17:17:53.832Z 23 460-479 173 173 173 296 296 296 F F F
189892024-02-03T17:17:53.832Z 24 480-499 170 170 170 293 293 293 F F F
189902024-02-03T17:17:53.832Z 25 500-519 174 170 174 297 293 297 F T F <---
189912024-02-03T17:17:53.832Z 26 520-539 174 173 174 297 296 297 F T F <---
189922024-02-03T17:17:53.832Z 27 540-559 173 173 173 296 296 296 F F F
189932024-02-03T17:17:53.832Z 28 560-579 170 170 170 293 293 293 F F F
189942024-02-03T17:17:53.832Z 29 580-599 174 173 174 297 296 297 F T F <---
189952024-02-03T17:17:53.832ZMax gen: 174, Max flush: 299
189962024-02-03T17:17:53.832ZError: Difference in extent metadata found!
189972024-02-03T17:17:53.832ZA Difference in extent metadata is expected here
189982024-02-03T17:17:53.832ZOn loop 87
189992024-02-03T17:17:53.832Z
190002024-02-03T17:17:53.832ZVerifying data now
190012024-02-03T17:17:59.939ZNo change after verify
190022024-02-03T17:17:59.939ZLoop: 87 Downstairs dump after verify (and repair):
190032024-02-03T17:17:59.962ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
190042024-02-03T17:17:59.962Z 0 000-019 160 160 160 277 277 277 F F F
190052024-02-03T17:17:59.965Z 1 020-039 174 174 174 299 299 299 F F F
190062024-02-03T17:17:59.965Z 2 040-059 174 174 174 299 299 299 F F F
190072024-02-03T17:17:59.965Z 3 060-079 162 162 162 280 280 280 F F F
190082024-02-03T17:17:59.965Z 4 080-099 167 167 167 287 287 287 F F F
190092024-02-03T17:17:59.965Z 5 100-119 174 174 174 298 298 298 F F F
190102024-02-03T17:17:59.965Z 6 120-139 170 170 170 293 293 293 F F F
190112024-02-03T17:17:59.966Z 7 140-159 172 172 172 295 295 295 F F F
190122024-02-03T17:17:59.966Z 8 160-179 174 174 174 299 299 299 F F F
190132024-02-03T17:17:59.966Z 9 180-199 174 174 174 298 298 298 F F F
190142024-02-03T17:17:59.966Z 10 200-219 170 170 170 293 293 293 F F F
190152024-02-03T17:17:59.966Z 11 220-239 170 170 170 293 293 293 F F F
190162024-02-03T17:17:59.966Z 12 240-259 170 170 170 293 293 293 F F F
190172024-02-03T17:17:59.966Z 13 260-279 174 174 174 298 298 298 F F F
190182024-02-03T17:17:59.966Z 14 280-299 174 174 174 297 297 297 F F F
190192024-02-03T17:17:59.966Z 15 300-319 173 173 173 296 296 296 F F F
190202024-02-03T17:17:59.966Z 16 320-339 174 174 174 298 298 298 F F F
190212024-02-03T17:17:59.966Z 17 340-359 175 175 175 300 300 300 F F F
190222024-02-03T17:17:59.966Z 18 360-379 173 173 173 296 296 296 F F F
190232024-02-03T17:17:59.966Z 19 380-399 170 170 170 293 293 293 F F F
190242024-02-03T17:17:59.966Z 20 400-419 166 166 166 285 285 285 F F F
190252024-02-03T17:17:59.966Z 21 420-439 166 166 166 286 286 286 F F F
190262024-02-03T17:17:59.966Z 22 440-459 174 174 174 299 299 299 F F F
190272024-02-03T17:17:59.966Z 23 460-479 173 173 173 296 296 296 F F F
190282024-02-03T17:17:59.966Z 24 480-499 170 170 170 293 293 293 F F F
190292024-02-03T17:17:59.966Z 25 500-519 174 174 174 297 297 297 F F F
190302024-02-03T17:17:59.966Z 26 520-539 174 174 174 297 297 297 F F F
190312024-02-03T17:17:59.966Z 27 540-559 173 173 173 296 296 296 F F F
190322024-02-03T17:17:59.966Z 28 560-579 170 170 170 293 293 293 F F F
190332024-02-03T17:17:59.966Z 29 580-599 174 174 174 297 297 297 F F F
190342024-02-03T17:17:59.966ZMax gen: 175, Max flush: 300
190352024-02-03T17:17:59.970Z
190362024-02-03T17:17:59.970ZBegin loop 88
190372024-02-03T17:17:59.970ZDownstairs to restart: 0
190382024-02-03T17:17:59.998Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.577342305Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839}
190392024-02-03T17:17:59.998Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.577777182Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839}
190402024-02-03T17:18:00.002Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.577822189Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839}
190412024-02-03T17:18:00.002Z{"msg":"Crucible 66dd47cb-151d-4213-abe7-e7155fbd076d has session id: e8de566b-9789-4946-8f07-debad1a6b48b","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.577863598Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190422024-02-03T17:18:00.002Z{"msg":"Upstairs opts: Upstairs UUID: 66dd47cb-151d-4213-abe7-e7155fbd076d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
190432024-02-03T17:18:00.002Z,"time":"2024-02-03T17:17:59.578007944Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190442024-02-03T17:18:00.002Z{"msg":"Crucible stats registered with UUID: 66dd47cb-151d-4213-abe7-e7155fbd076d","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.578064919Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190452024-02-03T17:18:00.002Z{"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.578144727Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839}
190462024-02-03T17:18:00.003Z{"msg":"66dd47cb-151d-4213-abe7-e7155fbd076d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.578338365Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190472024-02-03T17:18:00.003Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.578397083Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190482024-02-03T17:18:00.003Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.57842358Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190492024-02-03T17:18:00.003Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.578604219Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190502024-02-03T17:18:00.003Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.578650478Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190512024-02-03T17:18:00.003Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.578694355Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190522024-02-03T17:18:00.003Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.57878726Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190532024-02-03T17:18:00.003Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.57908522Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"":"io task","client":"0","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190542024-02-03T17:18:00.003Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.579175769Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"":"io task","client":"2","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190552024-02-03T17:18:00.003Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.579258973Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"":"io task","client":"1","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190562024-02-03T17:18:00.003Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.579400038Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"":"io task","client":"1","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190572024-02-03T17:18:00.003Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.579504722Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"":"io task","client":"0","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190582024-02-03T17:18:00.003Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.579554923Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"":"io task","client":"2","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190592024-02-03T17:18:00.003Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.580591091Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190602024-02-03T17:18:00.003Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.580664991Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190612024-02-03T17:18:00.003Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.580692202Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190622024-02-03T17:18:00.003Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.580714371Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190632024-02-03T17:18:00.003Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.58073637Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190642024-02-03T17:18:00.003Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.580861145Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190652024-02-03T17:18:00.009Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.581630427Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190662024-02-03T17:18:00.009Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.581878675Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190672024-02-03T17:18:00.010Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.582406113Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190682024-02-03T17:18:00.010Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.582565571Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190692024-02-03T17:18:00.010Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.582656953Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190702024-02-03T17:18:00.010Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.582701541Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190712024-02-03T17:18:00.010Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.582766899Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190722024-02-03T17:18:00.010Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.582801236Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190732024-02-03T17:18:00.010Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.582856603Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190742024-02-03T17:18:00.010Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.583092587Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190752024-02-03T17:18:00.010Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.583189308Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190762024-02-03T17:18:00.010Z{"msg":"[0]R flush_numbers[0..12]: [277, 299, 299, 280, 287, 298, 293, 295, 299, 298, 293, 293]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.583219418Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190772024-02-03T17:18:00.010Z{"msg":"[0]R generation[0..12]: [160, 174, 174, 162, 167, 174, 170, 172, 174, 174, 170, 170]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.583269157Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190782024-02-03T17:18:00.010Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.583295686Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190792024-02-03T17:18:00.010Z{"msg":"[1]R flush_numbers[0..12]: [277, 299, 299, 280, 287, 298, 293, 295, 299, 298, 293, 293]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.583319268Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190802024-02-03T17:18:00.010Z{"msg":"[1]R generation[0..12]: [160, 174, 174, 162, 167, 174, 170, 172, 174, 174, 170, 170]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.583368229Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190812024-02-03T17:18:00.010Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.583394154Z","hostname":"Wait for a query_work_queue command to finish before sending IO
190822024-02-03T17:18:00.010Zip-10-150-1-9.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190832024-02-03T17:18:00.010Z{"msg":"[2]R flush_numbers[0..12]: [277, 299, 299, 280, 287, 298, 293, 295, 299, 298, 293, 293]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.583496143Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190842024-02-03T17:18:00.010Z{"msg":"[2]R generation[0..12]: [160, 174, 174, 162, 167, 174, 170, 172, 174, 174, 170, 170]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.583551485Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190852024-02-03T17:18:00.010Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.583578583Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190862024-02-03T17:18:00.010Z{"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.583624837Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190872024-02-03T17:18:00.010Z{"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.583649281Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190882024-02-03T17:18:00.010Z{"msg":"Next flush: 301","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.583696914Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190892024-02-03T17:18:00.010Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.58372125Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190902024-02-03T17:18:00.010Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.583743845Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190912024-02-03T17:18:00.011Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.58378816Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190922024-02-03T17:18:00.011Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.583812252Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190932024-02-03T17:18:00.011Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.583859462Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190942024-02-03T17:18:00.011Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.583883733Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190952024-02-03T17:18:00.011Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.583904133Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190962024-02-03T17:18:00.011Z{"msg":"66dd47cb-151d-4213-abe7-e7155fbd076d is now active with session: e8de566b-9789-4946-8f07-debad1a6b48b","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.583950535Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190972024-02-03T17:18:00.011Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.583972272Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839,"session_id":"e8de566b-9789-4946-8f07-debad1a6b48b"}
190982024-02-03T17:18:00.011Z{"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:17:59.584042393Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839}
190992024-02-03T17:18:00.011ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
191002024-02-03T17:18:00.011ZLoading write count information from file "/tmp/test_repair_verify.data"
191012024-02-03T17:18:00.011ZRun Repair workload
191022024-02-03T17:18:00.011Z01/30 Flush
191032024-02-03T17:18:00.020Z02/30 Write block 212 len 32768 data: 12 11 6 7 11 10 10 9
191042024-02-03T17:18:00.020Z03/30 Read block 318 len 4096
191052024-02-03T17:18:00.020Z04/30 Read block 176 len 24576
191062024-02-03T17:18:00.021Z05/30 Write block 350 len 36864 data: 12 11 13 12 13 14 11 12 11
191072024-02-03T17:18:00.021Z06/30 Read block 554 len 32768
191082024-02-03T17:18:00.021Z07/30 Read block 92 len 12288
191092024-02-03T17:18:00.021Z08/30 Flush
191102024-02-03T17:18:00.024Z09/30 Write block 301 len 32768 data: 12 12 12 14 14 15 13 14
191112024-02-03T17:18:00.024Z10/30 Read block 244 len 32768
191122024-02-03T17:18:00.027Z11/30 Write block 71 len 24576 data: 6 5 6 9 12 14
191132024-02-03T17:18:00.027Z12/30 Write block 581 len 32768 data: 11 10 8 7 8 12 13 16
191142024-02-03T17:18:00.030Z13/30 Flush
191152024-02-03T17:18:00.033Z14/30 Write block 522 len 40960 data: 12 13 12 16 15 15 14 14 15 13
191162024-02-03T17:18:00.033Z15/30 Write block 518 len 28672 data: 11 11 12 10 13 14 13
191172024-02-03T17:18:00.036Z16/30 Write block 223 len 24576 data: 6 5 5 7 8 12
191182024-02-03T17:18:00.036Z17/30 Read block 426 len 12288
191192024-02-03T17:18:00.039Z18/30 Read block 150 len 4096
191202024-02-03T17:18:00.039Z19/30 Read block 101 len 32768
191212024-02-03T17:18:00.043Z20/30 Write block 403 len 32768 data: 8 8 12 15 16 14 13 14
191222024-02-03T17:18:00.043Z21/30 Read block 469 len 36864
191232024-02-03T17:18:00.054Z22/30 Write block 200 len 16384 data: 9 8 12 11
191242024-02-03T17:18:00.054Z23/30 Write block 320 len 12288 data: 16 15 13
191252024-02-03T17:18:00.054Z24/30 Write block 304 len 28672 data: 15 15 16 14 15 15 17
191262024-02-03T17:18:00.061Z25/30 Write block 313 len 36864 data: 15 16 18 18 18 14 16 17 16
191272024-02-03T17:18:00.061Z26/30 Write block 360 len 36864 data: 12 12 10 10 11 13 11 13 12
191282024-02-03T17:18:00.061Z27/30 Read block 384 len 28672
191292024-02-03T17:18:00.061Z28/30 Read block 239 len 36864
191302024-02-03T17:18:00.061Z29/30 Write block 383 len 36864 data: 15 13 11 10 10 9 7 14 15
191312024-02-03T17:18:00.061Z30/30 Write block 585 len 28672 data: 9 13 14 17 14 15 14
191322024-02-03T17:18:00.061Z----------------------------------------------------------------
191332024-02-03T17:18:00.061Z Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:30
191342024-02-03T17:18:00.061ZStates: Active Active Active
191352024-02-03T17:18:00.061ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
191362024-02-03T17:18:00.061Z 1 Acked 1000 Flush 0 Sent Done Done false
191372024-02-03T17:18:00.061Z 2 Acked 1001 Write 8 Sent Done Done false
191382024-02-03T17:18:00.061Z 3 Acked 1002 Read 1 Sent Done Done false
191392024-02-03T17:18:00.061Z 4 Acked 1003 Read 6 Sent Done Done false
191402024-02-03T17:18:00.061Z 5 Acked 1004 Write 9 Sent Done Done false
191412024-02-03T17:18:00.061Z 6 Acked 1005 Read 8 Sent Done Done false
191422024-02-03T17:18:00.061Z 7 Acked 1006 Read 3 Sent Done Done false
191432024-02-03T17:18:00.062Z 8 Acked 1007 Flush 0 Sent Done Done false
191442024-02-03T17:18:00.062Z 9 Acked 1008 Write 8 Sent Done Done false
191452024-02-03T17:18:00.062Z 10 Acked 1009 Read 8 Sent Done Done false
191462024-02-03T17:18:00.062Z 11 Acked 1010 Write 6 Sent Done Done false
191472024-02-03T17:18:00.062Z 12 Acked 1011 Write 8 Sent Done Done false
191482024-02-03T17:18:00.062Z 13 Acked 1012 Flush 0 Sent Done Done false
191492024-02-03T17:18:00.062Z 14 Acked 1013 Write 10 Sent Done Done false
191502024-02-03T17:18:00.062Z 15 Acked 1014 Write 7 Sent Done Done false
191512024-02-03T17:18:00.062Z 16 Acked 1015 Write 6 Sent Done Done false
191522024-02-03T17:18:00.062Z 17 Acked 1016 Read 3 Sent Done Done false
191532024-02-03T17:18:00.062Z 18 Acked 1017 Read 1 Sent Done Done false
191542024-02-03T17:18:00.062Z 19 Acked 1018 Read 8 Sent Done Done false
191552024-02-03T17:18:00.062Z 20 Acked 1019 Write 8 Sent Done Done false
191562024-02-03T17:18:00.062Z 21 Acked 1020 Read 9 Sent Done Done false
191572024-02-03T17:18:00.062Z 22 Acked 1021 Write 4 Sent Done Done false
191582024-02-03T17:18:00.062Z 23 Acked 1022 Write 3 Sent Done Done false
191592024-02-03T17:18:00.062Z 24 Acked 1023 Write 7 Sent Done Done false
191602024-02-03T17:18:00.062Z 25 Acked 1024 Write 9 Sent Done Done false
191612024-02-03T17:18:00.062Z 26 Acked 1025 Write 9 Sent Done Done false
191622024-02-03T17:18:00.062Z 27 Acked 1026 Read 7 Sent Done Done false
191632024-02-03T17:18:00.062Z 28 Acked 1027 Read 9 Sent Done Done false
191642024-02-03T17:18:00.062Z 29 Acked 1028 Write 9 Sent Done Done false
191652024-02-03T17:18:00.062Z 30 Acked 1029 Write 7 Sent Sent Sent false
191662024-02-03T17:18:00.062Z STATES DS:0 DS:1 DS:2 TOTAL
191672024-02-03T17:18:00.062Z New 0 0 0 0
191682024-02-03T17:18:00.063Z Sent 30 1 1 32
191692024-02-03T17:18:00.063Z Done 0 29 29 58
191702024-02-03T17:18:00.063Z Skipped 0 0 0 0
191712024-02-03T17:18:00.063Z Error 0 0 0 0
191722024-02-03T17:18:00.063ZLast Flush: 0 1012 1012
191732024-02-03T17:18:00.063ZDownstairs last five completed:
191742024-02-03T17:18:00.063ZUpstairs last five completed: 30 29 28 27 26
191752024-02-03T17:18:00.063Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:17:59.636726867Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1839}
191762024-02-03T17:18:00.067ZWrote out file "/tmp/test_repair_verify.data"
191772024-02-03T17:18:00.067Z
191782024-02-03T17:18:00.071ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
191792024-02-03T17:18:00.100ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
191802024-02-03T17:18:00.100Z 0 000-019 160 160 160 277 277 277 F F F
191812024-02-03T17:18:00.100Z 1 020-039 174 174 174 299 299 299 F F F
191822024-02-03T17:18:00.100Z 2 040-059 174 174 174 299 299 299 F F F
191832024-02-03T17:18:00.100Z 3 060-079 162 176 176 280 303 303 F F F <---
191842024-02-03T17:18:00.100Z 4 080-099 167 167 167 287 287 287 F F F
191852024-02-03T17:18:00.107Z 5 100-119 174 174 174 298 298 298 F F F
191862024-02-03T17:18:00.107Z 6 120-139 170 170 170 293 293 293 F F F
191872024-02-03T17:18:00.107Z 7 140-159 172 172 172 295 295 295 F F F
191882024-02-03T17:18:00.107Z 8 160-179 174 174 174 299 299 299 F F F
191892024-02-03T17:18:00.107Z 9 180-199 174 174 174 298 298 298 F F F
191902024-02-03T17:18:00.107Z 10 200-219 170 176 176 293 302 302 F T T <---
191912024-02-03T17:18:00.107Z 11 220-239 170 170 170 293 293 293 F T T <---
191922024-02-03T17:18:00.107Z 12 240-259 170 170 170 293 293 293 F F F
191932024-02-03T17:18:00.107Z 13 260-279 174 174 174 298 298 298 F F F
191942024-02-03T17:18:00.107Z 14 280-299 174 174 174 297 297 297 F F F
191952024-02-03T17:18:00.107Z 15 300-319 173 176 176 296 303 303 F T T <---
191962024-02-03T17:18:00.107Z 16 320-339 174 174 174 298 298 298 F T T <---
191972024-02-03T17:18:00.107Z 17 340-359 175 176 176 300 302 302 F F F <---
191982024-02-03T17:18:00.107Z 18 360-379 173 173 173 296 296 296 F T T <---
191992024-02-03T17:18:00.107Z 19 380-399 170 170 170 293 293 293 F T T <---
192002024-02-03T17:18:00.107Z 20 400-419 166 166 166 285 285 285 F T T <---
192012024-02-03T17:18:00.107Z 21 420-439 166 166 166 286 286 286 F F F
192022024-02-03T17:18:00.107Z 22 440-459 174 174 174 299 299 299 F F F
192032024-02-03T17:18:00.107Z 23 460-479 173 173 173 296 296 296 F F F
192042024-02-03T17:18:00.107Z 24 480-499 170 170 170 293 293 293 F F F
192052024-02-03T17:18:00.107Z 25 500-519 174 174 174 297 297 297 F T T <---
192062024-02-03T17:18:00.108Z 26 520-539 174 174 174 297 297 297 F T T <---
192072024-02-03T17:18:00.108Z 27 540-559 173 173 173 296 296 296 F F F
192082024-02-03T17:18:00.108Z 28 560-579 170 170 170 293 293 293 F F F
192092024-02-03T17:18:00.108Z 29 580-599 174 176 176 297 303 303 F T T <---
192102024-02-03T17:18:00.108ZMax gen: 176, Max flush: 303
192112024-02-03T17:18:00.108ZError: Difference in extent metadata found!
192122024-02-03T17:18:00.111ZA Difference in extent metadata is expected here
192132024-02-03T17:18:00.111ZOn loop 88
192142024-02-03T17:18:00.111Z
192152024-02-03T17:18:00.111ZVerifying data now
192162024-02-03T17:18:05.176ZNo change after verify
192172024-02-03T17:18:05.176ZLoop: 88 Downstairs dump after verify (and repair):
192182024-02-03T17:18:05.199ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
192192024-02-03T17:18:05.199Z 0 000-019 160 160 160 277 277 277 F F F
192202024-02-03T17:18:05.199Z 1 020-039 174 174 174 299 299 299 F F F
192212024-02-03T17:18:05.199Z 2 040-059 174 174 174 299 299 299 F F F
192222024-02-03T17:18:05.202Z 3 060-079 176 176 176 303 303 303 F F F
192232024-02-03T17:18:05.202Z 4 080-099 167 167 167 287 287 287 F F F
192242024-02-03T17:18:05.202Z 5 100-119 174 174 174 298 298 298 F F F
192252024-02-03T17:18:05.202Z 6 120-139 170 170 170 293 293 293 F F F
192262024-02-03T17:18:05.202Z 7 140-159 172 172 172 295 295 295 F F F
192272024-02-03T17:18:05.202Z 8 160-179 174 174 174 299 299 299 F F F
192282024-02-03T17:18:05.202Z 9 180-199 174 174 174 298 298 298 F F F
192292024-02-03T17:18:05.203Z 10 200-219 177 177 177 304 304 304 F F F
192302024-02-03T17:18:05.203Z 11 220-239 177 177 177 304 304 304 F F F
192312024-02-03T17:18:05.203Z 12 240-259 170 170 170 293 293 293 F F F
192322024-02-03T17:18:05.203Z 13 260-279 174 174 174 298 298 298 F F F
192332024-02-03T17:18:05.203Z 14 280-299 174 174 174 297 297 297 F F F
192342024-02-03T17:18:05.203Z 15 300-319 177 177 177 304 304 304 F F F
192352024-02-03T17:18:05.203Z 16 320-339 177 177 177 304 304 304 F F F
192362024-02-03T17:18:05.203Z 17 340-359 176 176 176 302 302 302 F F F
192372024-02-03T17:18:05.203Z 18 360-379 177 177 177 304 304 304 F F F
192382024-02-03T17:18:05.203Z 19 380-399 177 177 177 304 304 304 F F F
192392024-02-03T17:18:05.203Z 20 400-419 177 177 177 304 304 304 F F F
192402024-02-03T17:18:05.203Z 21 420-439 166 166 166 286 286 286 F F F
192412024-02-03T17:18:05.203Z 22 440-459 174 174 174 299 299 299 F F F
192422024-02-03T17:18:05.203Z 23 460-479 173 173 173 296 296 296 F F F
192432024-02-03T17:18:05.203Z 24 480-499 170 170 170 293 293 293 F F F
192442024-02-03T17:18:05.203Z 25 500-519 177 177 177 304 304 304 F F F
192452024-02-03T17:18:05.203Z 26 520-539 177 177 177 304 304 304 F F F
192462024-02-03T17:18:05.203Z 27 540-559 173 173 173 296 296 296 F F F
192472024-02-03T17:18:05.203Z 28 560-579 170 170 170 293 293 293 F F F
192482024-02-03T17:18:05.203Z 29 580-599 177 177 177 304 304 304 F F F
192492024-02-03T17:18:05.203ZMax gen: 177, Max flush: 304
192502024-02-03T17:18:05.207Z
192512024-02-03T17:18:05.207ZBegin loop 89
192522024-02-03T17:18:05.207ZDownstairs to restart: 1
192532024-02-03T17:18:05.236Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.815526732Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847}
192542024-02-03T17:18:05.236Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.816134969Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847}
192552024-02-03T17:18:05.240Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":Crucible runtime is spawned
192562024-02-03T17:18:05.240Z30,"time":"2024-02-03T17:18:04.816363519Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847}
192572024-02-03T17:18:05.240Z{"msg":"Crucible caaf86bd-277c-40a5-9e31-cc5a0c3c437c has session id: b1c7863c-4d38-4703-96ca-9728c1f19cbf","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.816464711Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192582024-02-03T17:18:05.241Z{"msg":"Upstairs opts: Upstairs UUID: caaf86bd-277c-40a5-9e31-cc5a0c3c437c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.816531559Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192592024-02-03T17:18:05.241Z{"msg":"Crucible stats registered with UUID: caaf86bd-277c-40a5-9e31-cc5a0c3c437c","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.816612835Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192602024-02-03T17:18:05.241Z{"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.816769794Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847}
192612024-02-03T17:18:05.241Z{"msg":"caaf86bd-277c-40a5-9e31-cc5a0c3c437c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.816888521Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192622024-02-03T17:18:05.241Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.81693104Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192632024-02-03T17:18:05.241Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.816969883Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192642024-02-03T17:18:05.241Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.817009533Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192652024-02-03T17:18:05.241Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.817170409Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192662024-02-03T17:18:05.241Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.817227112Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192672024-02-03T17:18:05.241Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.817309495Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192682024-02-03T17:18:05.241Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.81735454Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"":"io task","client":"0","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192692024-02-03T17:18:05.241Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.817434349Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"":"io task","client":"2","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192702024-02-03T17:18:05.241Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.817477977Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"":"io task","client":"1","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192712024-02-03T17:18:05.241Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.817561309Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"":"io task","client":"0","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192722024-02-03T17:18:05.241Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.81760385Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"":"io task","client":"1","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192732024-02-03T17:18:05.241Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.817680359Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"":"io task","client":"2","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192742024-02-03T17:18:05.241Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.818593118Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192752024-02-03T17:18:05.241Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.818692971Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192762024-02-03T17:18:05.241Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.818904781Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192772024-02-03T17:18:05.241Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.81897674Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192782024-02-03T17:18:05.241Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.819546177Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192792024-02-03T17:18:05.247Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.819795678Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192802024-02-03T17:18:05.247Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.820047817Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192812024-02-03T17:18:05.247Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.820252064Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192822024-02-03T17:18:05.247Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.820447132Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192832024-02-03T17:18:05.247Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.820494278Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192842024-02-03T17:18:05.247Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.820608872Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192852024-02-03T17:18:05.247Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.820748276Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192862024-02-03T17:18:05.247Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.820809784Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192872024-02-03T17:18:05.247Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.820834965Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192882024-02-03T17:18:05.247Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.821193464Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192892024-02-03T17:18:05.247Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.821917731Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192902024-02-03T17:18:05.247Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.822025377Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192912024-02-03T17:18:05.247Z{"msg":"[0]R flush_numbers[0..12]: [277, 299, 299, 303, 287, 298, 293, 295, 299, 298, 304, 304]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.822072236Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192922024-02-03T17:18:05.247Z{"msg":"[0]R generation[0..12]: [160, 174, 174, 176, 167, 174, 170, 172, 174, 174, 177, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.822107773Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192932024-02-03T17:18:05.247Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.822166013Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192942024-02-03T17:18:05.247Z{"msg":"[1]R flush_numbers[0..12]: [277, 299, 299, 303, 287, 298, 293, 295, 299, 298, 304, 304]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.8221936Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192952024-02-03T17:18:05.247Z{"msg":"[1]R generation[0..12]: [160, 174, 174, 176, 167, 174, 170, 172, 174, 174, 177, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.822217201Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192962024-02-03T17:18:05.247Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
192972024-02-03T17:18:05.247Z,"time":"2024-02-03T17:18:04.822290221Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192982024-02-03T17:18:05.247Z{"msg":"[2]R flush_numbers[0..12]: [277, 299, 299, 303, 287, 298, 293, 295, 299, 298, 304, 304]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.822341949Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
192992024-02-03T17:18:05.248Z{"msg":"[2]R generation[0..12]: [160, 174, 174, 176, 167, 174, 170, 172, 174, 174, 177, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.822367256Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
193002024-02-03T17:18:05.248Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.822418081Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
193012024-02-03T17:18:05.248Z{"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.822442198Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
193022024-02-03T17:18:05.248Z{"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.822489448Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
193032024-02-03T17:18:05.248Z{"msg":"Next flush: 305","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.822512858Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
193042024-02-03T17:18:05.248Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.822535933Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
193052024-02-03T17:18:05.248Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.822558216Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
193062024-02-03T17:18:05.248Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.822590936Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
193072024-02-03T17:18:05.248Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.822631328Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
193082024-02-03T17:18:05.248Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.822656598Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
193092024-02-03T17:18:05.248Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.822723659Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
193102024-02-03T17:18:05.248Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.822753596Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
193112024-02-03T17:18:05.248Z{"msg":"caaf86bd-277c-40a5-9e31-cc5a0c3c437c is now active with session: b1c7863c-4d38-4703-96ca-9728c1f19cbf","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.822818576Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
193122024-02-03T17:18:05.248Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.82284821Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847,"session_id":"b1c7863c-4d38-4703-96ca-9728c1f19cbf"}
193132024-02-03T17:18:05.248Z{"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:04.822909395Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847}
193142024-02-03T17:18:05.248ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
193152024-02-03T17:18:05.248ZLoading write count information from file "/tmp/test_repair_verify.data"
193162024-02-03T17:18:05.248ZRun Repair workload
193172024-02-03T17:18:05.248Z01/30 Write block 543 len 8192 data: 13 11
193182024-02-03T17:18:05.252Z02/30 Read block 452 len 8192
193192024-02-03T17:18:05.252Z03/30 Write block 181 len 12288 data: 16 15 16
193202024-02-03T17:18:05.252Z04/30 Write block 392 len 20480 data: 20 19 17 16 16
193212024-02-03T17:18:05.252Z05/30 Write block 1 len 36864 data: 5 9 10 11 11 13 11 16 18
193222024-02-03T17:18:05.255Z06/30 Write block 575 len 24576 data: 13 13 10 10 11 10
193232024-02-03T17:18:05.255Z07/30 Read block 396 len 28672
193242024-02-03T17:18:05.258Z08/30 Flush
193252024-02-03T17:18:05.262Z09/30 Write block 117 len 12288 data: 18 19 19
193262024-02-03T17:18:05.262Z10/30 Read block 423 len 16384
193272024-02-03T17:18:05.265Z11/30 Read block 205 len 32768
193282024-02-03T17:18:05.265Z12/30 Flush
193292024-02-03T17:18:05.268Z13/30 Write block 277 len 36864 data: 14 14 15 14 16 17 19 15 16
193302024-02-03T17:18:05.271Z14/30 Write block 491 len 40960 data: 9 8 9 10 11 10 11 11 9 7
193312024-02-03T17:18:05.271Z15/30 Read block 554 len 4096
193322024-02-03T17:18:05.274Z16/30 Write block 203 len 8192 data: 12 11
193332024-02-03T17:18:05.274Z17/30 Read block 240 len 8192
193342024-02-03T17:18:05.277Z18/30 Read block 85 len 8192
193352024-02-03T17:18:05.277Z19/30 Write block 596 len 8192 data: 12 8
193362024-02-03T17:18:05.277Z20/30 Read block 466 len 24576
193372024-02-03T17:18:05.280Z21/30 Read block 315 len 24576
193382024-02-03T17:18:05.280Z22/30 Read block 67 len 28672
193392024-02-03T17:18:05.284Z23/30 Write block 446 len 12288 data: 17 17 16
193402024-02-03T17:18:05.284Z24/30 Read block 586 len 32768
193412024-02-03T17:18:05.291Z25/30 Write block 457 len 32768 data: 9 12 14 18 16 13 15 14
193422024-02-03T17:18:05.291Z26/30 Read block 170 len 32768
193432024-02-03T17:18:05.291Z27/30 Read block 405 len 28672
193442024-02-03T17:18:05.297Z28/30 Read block 136 len 36864
193452024-02-03T17:18:05.298Z29/30 Write block 48 len 20480 data: 13 15 15 15 17
193462024-02-03T17:18:05.298Z30/30 Read block 338 len 28672
193472024-02-03T17:18:05.298Z----------------------------------------------------------------
193482024-02-03T17:18:05.298Z Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30
193492024-02-03T17:18:05.298ZStates: Active Active Active
193502024-02-03T17:18:05.298ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
193512024-02-03T17:18:05.298Z 1 Acked 1000 Write 2 Done Done Done false
193522024-02-03T17:18:05.298Z 2 Acked 1001 Read 2 Done Done Done false
193532024-02-03T17:18:05.298Z 3 Acked 1002 Write 3 Done Done Done false
193542024-02-03T17:18:05.298Z 4 Acked 1003 Write 5 Done Done Done false
193552024-02-03T17:18:05.298Z 5 Acked 1004 Write 9 Done Done Done false
193562024-02-03T17:18:05.298Z 6 Acked 1005 Write 6 Done Done Done false
193572024-02-03T17:18:05.299Z 7 Acked 1006 Read 7 Done Sent Done false
193582024-02-03T17:18:05.299Z 8 Acked 1007 Flush 0 Done Sent Done false
193592024-02-03T17:18:05.299Z 9 Acked 1008 Write 3 Done Sent Done false
193602024-02-03T17:18:05.299Z 10 Acked 1009 Read 4 Done Sent Done false
193612024-02-03T17:18:05.299Z 11 Acked 1010 Read 8 Done Sent Done false
193622024-02-03T17:18:05.299Z 12 Acked 1011 Flush 0 Done Sent Done false
193632024-02-03T17:18:05.299Z 13 Acked 1012 Write 9 Done Sent Done false
193642024-02-03T17:18:05.299Z 14 Acked 1013 Write 10 Done Sent Done false
193652024-02-03T17:18:05.299Z 15 Acked 1014 Read 1 Done Sent Done false
193662024-02-03T17:18:05.299Z 16 Acked 1015 Write 2 Done Sent Done false
193672024-02-03T17:18:05.299Z 17 Acked 1016 Read 2 Done Sent Done false
193682024-02-03T17:18:05.299Z 18 Acked 1017 Read 2 Done Sent Done false
193692024-02-03T17:18:05.299Z 19 Acked 1018 Write 2 Done Sent Done false
193702024-02-03T17:18:05.300Z 20 Acked 1019 Read 6 Done Sent Done false
193712024-02-03T17:18:05.300Z 21 Acked 1020 Read 6 Done Sent Done false
193722024-02-03T17:18:05.300Z 22 Acked 1021 Read 7 Done Sent Done false
193732024-02-03T17:18:05.300Z 23 Acked 1022 Write 3 Done Sent Done false
193742024-02-03T17:18:05.300Z 24 Acked 1023 Read 8 Done Sent Done false
193752024-02-03T17:18:05.300Z 25 Acked 1024 Write 8 Done Sent Done false
193762024-02-03T17:18:05.300Z 26 Acked 1025 Read 8 Done Sent Done false
193772024-02-03T17:18:05.300Z 27 Acked 1026 Read 7 Done Sent Done false
193782024-02-03T17:18:05.300Z 28 Acked 1027 Read 9 Done Sent Done false
193792024-02-03T17:18:05.300Z 29 Acked 1028 Write 5 Done Sent Done false
193802024-02-03T17:18:05.300Z 30 Acked 1029 Read 7 Done Sent Done false
193812024-02-03T17:18:05.300Z STATES DS:0 DS:1 DS:2 TOTAL
193822024-02-03T17:18:05.300Z New 0 0 0 0
193832024-02-03T17:18:05.300Z Sent 0 24 0 24
193842024-02-03T17:18:05.301Z Done 30 6 30 66
193852024-02-03T17:18:05.301Z Skipped 0 0 0 0
193862024-02-03T17:18:05.301Z Error 0 0 0 0
193872024-02-03T17:18:05.301ZLast Flush: 1011 0 1011
193882024-02-03T17:18:05.301ZDownstairs last five completed:
193892024-02-03T17:18:05.301ZUpstairs last five completed: 30 29 28 27 26
193902024-02-03T17:18:05.301Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:18:04.876220895Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1847}
193912024-02-03T17:18:05.301ZWrote out file "/tmp/test_repair_verify.data"
193922024-02-03T17:18:05.305Z
193932024-02-03T17:18:05.310ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
193942024-02-03T17:18:05.754ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
193952024-02-03T17:18:05.755Z 0 000-019 178 160 178 305 277 305 F T F <---
193962024-02-03T17:18:05.755Z 1 020-039 174 174 174 299 299 299 F F F
193972024-02-03T17:18:05.755Z 2 040-059 174 174 174 299 299 299 T F T <---
193982024-02-03T17:18:05.755Z 3 060-079 176 176 176 303 303 303 F F F
193992024-02-03T17:18:05.755Z 4 080-099 167 167 167 287 287 287 F F F
194002024-02-03T17:18:05.755Z 5 100-119 178 174 178 306 298 306 F F F <---
194012024-02-03T17:18:05.755Z 6 120-139 170 170 170 293 293 293 F F F
194022024-02-03T17:18:05.759Z 7 140-159 172 172 172 295 295 295 F F F
194032024-02-03T17:18:05.759Z 8 160-179 174 174 174 299 299 299 F F F
194042024-02-03T17:18:05.759Z 9 180-199 178 174 178 305 298 305 F T F <---
194052024-02-03T17:18:05.759Z 10 200-219 177 177 177 304 304 304 T F T <---
194062024-02-03T17:18:05.760Z 11 220-239 177 177 177 304 304 304 F F F
194072024-02-03T17:18:05.760Z 12 240-259 170 170 170 293 293 293 F F F
194082024-02-03T17:18:05.760Z 13 260-279 174 174 174 298 298 298 T F T <---
194092024-02-03T17:18:05.760Z 14 280-299 174 174 174 297 297 297 T F T <---
194102024-02-03T17:18:05.760Z 15 300-319 177 177 177 304 304 304 F F F
194112024-02-03T17:18:05.760Z 16 320-339 177 177 177 304 304 304 F F F
194122024-02-03T17:18:05.760Z 17 340-359 176 176 176 302 302 302 F F F
194132024-02-03T17:18:05.760Z 18 360-379 177 177 177 304 304 304 F F F
194142024-02-03T17:18:05.760Z 19 380-399 178 177 178 305 304 305 F T F <---
194152024-02-03T17:18:05.760Z 20 400-419 177 177 177 304 304 304 F F F
194162024-02-03T17:18:05.760Z 21 420-439 166 166 166 286 286 286 F F F
194172024-02-03T17:18:05.760Z 22 440-459 174 174 174 299 299 299 T F T <---
194182024-02-03T17:18:05.761Z 23 460-479 173 173 173 296 296 296 T F T <---
194192024-02-03T17:18:05.761Z 24 480-499 170 170 170 293 293 293 T F T <---
194202024-02-03T17:18:05.761Z 25 500-519 177 177 177 304 304 304 T F T <---
194212024-02-03T17:18:05.761Z 26 520-539 177 177 177 304 304 304 F F F
194222024-02-03T17:18:05.761Z 27 540-559 178 173 178 305 296 305 F T F <---
194232024-02-03T17:18:05.761Z 28 560-579 178 170 178 305 293 305 F T F <---
194242024-02-03T17:18:05.761Z 29 580-599 178 177 178 305 304 305 T T T <---
194252024-02-03T17:18:05.761ZMax gen: 178, Max flush: 306
194262024-02-03T17:18:05.761ZError: Difference in extent metadata found!
194272024-02-03T17:18:05.761ZA Difference in extent metadata is expected here
194282024-02-03T17:18:05.761ZOn loop 89
194292024-02-03T17:18:05.761Z
194302024-02-03T17:18:05.761ZVerifying data now
194312024-02-03T17:18:11.643ZNo change after verify
194322024-02-03T17:18:11.643ZLoop: 89 Downstairs dump after verify (and repair):
194332024-02-03T17:18:11.666ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
194342024-02-03T17:18:11.666Z 0 000-019 178 178 178 305 305 305 F F F
194352024-02-03T17:18:11.669Z 1 020-039 174 174 174 299 299 299 F F F
194362024-02-03T17:18:11.669Z 2 040-059 179 179 179 307 307 307 F F F
194372024-02-03T17:18:11.669Z 3 060-079 176 176 176 303 303 303 F F F
194382024-02-03T17:18:11.670Z 4 080-099 167 167 167 287 287 287 F F F
194392024-02-03T17:18:11.670Z 5 100-119 178 178 178 306 306 306 F F F
194402024-02-03T17:18:11.670Z 6 120-139 170 170 170 293 293 293 F F F
194412024-02-03T17:18:11.670Z 7 140-159 172 172 172 295 295 295 F F F
194422024-02-03T17:18:11.670Z 8 160-179 174 174 174 299 299 299 F F F
194432024-02-03T17:18:11.670Z 9 180-199 178 178 178 305 305 305 F F F
194442024-02-03T17:18:11.670Z 10 200-219 179 179 179 307 307 307 F F F
194452024-02-03T17:18:11.670Z 11 220-239 177 177 177 304 304 304 F F F
194462024-02-03T17:18:11.670Z 12 240-259 170 170 170 293 293 293 F F F
194472024-02-03T17:18:11.670Z 13 260-279 179 179 179 307 307 307 F F F
194482024-02-03T17:18:11.670Z 14 280-299 179 179 179 307 307 307 F F F
194492024-02-03T17:18:11.670Z 15 300-319 177 177 177 304 304 304 F F F
194502024-02-03T17:18:11.670Z 16 320-339 177 177 177 304 304 304 F F F
194512024-02-03T17:18:11.670Z 17 340-359 176 176 176 302 302 302 F F F
194522024-02-03T17:18:11.670Z 18 360-379 177 177 177 304 304 304 F F F
194532024-02-03T17:18:11.670Z 19 380-399 178 178 178 305 305 305 F F F
194542024-02-03T17:18:11.670Z 20 400-419 177 177 177 304 304 304 F F F
194552024-02-03T17:18:11.670Z 21 420-439 166 166 166 286 286 286 F F F
194562024-02-03T17:18:11.670Z 22 440-459 179 179 179 307 307 307 F F F
194572024-02-03T17:18:11.670Z 23 460-479 179 179 179 307 307 307 F F F
194582024-02-03T17:18:11.670Z 24 480-499 179 179 179 307 307 307 F F F
194592024-02-03T17:18:11.670Z 25 500-519 179 179 179 307 307 307 F F F
194602024-02-03T17:18:11.670Z 26 520-539 177 177 177 304 304 304 F F F
194612024-02-03T17:18:11.670Z 27 540-559 178 178 178 305 305 305 F F F
194622024-02-03T17:18:11.671Z 28 560-579 178 178 178 305 305 305 F F F
194632024-02-03T17:18:11.671Z 29 580-599 179 179 179 307 307 307 F F F
194642024-02-03T17:18:11.671ZMax gen: 179, Max flush: 307
194652024-02-03T17:18:11.674Z
194662024-02-03T17:18:11.674ZBegin loop 90
194672024-02-03T17:18:11.674ZDownstairs to restart: 0
194682024-02-03T17:18:11.702Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.282133848Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855}
194692024-02-03T17:18:11.706ZCrucible runtime is spawned
194702024-02-03T17:18:11.706Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.282946561Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855}
194712024-02-03T17:18:11.706Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.283023296Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855}
194722024-02-03T17:18:11.706Z{"msg":"Crucible 42a14947-8be4-4ecf-a19b-c317ae115cb3 has session id: 370049d1-79f8-4250-be21-ce79aa6bc2b9","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.283048844Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
194732024-02-03T17:18:11.706Z{"msg":"Upstairs opts: Upstairs UUID: 42a14947-8be4-4ecf-a19b-c317ae115cb3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.283092117Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
194742024-02-03T17:18:11.706Z{"msg":"Crucible stats registered with UUID: 42a14947-8be4-4ecf-a19b-c317ae115cb3","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.283118565Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
194752024-02-03T17:18:11.706Z{"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.28313934Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855}
194762024-02-03T17:18:11.706Z{"msg":"42a14947-8be4-4ecf-a19b-c317ae115cb3 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.283171133Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
194772024-02-03T17:18:11.706Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.283192464Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
194782024-02-03T17:18:11.706Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.283287499Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
194792024-02-03T17:18:11.706Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.283343193Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
194802024-02-03T17:18:11.706Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.283368674Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
194812024-02-03T17:18:11.706Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.283391252Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
194822024-02-03T17:18:11.706Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.283415927Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
194832024-02-03T17:18:11.706Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.283445202Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"":"io task","client":"2","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
194842024-02-03T17:18:11.706Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.283651486Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"":"io task","client":"0","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
194852024-02-03T17:18:11.706Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.283749193Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"":"io task","client":"1","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
194862024-02-03T17:18:11.706Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.283931618Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"":"io task","client":"0","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
194872024-02-03T17:18:11.706Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.284058776Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"":"io task","client":"2","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
194882024-02-03T17:18:11.706Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.284105518Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"":"io task","client":"1","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
194892024-02-03T17:18:11.706Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.285373762Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
194902024-02-03T17:18:11.707Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.285547853Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
194912024-02-03T17:18:11.707Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.285635323Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
194922024-02-03T17:18:11.707Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.285720919Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
194932024-02-03T17:18:11.712Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.286586488Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
194942024-02-03T17:18:11.712Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.286657629Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
194952024-02-03T17:18:11.712Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.2869096Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
194962024-02-03T17:18:11.712Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.287179628Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
194972024-02-03T17:18:11.712Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.287653938Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
194982024-02-03T17:18:11.712Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.2879965Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
194992024-02-03T17:18:11.712Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.288091285Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
195002024-02-03T17:18:11.712Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.288121289Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
195012024-02-03T17:18:11.712Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.288197853Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
195022024-02-03T17:18:11.712Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.288223302Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
195032024-02-03T17:18:11.712Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.288245697Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
195042024-02-03T17:18:11.712Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.288437711Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
195052024-02-03T17:18:11.712Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.288522434Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
195062024-02-03T17:18:11.712Z{"msg":"[0]R flush_numbers[0..12]: [305, 299, 307, 303, 287, 306, 293, 295, 299, 305, 307, 304]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.288554262Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
195072024-02-03T17:18:11.712Z{"msg":"[0]R generation[0..12]: [178, 174, 179, 176, 167, 178, 170, 172, 174, 178, 179, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.288606869Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
195082024-02-03T17:18:11.712Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.288633188Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
195092024-02-03T17:18:11.712Z{"msg":"[1]R flush_numbers[0..12]: [305, 299, 307, 303, 287, 306, 293, 295, 299, 305, 307, 304]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.288683723Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
195102024-02-03T17:18:11.712Z{"msg":"[1]R generation[0..12]: [178, 174, 179, 176, 167, 178, 170, 172, 174, 178, 179, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.288708994Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
195112024-02-03T17:18:11.712Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
195122024-02-03T17:18:11.713Z,"time":"2024-02-03T17:18:11.288775667Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
195132024-02-03T17:18:11.713Z{"msg":"[2]R flush_numbers[0..12]: [305, 299, 307, 303, 287, 306, 293, 295, 299, 305, 307, 304]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.288859374Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
195142024-02-03T17:18:11.713Z{"msg":"[2]R generation[0..12]: [178, 174, 179, 176, 167, 178, 170, 172, 174, 178, 179, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.288924421Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
195152024-02-03T17:18:11.713Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.288951211Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
195162024-02-03T17:18:11.713Z{"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.289000248Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
195172024-02-03T17:18:11.713Z{"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.289030152Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
195182024-02-03T17:18:11.713Z{"msg":"Next flush: 308","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.289106341Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
195192024-02-03T17:18:11.713Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.289134717Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
195202024-02-03T17:18:11.713Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.289213379Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
195212024-02-03T17:18:11.713Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.2892452Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
195222024-02-03T17:18:11.713Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.289297748Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
195232024-02-03T17:18:11.713Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.289327588Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
195242024-02-03T17:18:11.713Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.289407701Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
195252024-02-03T17:18:11.713Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.289475159Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
195262024-02-03T17:18:11.713Z{"msg":"42a14947-8be4-4ecf-a19b-c317ae115cb3 is now active with session: 370049d1-79f8-4250-be21-ce79aa6bc2b9","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.289508097Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
195272024-02-03T17:18:11.713Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.289582084Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855,"session_id":"370049d1-79f8-4250-be21-ce79aa6bc2b9"}
195282024-02-03T17:18:11.713Z{"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:11.289654714Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855}
195292024-02-03T17:18:11.713ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
195302024-02-03T17:18:11.713ZLoading write count information from file "/tmp/test_repair_verify.data"
195312024-02-03T17:18:11.713ZRun Repair workload
195322024-02-03T17:18:11.713Z01/30 Write block 486 len 4096 data: 12
195332024-02-03T17:18:11.722Z02/30 Flush
195342024-02-03T17:18:11.722Z03/30 Write block 270 len 12288 data: 15 14 15
195352024-02-03T17:18:11.722Z04/30 Flush
195362024-02-03T17:18:11.726Z05/30 Write block 266 len 12288 data: 18 18 22
195372024-02-03T17:18:11.726Z06/30 Flush
195382024-02-03T17:18:11.726Z07/30 Write block 242 len 12288 data: 10 10 10
195392024-02-03T17:18:11.726Z08/30 Read block 57 len 20480
195402024-02-03T17:18:11.726Z09/30 Read block 139 len 8192
195412024-02-03T17:18:11.729Z10/30 Read block 130 len 16384
195422024-02-03T17:18:11.729Z11/30 Flush
195432024-02-03T17:18:11.732Z12/30 Write block 553 len 36864 data: 17 17 14 15 14 11 12 12 13
195442024-02-03T17:18:11.732Z13/30 Write block 88 len 24576 data: 11 13 14 14 13 15
195452024-02-03T17:18:11.735Z14/30 Flush
195462024-02-03T17:18:11.740Z15/30 Write block 568 len 32768 data: 17 13 14 13 12 12 14 14
195472024-02-03T17:18:11.740Z16/30 Flush
195482024-02-03T17:18:11.743Z17/30 Write block 9 len 40960 data: 19 17 11 13 14 16 14 13 16 18
195492024-02-03T17:18:11.743Z18/30 Read block 429 len 8192
195502024-02-03T17:18:11.746Z19/30 Write block 462 len 36864 data: 14 16 15 12 15 14 13 13 13
195512024-02-03T17:18:11.746Z20/30 Write block 340 len 12288 data: 15 13 13
195522024-02-03T17:18:11.746Z21/30 Read block 332 len 4096
195532024-02-03T17:18:11.752Z22/30 Read block 67 len 16384
195542024-02-03T17:18:11.752Z23/30 Write block 485 len 4096 data: 11
195552024-02-03T17:18:11.755Z24/30 Write block 20 len 36864 data: 20 23 19 18 17 14 12 13 14
195562024-02-03T17:18:11.755Z25/30 Write block 92 len 36864 data: 14 16 16 14 14 13 12 13 14
195572024-02-03T17:18:11.755Z26/30 Read block 263 len 24576
195582024-02-03T17:18:11.755Z27/30 Read block 106 len 24576
195592024-02-03T17:18:11.758Z28/30 Write block 15 len 36864 data: 15 14 17 19 24 21 24 20 19
195602024-02-03T17:18:11.758Z29/30 Read block 92 len 40960
195612024-02-03T17:18:11.761Z30/30 Write block 576 len 40960 data: 14 11 11 12 11 12 11 9 8 10
195622024-02-03T17:18:11.765Z----------------------------------------------------------------
195632024-02-03T17:18:11.765Z Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:30
195642024-02-03T17:18:11.765ZStates: Active Active Active
195652024-02-03T17:18:11.765ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
195662024-02-03T17:18:11.765Z 1 Acked 1000 Write 1 Sent Done Done false
195672024-02-03T17:18:11.765Z 2 Acked 1001 Flush 0 Sent Done Done false
195682024-02-03T17:18:11.765Z 3 Acked 1002 Write 3 Sent Done Done false
195692024-02-03T17:18:11.765Z 4 Acked 1003 Flush 0 Sent Done Done false
195702024-02-03T17:18:11.765Z 5 Acked 1004 Write 3 Sent Done Done false
195712024-02-03T17:18:11.765Z 6 Acked 1005 Flush 0 Sent Done Done false
195722024-02-03T17:18:11.765Z 7 Acked 1006 Write 3 Sent Done Done false
195732024-02-03T17:18:11.765Z 8 Acked 1007 Read 5 Sent Done Done false
195742024-02-03T17:18:11.765Z 9 Acked 1008 Read 2 Sent Done Done false
195752024-02-03T17:18:11.765Z 10 Acked 1009 Read 4 Sent Done Done false
195762024-02-03T17:18:11.765Z 11 Acked 1010 Flush 0 Sent Done Done false
195772024-02-03T17:18:11.765Z 12 Acked 1011 Write 9 Sent Done Done false
195782024-02-03T17:18:11.765Z 13 Acked 1012 Write 6 Sent Done Done false
195792024-02-03T17:18:11.765Z 14 Acked 1013 Flush 0 Sent Done Done false
195802024-02-03T17:18:11.765Z 15 Acked 1014 Write 8 Sent Done Done false
195812024-02-03T17:18:11.765Z 16 Acked 1015 Flush 0 Sent Done Done false
195822024-02-03T17:18:11.765Z 17 Acked 1016 Write 10 Sent Done Done false
195832024-02-03T17:18:11.766Z 18 Acked 1017 Read 2 Sent Done Done false
195842024-02-03T17:18:11.766Z 19 Acked 1018 Write 9 Sent Done Done false
195852024-02-03T17:18:11.766Z 20 Acked 1019 Write 3 Sent Done Done false
195862024-02-03T17:18:11.766Z 21 Acked 1020 Read 1 Sent Done Done false
195872024-02-03T17:18:11.766Z 22 Acked 1021 Read 4 Sent Done Done false
195882024-02-03T17:18:11.766Z 23 Acked 1022 Write 1 Sent Done Done false
195892024-02-03T17:18:11.766Z 24 Acked 1023 Write 9 Sent Done Done false
195902024-02-03T17:18:11.766Z 25 Acked 1024 Write 9 Sent Done Done false
195912024-02-03T17:18:11.766Z 26 Acked 1025 Read 6 Sent Done Done false
195922024-02-03T17:18:11.766Z 27 Acked 1026 Read 6 Sent Done Done false
195932024-02-03T17:18:11.766Z 28 Acked 1027 Write 9 Sent Done Done false
195942024-02-03T17:18:11.766Z 29 Acked 1028 Read 10 Sent Done Done false
195952024-02-03T17:18:11.766Z 30 Acked 1029 Write 10 Sent Sent Sent false
195962024-02-03T17:18:11.766Z STATES DS:0 DS:1 DS:2 TOTAL
195972024-02-03T17:18:11.766Z New 0 0 0 0
195982024-02-03T17:18:11.766Z Sent 30 1 1 32
195992024-02-03T17:18:11.766Z Done 0 29 29 58
196002024-02-03T17:18:11.766Z Skipped 0 0 0 0
196012024-02-03T17:18:11.766Z Error 0 0 0 0
196022024-02-03T17:18:11.766ZLast Flush: 0 1015 1015
196032024-02-03T17:18:11.766ZDownstairs last five completed:
196042024-02-03T17:18:11.766ZUpstairs last five completed: 30 29 28 27 26
196052024-02-03T17:18:11.766Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:18:11.342872455Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1855}
196062024-02-03T17:18:11.766ZWrote out file "/tmp/test_repair_verify.data"
196072024-02-03T17:18:11.770Z
196082024-02-03T17:18:11.775ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
196092024-02-03T17:18:11.803ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
196102024-02-03T17:18:11.803Z 0 000-019 178 178 178 305 305 305 F T T <---
196112024-02-03T17:18:11.803Z 1 020-039 174 174 174 299 299 299 F T T <---
196122024-02-03T17:18:11.803Z 2 040-059 179 179 179 307 307 307 F F F
196132024-02-03T17:18:11.804Z 3 060-079 176 176 176 303 303 303 F F F
196142024-02-03T17:18:11.807Z 4 080-099 167 180 180 287 312 312 F T T <---
196152024-02-03T17:18:11.807Z 5 100-119 178 178 178 306 306 306 F T T <---
196162024-02-03T17:18:11.807Z 6 120-139 170 170 170 293 293 293 F F F
196172024-02-03T17:18:11.807Z 7 140-159 172 172 172 295 295 295 F F F
196182024-02-03T17:18:11.807Z 8 160-179 174 174 174 299 299 299 F F F
196192024-02-03T17:18:11.807Z 9 180-199 178 178 178 305 305 305 F F F
196202024-02-03T17:18:11.807Z 10 200-219 179 179 179 307 307 307 F F F
196212024-02-03T17:18:11.807Z 11 220-239 177 177 177 304 304 304 F F F
196222024-02-03T17:18:11.807Z 12 240-259 170 180 180 293 311 311 F F F <---
196232024-02-03T17:18:11.807Z 13 260-279 179 180 180 307 310 310 F F F <---
196242024-02-03T17:18:11.807Z 14 280-299 179 179 179 307 307 307 F F F
196252024-02-03T17:18:11.807Z 15 300-319 177 177 177 304 304 304 F F F
196262024-02-03T17:18:11.807Z 16 320-339 177 177 177 304 304 304 F F F
196272024-02-03T17:18:11.807Z 17 340-359 176 176 176 302 302 302 F T T <---
196282024-02-03T17:18:11.807Z 18 360-379 177 177 177 304 304 304 F F F
196292024-02-03T17:18:11.807Z 19 380-399 178 178 178 305 305 305 F F F
196302024-02-03T17:18:11.807Z 20 400-419 177 177 177 304 304 304 F F F
196312024-02-03T17:18:11.808Z 21 420-439 166 166 166 286 286 286 F F F
196322024-02-03T17:18:11.808Z 22 440-459 179 179 179 307 307 307 F F F
196332024-02-03T17:18:11.808Z 23 460-479 179 179 179 307 307 307 F T T <---
196342024-02-03T17:18:11.808Z 24 480-499 179 180 180 307 308 308 F T T <---
196352024-02-03T17:18:11.808Z 25 500-519 179 179 179 307 307 307 F F F
196362024-02-03T17:18:11.808Z 26 520-539 177 177 177 304 304 304 F F F
196372024-02-03T17:18:11.808Z 27 540-559 178 180 180 305 312 312 F F F <---
196382024-02-03T17:18:11.808Z 28 560-579 178 180 180 305 313 313 F T T <---
196392024-02-03T17:18:11.808Z 29 580-599 179 179 179 307 307 307 F T T <---
196402024-02-03T17:18:11.808ZMax gen: 180, Max flush: 313
196412024-02-03T17:18:11.808ZError: Difference in extent metadata found!
196422024-02-03T17:18:11.811ZA Difference in extent metadata is expected here
196432024-02-03T17:18:11.811ZOn loop 90
196442024-02-03T17:18:11.811Z
196452024-02-03T17:18:11.814ZVerifying data now
196462024-02-03T17:18:16.887ZNo change after verify
196472024-02-03T17:18:16.888ZLoop: 90 Downstairs dump after verify (and repair):
196482024-02-03T17:18:16.908ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
196492024-02-03T17:18:16.908Z 0 000-019 181 181 181 314 314 314 F F F
196502024-02-03T17:18:16.913Z 1 020-039 181 181 181 314 314 314 F F F
196512024-02-03T17:18:16.913Z 2 040-059 179 179 179 307 307 307 F F F
196522024-02-03T17:18:16.913Z 3 060-079 176 176 176 303 303 303 F F F
196532024-02-03T17:18:16.913Z 4 080-099 181 181 181 314 314 314 F F F
196542024-02-03T17:18:16.913Z 5 100-119 181 181 181 314 314 314 F F F
196552024-02-03T17:18:16.913Z 6 120-139 170 170 170 293 293 293 F F F
196562024-02-03T17:18:16.913Z 7 140-159 172 172 172 295 295 295 F F F
196572024-02-03T17:18:16.913Z 8 160-179 174 174 174 299 299 299 F F F
196582024-02-03T17:18:16.913Z 9 180-199 178 178 178 305 305 305 F F F
196592024-02-03T17:18:16.914Z 10 200-219 179 179 179 307 307 307 F F F
196602024-02-03T17:18:16.914Z 11 220-239 177 177 177 304 304 304 F F F
196612024-02-03T17:18:16.914Z 12 240-259 180 180 180 311 311 311 F F F
196622024-02-03T17:18:16.914Z 13 260-279 180 180 180 310 310 310 F F F
196632024-02-03T17:18:16.914Z 14 280-299 179 179 179 307 307 307 F F F
196642024-02-03T17:18:16.914Z 15 300-319 177 177 177 304 304 304 F F F
196652024-02-03T17:18:16.914Z 16 320-339 177 177 177 304 304 304 F F F
196662024-02-03T17:18:16.914Z 17 340-359 181 181 181 314 314 314 F F F
196672024-02-03T17:18:16.914Z 18 360-379 177 177 177 304 304 304 F F F
196682024-02-03T17:18:16.914Z 19 380-399 178 178 178 305 305 305 F F F
196692024-02-03T17:18:16.914Z 20 400-419 177 177 177 304 304 304 F F F
196702024-02-03T17:18:16.914Z 21 420-439 166 166 166 286 286 286 F F F
196712024-02-03T17:18:16.914Z 22 440-459 179 179 179 307 307 307 F F F
196722024-02-03T17:18:16.915Z 23 460-479 181 181 181 314 314 314 F F F
196732024-02-03T17:18:16.915Z 24 480-499 181 181 181 314 314 314 F F F
196742024-02-03T17:18:16.915Z 25 500-519 179 179 179 307 307 307 F F F
196752024-02-03T17:18:16.915Z 26 520-539 177 177 177 304 304 304 F F F
196762024-02-03T17:18:16.915Z 27 540-559 180 180 180 312 312 312 F F F
196772024-02-03T17:18:16.915Z 28 560-579 181 181 181 314 314 314 F F F
196782024-02-03T17:18:16.915Z 29 580-599 181 181 181 314 314 314 F F F
196792024-02-03T17:18:16.915ZMax gen: 181, Max flush: 314
196802024-02-03T17:18:16.919Z
196812024-02-03T17:18:16.919ZBegin loop 91
196822024-02-03T17:18:16.919ZDownstairs to restart: 2
196832024-02-03T17:18:16.946Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.525859055Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863}
196842024-02-03T17:18:16.946Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.526457879Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863}
196852024-02-03T17:18:16.946Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.526543946Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863}
196862024-02-03T17:18:16.962Z{"msg":"Crucible 2c019933-f04f-469e-8ac4-0c8639bb1f17 has session id: bc48dca0-60d2-46b5-b787-9512242340cf","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.526586598Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
196872024-02-03T17:18:16.962Z{"msg":"Upstairs opts: Upstairs UUID: 2c019933-f04f-469e-8ac4-0c8639bb1f17, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.526627431Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
196882024-02-03T17:18:16.962Z{"msg":"Crucible stats registered with UUID: 2c019933-f04f-469e-8ac4-0c8639bb1f17","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.52665943Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
196892024-02-03T17:18:16.962ZCrucible runtime is spawned
196902024-02-03T17:18:16.962Z{"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.526940485Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863}
196912024-02-03T17:18:16.963Z{"msg":"2c019933-f04f-469e-8ac4-0c8639bb1f17 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.527122327Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
196922024-02-03T17:18:16.963Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.527219898Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
196932024-02-03T17:18:16.963Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.527269515Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
196942024-02-03T17:18:16.963Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.527308728Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
196952024-02-03T17:18:16.963Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.527350273Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
196962024-02-03T17:18:16.963Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.527388968Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
196972024-02-03T17:18:16.963Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.527470968Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
196982024-02-03T17:18:16.963Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.527515915Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"":"io task","client":"0","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
196992024-02-03T17:18:16.963Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.527562893Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"":"io task","client":"1","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197002024-02-03T17:18:16.963Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.527649506Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"":"io task","client":"2","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197012024-02-03T17:18:16.963Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.5277006Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"":"io task","client":"0","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197022024-02-03T17:18:16.963Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.527782761Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"":"io task","client":"2","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197032024-02-03T17:18:16.964Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.527826257Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"":"io task","client":"1","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197042024-02-03T17:18:16.964Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.528871082Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197052024-02-03T17:18:16.964Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.528942358Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197062024-02-03T17:18:16.964Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.529062924Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197072024-02-03T17:18:16.964Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.529150815Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197082024-02-03T17:18:16.964Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.52985914Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197092024-02-03T17:18:16.964Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.530002302Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197102024-02-03T17:18:16.964Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.530059656Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197112024-02-03T17:18:16.964Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.530105186Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197122024-02-03T17:18:16.970Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.531004632Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197132024-02-03T17:18:16.970Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.531167617Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197142024-02-03T17:18:16.970Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.531219097Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197152024-02-03T17:18:16.970Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.531304206Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197162024-02-03T17:18:16.970Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.531352296Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197172024-02-03T17:18:16.970Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.53142372Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197182024-02-03T17:18:16.970Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.531453021Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197192024-02-03T17:18:16.970Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.532017513Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197202024-02-03T17:18:16.970Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.532143064Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197212024-02-03T17:18:16.970Z{"msg":"[0]R flush_numbers[0..12]: [314, 314, 307, 303, 314, 314, 293, 295, 299, 305, 307, 304]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.532175657Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197222024-02-03T17:18:16.971Z{"msg":"[0]R generation[0..12]: [181, 181, 179, 176, 181, 181, 170, 172, 174, 178, 179, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.532243811Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197232024-02-03T17:18:16.971Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.532277978Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197242024-02-03T17:18:16.971Z{"msg":"[1]R flush_numbers[0..12]: [314, 314, 307, 303, 314, 314, 293, 295, 299, 305, 307, 304]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.532341056Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197252024-02-03T17:18:16.971Z{"msg":"[1]R generation[0..12]: [181, 181, 179, 176, 181, 181, 170, 172, 174, 178, 179, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.532372467Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"Wait for a query_work_queue command to finish before sending IO
197262024-02-03T17:18:16.971Z}
197272024-02-03T17:18:16.971Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.532490882Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197282024-02-03T17:18:16.971Z{"msg":"[2]R flush_numbers[0..12]: [314, 314, 307, 303, 314, 314, 293, 295, 299, 305, 307, 304]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.532551621Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197292024-02-03T17:18:16.971Z{"msg":"[2]R generation[0..12]: [181, 181, 179, 176, 181, 181, 170, 172, 174, 178, 179, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.532578235Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197302024-02-03T17:18:16.971Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.532644396Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197312024-02-03T17:18:16.971Z{"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.532671641Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197322024-02-03T17:18:16.971Z{"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.532737938Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197332024-02-03T17:18:16.971Z{"msg":"Next flush: 315","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.532830052Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197342024-02-03T17:18:16.971Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.53287329Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197352024-02-03T17:18:16.972Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.532913673Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197362024-02-03T17:18:16.972Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.532946611Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197372024-02-03T17:18:16.972Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.532975954Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197382024-02-03T17:18:16.972Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.533055001Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197392024-02-03T17:18:16.972Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.533082658Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197402024-02-03T17:18:16.972Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.533149658Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197412024-02-03T17:18:16.972Z{"msg":"2c019933-f04f-469e-8ac4-0c8639bb1f17 is now active with session: bc48dca0-60d2-46b5-b787-9512242340cf","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.533176319Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197422024-02-03T17:18:16.972Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.533237689Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863,"session_id":"bc48dca0-60d2-46b5-b787-9512242340cf"}
197432024-02-03T17:18:16.972Z{"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:16.533262978Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863}
197442024-02-03T17:18:16.972ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
197452024-02-03T17:18:16.972ZLoading write count information from file "/tmp/test_repair_verify.data"
197462024-02-03T17:18:16.972ZRun Repair workload
197472024-02-03T17:18:16.972Z01/30 Write block 361 len 28672 data: 13 11 11 12 14 12 14
197482024-02-03T17:18:16.973Z02/30 Read block 34 len 16384
197492024-02-03T17:18:16.973Z03/30 Write block 210 len 32768 data: 17 14 13 12 7 8 12 11
197502024-02-03T17:18:16.973Z04/30 Read block 38 len 8192
197512024-02-03T17:18:16.973Z05/30 Read block 148 len 28672
197522024-02-03T17:18:16.973Z06/30 Write block 492 len 40960 data: 9 10 11 12 11 12 12 10 8 9
197532024-02-03T17:18:16.973Z07/30 Write block 477 len 12288 data: 11 13 15
197542024-02-03T17:18:16.973Z08/30 Write block 50 len 36864 data: 16 16 18 13 12 13 14 12 10
197552024-02-03T17:18:16.973Z09/30 Read block 90 len 12288
197562024-02-03T17:18:16.977Z10/30 Read block 592 len 8192
197572024-02-03T17:18:16.977Z11/30 Read block 204 len 28672
197582024-02-03T17:18:16.977Z12/30 Write block 133 len 8192 data: 12 12
197592024-02-03T17:18:16.977Z13/30 Write block 456 len 32768 data: 8 10 13 15 19 17 15 17
197602024-02-03T17:18:16.977Z14/30 Write block 306 len 4096 data: 17
197612024-02-03T17:18:16.977Z15/30 Write block 123 len 32768 data: 16 14 11 11 13 14 15 14
197622024-02-03T17:18:16.977Z16/30 Read block 536 len 24576
197632024-02-03T17:18:16.981Z17/30 Read block 288 len 36864
197642024-02-03T17:18:16.981Z18/30 Read block 220 len 40960
197652024-02-03T17:18:16.985Z19/30 Write block 197 len 32768 data: 16 16 10 10 9 13 13 12
197662024-02-03T17:18:16.985Z20/30 Read block 33 len 16384
197672024-02-03T17:18:16.989Z21/30 Read block 228 len 16384
197682024-02-03T17:18:16.989Z22/30 Read block 488 len 8192
197692024-02-03T17:18:16.989Z23/30 Read block 468 len 16384
197702024-02-03T17:18:16.993Z24/30 Read block 582 len 36864
197712024-02-03T17:18:16.993Z25/30 Read block 133 len 4096
197722024-02-03T17:18:16.993Z26/30 Write block 581 len 24576 data: 13 12 10 9 11 14
197732024-02-03T17:18:16.998Z27/30 Read block 362 len 36864
197742024-02-03T17:18:16.998Z28/30 Read block 190 len 12288
197752024-02-03T17:18:16.998Z29/30 Write block 48 len 28672 data: 14 16 17 17 19 14 13
197762024-02-03T17:18:16.998Z30/30 Read block 570 len 40960
197772024-02-03T17:18:17.003Z----------------------------------------------------------------
197782024-02-03T17:18:17.003Z Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30
197792024-02-03T17:18:17.003ZStates: Active Active Active
197802024-02-03T17:18:17.003ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
197812024-02-03T17:18:17.003Z 1 Acked 1000 Write 7 Done Done Done false
197822024-02-03T17:18:17.003Z 2 Acked 1001 Read 4 Done Done Done false
197832024-02-03T17:18:17.003Z 3 Acked 1002 Write 8 Done Done Done false
197842024-02-03T17:18:17.003Z 4 Acked 1003 Read 2 Done Done Sent false
197852024-02-03T17:18:17.004Z 5 Acked 1004 Read 7 Done Done Sent false
197862024-02-03T17:18:17.004Z 6 Acked 1005 Write 10 Done Done Sent false
197872024-02-03T17:18:17.004Z 7 Acked 1006 Write 3 Done Done Sent false
197882024-02-03T17:18:17.004Z 8 Acked 1007 Write 9 Done Done Sent false
197892024-02-03T17:18:17.004Z 9 Acked 1008 Read 3 Done Done Sent false
197902024-02-03T17:18:17.004Z 10 Acked 1009 Read 2 Done Done Sent false
197912024-02-03T17:18:17.004Z 11 Acked 1010 Read 7 Done Done Sent false
197922024-02-03T17:18:17.004Z 12 Acked 1011 Write 2 Done Done Sent false
197932024-02-03T17:18:17.004Z 13 Acked 1012 Write 8 Done Done Sent false
197942024-02-03T17:18:17.004Z 14 Acked 1013 Write 1 Done Done Sent false
197952024-02-03T17:18:17.004Z 15 Acked 1014 Write 8 Done Done Sent false
197962024-02-03T17:18:17.004Z 16 Acked 1015 Read 6 Done Done Sent false
197972024-02-03T17:18:17.005Z 17 Acked 1016 Read 9 Done Done Sent false
197982024-02-03T17:18:17.005Z 18 Acked 1017 Read 10 Done Done Sent false
197992024-02-03T17:18:17.005Z 19 Acked 1018 Write 8 Done Done Sent false
198002024-02-03T17:18:17.005Z 20 Acked 1019 Read 4 Done Done Sent false
198012024-02-03T17:18:17.005Z 21 Acked 1020 Read 4 Done Done Sent false
198022024-02-03T17:18:17.005Z 22 Acked 1021 Read 2 Done Done Sent false
198032024-02-03T17:18:17.005Z 23 Acked 1022 Read 4 Done Done Sent false
198042024-02-03T17:18:17.005Z 24 Acked 1023 Read 9 Done Done Sent false
198052024-02-03T17:18:17.005Z 25 Acked 1024 Read 1 Done Done Sent false
198062024-02-03T17:18:17.005Z 26 Acked 1025 Write 6 Done Done Sent false
198072024-02-03T17:18:17.005Z 27 Acked 1026 Read 9 Done Done Sent false
198082024-02-03T17:18:17.005Z 28 Acked 1027 Read 3 Done Done Sent false
198092024-02-03T17:18:17.005Z 29 Acked 1028 Write 7 Done Done Sent false
198102024-02-03T17:18:17.006Z 30 Acked 1029 Read 10 Done Sent Sent false
198112024-02-03T17:18:17.006Z STATES DS:0 DS:1 DS:2 TOTAL
198122024-02-03T17:18:17.006Z New 0 0 0 0
198132024-02-03T17:18:17.006Z Sent 0 1 27 28
198142024-02-03T17:18:17.006Z Done 30 29 3 62
198152024-02-03T17:18:17.006Z Skipped 0 0 0 0
198162024-02-03T17:18:17.006Z Error 0 0 0 0
198172024-02-03T17:18:17.006ZLast Flush: 0 0 0
198182024-02-03T17:18:17.006ZDownstairs last five completed:
198192024-02-03T17:18:17.006ZUpstairs last five completed: 30 29 28 27 26
198202024-02-03T17:18:17.006Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:18:16.58160491Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1863}
198212024-02-03T17:18:17.006ZWrote out file "/tmp/test_repair_verify.data"
198222024-02-03T17:18:17.010Z
198232024-02-03T17:18:17.015ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
198242024-02-03T17:18:17.045ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
198252024-02-03T17:18:17.045Z 0 000-019 181 181 181 314 314 314 F F F
198262024-02-03T17:18:17.045Z 1 020-039 181 181 181 314 314 314 F F F
198272024-02-03T17:18:17.049Z 2 040-059 179 179 179 307 307 307 T T F <---
198282024-02-03T17:18:17.049Z 3 060-079 176 176 176 303 303 303 F F F
198292024-02-03T17:18:17.049Z 4 080-099 181 181 181 314 314 314 F F F
198302024-02-03T17:18:17.049Z 5 100-119 181 181 181 314 314 314 F F F
198312024-02-03T17:18:17.049Z 6 120-139 170 170 170 293 293 293 T T F <---
198322024-02-03T17:18:17.049Z 7 140-159 172 172 172 295 295 295 F F F
198332024-02-03T17:18:17.049Z 8 160-179 174 174 174 299 299 299 F F F
198342024-02-03T17:18:17.049Z 9 180-199 178 178 178 305 305 305 T T F <---
198352024-02-03T17:18:17.049Z 10 200-219 179 179 179 307 307 307 T T T
198362024-02-03T17:18:17.049Z 11 220-239 177 177 177 304 304 304 F F F
198372024-02-03T17:18:17.049Z 12 240-259 180 180 180 311 311 311 F F F
198382024-02-03T17:18:17.049Z 13 260-279 180 180 180 310 310 310 F F F
198392024-02-03T17:18:17.049Z 14 280-299 179 179 179 307 307 307 F F F
198402024-02-03T17:18:17.050Z 15 300-319 177 177 177 304 304 304 T T F <---
198412024-02-03T17:18:17.050Z 16 320-339 177 177 177 304 304 304 F F F
198422024-02-03T17:18:17.050Z 17 340-359 181 181 181 314 314 314 F F F
198432024-02-03T17:18:17.050Z 18 360-379 177 177 177 304 304 304 T T T
198442024-02-03T17:18:17.050Z 19 380-399 178 178 178 305 305 305 F F F
198452024-02-03T17:18:17.050Z 20 400-419 177 177 177 304 304 304 F F F
198462024-02-03T17:18:17.050Z 21 420-439 166 166 166 286 286 286 F F F
198472024-02-03T17:18:17.050Z 22 440-459 179 179 179 307 307 307 T T F <---
198482024-02-03T17:18:17.050Z 23 460-479 181 181 181 314 314 314 T T F <---
198492024-02-03T17:18:17.050Z 24 480-499 181 181 181 314 314 314 T T F <---
198502024-02-03T17:18:17.050Z 25 500-519 179 179 179 307 307 307 T T F <---
198512024-02-03T17:18:17.050Z 26 520-539 177 177 177 304 304 304 F F F
198522024-02-03T17:18:17.050Z 27 540-559 180 180 180 312 312 312 F F F
198532024-02-03T17:18:17.051Z 28 560-579 181 181 181 314 314 314 F F F
198542024-02-03T17:18:17.051Z 29 580-599 181 181 181 314 314 314 T T F <---
198552024-02-03T17:18:17.051ZMax gen: 181, Max flush: 314
198562024-02-03T17:18:17.051ZError: Difference in extent metadata found!
198572024-02-03T17:18:17.054ZA Difference in extent metadata is expected here
198582024-02-03T17:18:17.054ZOn loop 91
198592024-02-03T17:18:17.054Z
198602024-02-03T17:18:17.055ZVerifying data now
198612024-02-03T17:18:22.112ZNo change after verify
198622024-02-03T17:18:22.112ZLoop: 91 Downstairs dump after verify (and repair):
198632024-02-03T17:18:22.135ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
198642024-02-03T17:18:22.135Z 0 000-019 181 181 181 314 314 314 F F F
198652024-02-03T17:18:22.135Z 1 020-039 181 181 181 314 314 314 F F F
198662024-02-03T17:18:22.135Z 2 040-059 182 182 182 315 315 315 F F F
198672024-02-03T17:18:22.135Z 3 060-079 176 176 176 303 303 303 F F F
198682024-02-03T17:18:22.135Z 4 080-099 181 181 181 314 314 314 F F F
198692024-02-03T17:18:22.135Z 5 100-119 181 181 181 314 314 314 F F F
198702024-02-03T17:18:22.139Z 6 120-139 182 182 182 315 315 315 F F F
198712024-02-03T17:18:22.139Z 7 140-159 172 172 172 295 295 295 F F F
198722024-02-03T17:18:22.139Z 8 160-179 174 174 174 299 299 299 F F F
198732024-02-03T17:18:22.139Z 9 180-199 182 182 182 315 315 315 F F F
198742024-02-03T17:18:22.139Z 10 200-219 182 182 182 315 315 315 F F F
198752024-02-03T17:18:22.139Z 11 220-239 177 177 177 304 304 304 F F F
198762024-02-03T17:18:22.139Z 12 240-259 180 180 180 311 311 311 F F F
198772024-02-03T17:18:22.139Z 13 260-279 180 180 180 310 310 310 F F F
198782024-02-03T17:18:22.139Z 14 280-299 179 179 179 307 307 307 F F F
198792024-02-03T17:18:22.139Z 15 300-319 182 182 182 315 315 315 F F F
198802024-02-03T17:18:22.139Z 16 320-339 177 177 177 304 304 304 F F F
198812024-02-03T17:18:22.139Z 17 340-359 181 181 181 314 314 314 F F F
198822024-02-03T17:18:22.139Z 18 360-379 182 182 182 315 315 315 F F F
198832024-02-03T17:18:22.139Z 19 380-399 178 178 178 305 305 305 F F F
198842024-02-03T17:18:22.139Z 20 400-419 177 177 177 304 304 304 F F F
198852024-02-03T17:18:22.139Z 21 420-439 166 166 166 286 286 286 F F F
198862024-02-03T17:18:22.139Z 22 440-459 182 182 182 315 315 315 F F F
198872024-02-03T17:18:22.139Z 23 460-479 182 182 182 315 315 315 F F F
198882024-02-03T17:18:22.139Z 24 480-499 182 182 182 315 315 315 F F F
198892024-02-03T17:18:22.139Z 25 500-519 182 182 182 315 315 315 F F F
198902024-02-03T17:18:22.139Z 26 520-539 177 177 177 304 304 304 F F F
198912024-02-03T17:18:22.140Z 27 540-559 180 180 180 312 312 312 F F F
198922024-02-03T17:18:22.140Z 28 560-579 181 181 181 314 314 314 F F F
198932024-02-03T17:18:22.140Z 29 580-599 182 182 182 315 315 315 F F F
198942024-02-03T17:18:22.140ZMax gen: 182, Max flush: 315
198952024-02-03T17:18:22.143Z
198962024-02-03T17:18:22.143ZBegin loop 92
198972024-02-03T17:18:22.143ZDownstairs to restart: 0
198982024-02-03T17:18:22.170Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.751092258Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871}
198992024-02-03T17:18:22.171Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
199002024-02-03T17:18:22.174Z,"time":"2024-02-03T17:18:21.751597128Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871}
199012024-02-03T17:18:22.175Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.751769452Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871}
199022024-02-03T17:18:22.175Z{"msg":"Crucible 7330fbc2-c986-4674-8760-2e9259b6d5ca has session id: 0b320054-6c9f-4b4d-9451-ce45ae92e23b","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.751883615Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199032024-02-03T17:18:22.175Z{"msg":"Upstairs opts: Upstairs UUID: 7330fbc2-c986-4674-8760-2e9259b6d5ca, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.752026829Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199042024-02-03T17:18:22.175Z{"msg":"Crucible stats registered with UUID: 7330fbc2-c986-4674-8760-2e9259b6d5ca","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.752076511Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199052024-02-03T17:18:22.175Z{"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.752158609Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871}
199062024-02-03T17:18:22.175Z{"msg":"7330fbc2-c986-4674-8760-2e9259b6d5ca active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.752198975Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199072024-02-03T17:18:22.175Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.752305051Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199082024-02-03T17:18:22.175Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.75258512Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199092024-02-03T17:18:22.175Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.752681026Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199102024-02-03T17:18:22.175Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.752811024Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199112024-02-03T17:18:22.175Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.752862447Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199122024-02-03T17:18:22.175Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.752973991Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199132024-02-03T17:18:22.175Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.753023537Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"":"io task","client":"0","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199142024-02-03T17:18:22.175Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.753066039Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"":"io task","client":"1","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199152024-02-03T17:18:22.175Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.753153976Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"":"io task","client":"2","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199162024-02-03T17:18:22.175Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.753199251Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"":"io task","client":"1","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199172024-02-03T17:18:22.175Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.753244766Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"":"io task","client":"2","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199182024-02-03T17:18:22.175Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.753330879Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"":"io task","client":"0","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199192024-02-03T17:18:22.175Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.754102982Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199202024-02-03T17:18:22.175Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.754192883Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199212024-02-03T17:18:22.175Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.754241062Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199222024-02-03T17:18:22.175Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.754281635Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199232024-02-03T17:18:22.175Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.754311267Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199242024-02-03T17:18:22.176Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.754394878Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199252024-02-03T17:18:22.181Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.755454647Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199262024-02-03T17:18:22.181Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.755560632Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199272024-02-03T17:18:22.181Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.755638936Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199282024-02-03T17:18:22.181Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.756601988Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199292024-02-03T17:18:22.181Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.756681596Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199302024-02-03T17:18:22.181Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.756729024Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199312024-02-03T17:18:22.182Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.756795268Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199322024-02-03T17:18:22.182Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.756841198Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199332024-02-03T17:18:22.182Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.756905116Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199342024-02-03T17:18:22.182Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.756946479Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199352024-02-03T17:18:22.182Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.757012815Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199362024-02-03T17:18:22.182Z{"Wait for a query_work_queue command to finish before sending IO
199372024-02-03T17:18:22.182Zmsg":"[0]R flush_numbers[0..12]: [314, 314, 315, 303, 314, 314, 315, 295, 299, 315, 315, 304]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.757091046Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199382024-02-03T17:18:22.182Z{"msg":"[0]R generation[0..12]: [181, 181, 182, 176, 181, 181, 182, 172, 174, 182, 182, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.757139544Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199392024-02-03T17:18:22.182Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.757204511Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199402024-02-03T17:18:22.182Z{"msg":"[1]R flush_numbers[0..12]: [314, 314, 315, 303, 314, 314, 315, 295, 299, 315, 315, 304]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.757250768Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199412024-02-03T17:18:22.182Z{"msg":"[1]R generation[0..12]: [181, 181, 182, 176, 181, 181, 182, 172, 174, 182, 182, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.757318754Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199422024-02-03T17:18:22.182Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.757364875Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199432024-02-03T17:18:22.182Z{"msg":"[2]R flush_numbers[0..12]: [314, 314, 315, 303, 314, 314, 315, 295, 299, 315, 315, 304]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.757430728Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199442024-02-03T17:18:22.182Z{"msg":"[2]R generation[0..12]: [181, 181, 182, 176, 181, 181, 182, 172, 174, 182, 182, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.757476367Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199452024-02-03T17:18:22.182Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.757541756Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199462024-02-03T17:18:22.182Z{"msg":"Max found gen is 183","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.75758475Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199472024-02-03T17:18:22.182Z{"msg":"Generation requested: 184 >= found:183","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.757651798Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199482024-02-03T17:18:22.182Z{"msg":"Next flush: 316","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.75769462Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199492024-02-03T17:18:22.182Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.757759578Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199502024-02-03T17:18:22.182Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
199512024-02-03T17:18:22.182Z2024-02-03T17:18:21.757803975Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199522024-02-03T17:18:22.182Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.757921797Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199532024-02-03T17:18:22.182Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.757970132Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199542024-02-03T17:18:22.182Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.758012813Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199552024-02-03T17:18:22.182Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.758076744Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199562024-02-03T17:18:22.183Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.758118194Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199572024-02-03T17:18:22.183Z{"msg":"7330fbc2-c986-4674-8760-2e9259b6d5ca is now active with session: 0b320054-6c9f-4b4d-9451-ce45ae92e23b","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.7581828Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199582024-02-03T17:18:22.183Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.75822387Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871,"session_id":"0b320054-6c9f-4b4d-9451-ce45ae92e23b"}
199592024-02-03T17:18:22.183Z{"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:21.758286092Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871}
199602024-02-03T17:18:22.183ZLoading write count information from file "/tmp/test_repair_verify.data"
199612024-02-03T17:18:22.183ZRun Repair workload
199622024-02-03T17:18:22.183Z01/30 Flush
199632024-02-03T17:18:22.183Z02/30 Write block 471 len 4096 data: 12
199642024-02-03T17:18:22.187Z03/30 Write block 270 len 16384 data: 16 15 16 14
199652024-02-03T17:18:22.187Z04/30 Write block 315 len 12288 data: 19 19 19
199662024-02-03T17:18:22.187Z05/30 Read block 452 len 20480
199672024-02-03T17:18:22.187Z06/30 Read block 152 len 28672
199682024-02-03T17:18:22.190Z07/30 Read block 34 len 16384
199692024-02-03T17:18:22.190Z08/30 Write block 390 len 40960 data: 15 16 21 20 18 17 17 13 15 15
199702024-02-03T17:18:22.190Z09/30 Read block 130 len 8192
199712024-02-03T17:18:22.193Z10/30 Write block 79 len 24576 data: 14 14 18 17 14 13
199722024-02-03T17:18:22.194Z11/30 Write block 119 len 12288 data: 20 17 16
199732024-02-03T17:18:22.197Z12/30 Write block 142 len 40960 data: 17 15 11 12 12 9 14 15 14 16
199742024-02-03T17:18:22.197Z13/30 Read block 96 len 24576
199752024-02-03T17:18:22.200Z14/30 Read block 106 len 20480
199762024-02-03T17:18:22.203Z15/30 Write block 530 len 20480 data: 16 14 11 12 10
199772024-02-03T17:18:22.203Z16/30 Read block 141 len 16384
199782024-02-03T17:18:22.203Z17/30 Read block 494 len 20480
199792024-02-03T17:18:22.207Z18/30 Read block 522 len 8192
199802024-02-03T17:18:22.207Z19/30 Write block 523 len 32768 data: 15 14 17 16 16 15 15 17
199812024-02-03T17:18:22.207Z20/30 Write block 306 len 20480 data: 18 15 16 16 18
199822024-02-03T17:18:22.210Z21/30 Flush
199832024-02-03T17:18:22.213Z22/30 Write block 36 len 28672 data: 16 14 13 13 10 9 12
199842024-02-03T17:18:22.217Z23/30 Write block 151 len 8192 data: 17 17
199852024-02-03T17:18:22.217Z24/30 Read block 497 len 24576
199862024-02-03T17:18:22.220Z25/30 Read block 401 len 32768
199872024-02-03T17:18:22.220Z26/30 Write block 53 len 40960 data: 15 14 14 15 13 11 12 11 11 13
199882024-02-03T17:18:22.223Z27/30 Write block 576 len 24576 data: 15 12 12 13 12 14
199892024-02-03T17:18:22.223Z28/30 Read block 509 len 8192
199902024-02-03T17:18:22.223Z29/30 Read block 282 len 28672
199912024-02-03T17:18:22.227Z30/30 Write block 286 len 28672 data: 16 15 13 16 14 13 10
199922024-02-03T17:18:22.227Z----------------------------------------------------------------
199932024-02-03T17:18:22.227Z Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:29
199942024-02-03T17:18:22.227ZStates: Active Active Active
199952024-02-03T17:18:22.227ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
199962024-02-03T17:18:22.227Z 2 Acked 1001 Write 1 Done Done Done false
199972024-02-03T17:18:22.227Z 3 Acked 1002 Write 4 Done Done Done false
199982024-02-03T17:18:22.227Z 4 Acked 1003 Write 3 Done Done Done false
199992024-02-03T17:18:22.227Z 5 Acked 1004 Read 5 Done Done Done false
200002024-02-03T17:18:22.227Z 6 Acked 1005 Read 7 Done Done Done false
200012024-02-03T17:18:22.227Z 7 Acked 1006 Read 4 Done Done Done false
200022024-02-03T17:18:22.227Z 8 Acked 1007 Write 10 Sent Done Done false
200032024-02-03T17:18:22.227Z 9 Acked 1008 Read 2 Sent Done Done false
200042024-02-03T17:18:22.227Z 10 Acked 1009 Write 6 Sent Done Done false
200052024-02-03T17:18:22.227Z 11 Acked 1010 Write 3 Sent Done Done false
200062024-02-03T17:18:22.228Z 12 Acked 1011 Write 10 Sent Done Done false
200072024-02-03T17:18:22.228Z 13 Acked 1012 Read 6 Sent Done Done false
200082024-02-03T17:18:22.228Z 14 Acked 1013 Read 5 Sent Done Done false
200092024-02-03T17:18:22.228Z 15 Acked 1014 Write 5 Sent Done Done false
200102024-02-03T17:18:22.228Z 16 Acked 1015 Read 4 Sent Done Done false
200112024-02-03T17:18:22.228Z 17 Acked 1016 Read 5 Sent Done Done false
200122024-02-03T17:18:22.228Z 18 Acked 1017 Read 2 Sent Done Done false
200132024-02-03T17:18:22.228Z 19 Acked 1018 Write 8 Sent Done Done false
200142024-02-03T17:18:22.228Z 20 Acked 1019 Write 5 Sent Done Done false
200152024-02-03T17:18:22.228Z 21 Acked 1020 Flush 0 Sent Done Done false
200162024-02-03T17:18:22.228Z 22 Acked 1021 Write 7 Sent Done Done false
200172024-02-03T17:18:22.228Z 23 Acked 1022 Write 2 Sent Done Done false
200182024-02-03T17:18:22.228Z 24 Acked 1023 Read 6 Sent Done Done false
200192024-02-03T17:18:22.228Z 25 Acked 1024 Read 8 Sent Done Done false
200202024-02-03T17:18:22.228Z 26 Acked 1025 Write 10 Sent Done Done false
200212024-02-03T17:18:22.228Z 27 Acked 1026 Write 6 Sent Done Done false
200222024-02-03T17:18:22.228Z 28 Acked 1027 Read 2 Sent Done Done false
200232024-02-03T17:18:22.228Z 29 Acked 1028 Read 7 Sent Done Done false
200242024-02-03T17:18:22.228Z 30 Acked 1029 Write 7 Sent Sent Sent false
200252024-02-03T17:18:22.228Z STATES DS:0 DS:1 DS:2 TOTAL
200262024-02-03T17:18:22.232Z New 0 0 0 0
200272024-02-03T17:18:22.232Z Sent 23 1 1 25
200282024-02-03T17:18:22.232Z Done 6 28 28 62
200292024-02-03T17:18:22.232Z Skipped 0 0 0 0
200302024-02-03T17:18:22.232Z Error 0 0 0 0
200312024-02-03T17:18:22.232ZLast Flush: 1000 1020 1020
200322024-02-03T17:18:22.232ZDownstairs last five completed: 1000
200332024-02-03T17:18:22.232ZUpstairs last five completed: 30 29 28 27 26
200342024-02-03T17:18:22.232Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:18:21.808167566Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1871}
200352024-02-03T17:18:22.232ZWrote out file "/tmp/test_repair_verify.data"
200362024-02-03T17:18:22.236Z
200372024-02-03T17:18:22.241ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
200382024-02-03T17:18:22.269ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
200392024-02-03T17:18:22.269Z 0 000-019 181 181 181 314 314 314 F F F
200402024-02-03T17:18:22.269Z 1 020-039 181 181 181 314 314 314 F T T <---
200412024-02-03T17:18:22.269Z 2 040-059 182 182 182 315 315 315 F T T <---
200422024-02-03T17:18:22.269Z 3 060-079 176 184 184 303 317 317 F T T <---
200432024-02-03T17:18:22.269Z 4 080-099 181 184 184 314 317 317 F F F <---
200442024-02-03T17:18:22.269Z 5 100-119 181 184 184 314 317 317 F F F <---
200452024-02-03T17:18:22.269Z 6 120-139 182 184 184 315 317 317 F F F <---
200462024-02-03T17:18:22.274Z 7 140-159 172 184 184 295 317 317 F T T <---
200472024-02-03T17:18:22.274Z 8 160-179 174 174 174 299 299 299 F F F
200482024-02-03T17:18:22.274Z 9 180-199 182 182 182 315 315 315 F F F
200492024-02-03T17:18:22.274Z 10 200-219 182 182 182 315 315 315 F F F
200502024-02-03T17:18:22.274Z 11 220-239 177 177 177 304 304 304 F F F
200512024-02-03T17:18:22.274Z 12 240-259 180 180 180 311 311 311 F F F
200522024-02-03T17:18:22.274Z 13 260-279 180 184 184 310 317 317 T F F <---
200532024-02-03T17:18:22.274Z 14 280-299 179 179 179 307 307 307 F T T <---
200542024-02-03T17:18:22.274Z 15 300-319 182 184 184 315 317 317 T F F <---
200552024-02-03T17:18:22.274Z 16 320-339 177 177 177 304 304 304 F F F
200562024-02-03T17:18:22.274Z 17 340-359 181 181 181 314 314 314 F F F
200572024-02-03T17:18:22.274Z 18 360-379 182 182 182 315 315 315 F F F
200582024-02-03T17:18:22.274Z 19 380-399 178 184 184 305 317 317 F F F <---
200592024-02-03T17:18:22.274Z 20 400-419 177 177 177 304 304 304 F F F
200602024-02-03T17:18:22.274Z 21 420-439 166 166 166 286 286 286 F F F
200612024-02-03T17:18:22.274Z 22 440-459 182 182 182 315 315 315 F F F
200622024-02-03T17:18:22.274Z 23 460-479 182 184 184 315 317 317 T F F <---
200632024-02-03T17:18:22.274Z 24 480-499 182 182 182 315 315 315 F F F
200642024-02-03T17:18:22.274Z 25 500-519 182 182 182 315 315 315 F F F
200652024-02-03T17:18:22.275Z 26 520-539 177 184 184 304 317 317 F F F <---
200662024-02-03T17:18:22.275Z 27 540-559 180 180 180 312 312 312 F F F
200672024-02-03T17:18:22.275Z 28 560-579 181 181 181 314 314 314 F T T <---
200682024-02-03T17:18:22.275Z 29 580-599 182 182 182 315 315 315 F T T <---
200692024-02-03T17:18:22.275ZMax gen: 184, Max flush: 317
200702024-02-03T17:18:22.275ZError: Difference in extent metadata found!
200712024-02-03T17:18:22.278ZA Difference in extent metadata is expected here
200722024-02-03T17:18:22.278ZOn loop 92
200732024-02-03T17:18:22.278Z
200742024-02-03T17:18:22.278ZVerifying data now
200752024-02-03T17:18:28.560ZNo change after verify
200762024-02-03T17:18:28.560ZLoop: 92 Downstairs dump after verify (and repair):
200772024-02-03T17:18:28.583ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
200782024-02-03T17:18:28.583Z 0 000-019 181 181 181 314 314 314 F F F
200792024-02-03T17:18:28.583Z 1 020-039 185 185 185 318 318 318 F F F
200802024-02-03T17:18:28.583Z 2 040-059 185 185 185 318 318 318 F F F
200812024-02-03T17:18:28.587Z 3 060-079 185 185 185 318 318 318 F F F
200822024-02-03T17:18:28.587Z 4 080-099 184 184 184 317 317 317 F F F
200832024-02-03T17:18:28.587Z 5 100-119 184 184 184 317 317 317 F F F
200842024-02-03T17:18:28.587Z 6 120-139 184 184 184 317 317 317 F F F
200852024-02-03T17:18:28.587Z 7 140-159 185 185 185 318 318 318 F F F
200862024-02-03T17:18:28.587Z 8 160-179 174 174 174 299 299 299 F F F
200872024-02-03T17:18:28.587Z 9 180-199 182 182 182 315 315 315 F F F
200882024-02-03T17:18:28.587Z 10 200-219 182 182 182 315 315 315 F F F
200892024-02-03T17:18:28.588Z 11 220-239 177 177 177 304 304 304 F F F
200902024-02-03T17:18:28.588Z 12 240-259 180 180 180 311 311 311 F F F
200912024-02-03T17:18:28.588Z 13 260-279 184 184 184 317 317 317 F F F
200922024-02-03T17:18:28.588Z 14 280-299 185 185 185 318 318 318 F F F
200932024-02-03T17:18:28.588Z 15 300-319 184 184 184 317 317 317 F F F
200942024-02-03T17:18:28.588Z 16 320-339 177 177 177 304 304 304 F F F
200952024-02-03T17:18:28.588Z 17 340-359 181 181 181 314 314 314 F F F
200962024-02-03T17:18:28.588Z 18 360-379 182 182 182 315 315 315 F F F
200972024-02-03T17:18:28.588Z 19 380-399 184 184 184 317 317 317 F F F
200982024-02-03T17:18:28.588Z 20 400-419 177 177 177 304 304 304 F F F
200992024-02-03T17:18:28.588Z 21 420-439 166 166 166 286 286 286 F F F
201002024-02-03T17:18:28.588Z 22 440-459 182 182 182 315 315 315 F F F
201012024-02-03T17:18:28.588Z 23 460-479 184 184 184 317 317 317 F F F
201022024-02-03T17:18:28.589Z 24 480-499 182 182 182 315 315 315 F F F
201032024-02-03T17:18:28.589Z 25 500-519 182 182 182 315 315 315 F F F
201042024-02-03T17:18:28.589Z 26 520-539 184 184 184 317 317 317 F F F
201052024-02-03T17:18:28.589Z 27 540-559 180 180 180 312 312 312 F F F
201062024-02-03T17:18:28.589Z 28 560-579 185 185 185 318 318 318 F F F
201072024-02-03T17:18:28.589Z 29 580-599 185 185 185 318 318 318 F F F
201082024-02-03T17:18:28.589ZMax gen: 185, Max flush: 318
201092024-02-03T17:18:28.593Z
201102024-02-03T17:18:28.593ZBegin loop 93
201112024-02-03T17:18:28.593ZDownstairs to restart: 2
201122024-02-03T17:18:28.619Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.200696979Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879}
201132024-02-03T17:18:28.619Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.201260911Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879}
201142024-02-03T17:18:28.623Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.20134414Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879}
201152024-02-03T17:18:28.624Z{"msg":"Crucible 6b566869-ff3a-4605-8ef4-f8369f21e57a has session id: 9f41bb01-5ea8-419f-b769-812d76fab0ce","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.201475527Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201162024-02-03T17:18:28.624Z{"msg":"Upstairs opts: Upstairs UUID: 6b566869-ff3a-4605-8ef4-f8369f21e57a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.201539871Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879Crucible runtime is spawned
201172024-02-03T17:18:28.624Z,"session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201182024-02-03T17:18:28.624Z{"msg":"Crucible stats registered with UUID: 6b566869-ff3a-4605-8ef4-f8369f21e57a","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.201665209Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201192024-02-03T17:18:28.624Z{"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.201740193Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879}
201202024-02-03T17:18:28.624Z{"msg":"6b566869-ff3a-4605-8ef4-f8369f21e57a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.201833057Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201212024-02-03T17:18:28.624Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.201917844Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201222024-02-03T17:18:28.624Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.201946554Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201232024-02-03T17:18:28.624Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.202039654Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201242024-02-03T17:18:28.624Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.202157214Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201252024-02-03T17:18:28.624Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.202289708Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201262024-02-03T17:18:28.624Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.202355668Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201272024-02-03T17:18:28.625Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.202558575Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"":"io task","client":"2","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201282024-02-03T17:18:28.625Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.202650578Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"":"io task","client":"1","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201292024-02-03T17:18:28.625Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.202701837Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"":"io task","client":"0","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201302024-02-03T17:18:28.625Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.202784238Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"":"io task","client":"1","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201312024-02-03T17:18:28.625Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.202829608Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"":"io task","client":"2","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201322024-02-03T17:18:28.625Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.202902908Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"":"io task","client":"0","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201332024-02-03T17:18:28.625Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.203885562Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201342024-02-03T17:18:28.625Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.203952215Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201352024-02-03T17:18:28.625Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.203996233Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201362024-02-03T17:18:28.625Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.204157632Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201372024-02-03T17:18:28.625Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.2043342Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201382024-02-03T17:18:28.625Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.204419518Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201392024-02-03T17:18:28.631Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.205103461Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201402024-02-03T17:18:28.631Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.205183868Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201412024-02-03T17:18:28.631Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.205328653Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201422024-02-03T17:18:28.631Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.206196376Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201432024-02-03T17:18:28.631Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.206270286Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201442024-02-03T17:18:28.631Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.206298054Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201452024-02-03T17:18:28.631Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.206353105Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201462024-02-03T17:18:28.631Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.206378086Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201472024-02-03T17:18:28.631Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.206400499Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201482024-02-03T17:18:28.631Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.206446659Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201492024-02-03T17:18:28.632Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.206471371Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201502024-02-03T17:18:28.632Z{"msg":"[0]R flush_numbers[0..12]: [314, 318, 318, 318, 317, 317, 317, 318, 299, 315, 315, 304]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.206520539Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201512024-02-03T17:18:28.632Z{"msg":"[0]R generation[0..12]: [181, 185, 185, 185, 184, 184, 184, 185, 174, 182, 182, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.206546557Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201522024-02-03T17:18:28.632Z{"msg":"Wait for a query_work_queue command to finish before sending IO
201532024-02-03T17:18:28.632Z[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.206789883Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201542024-02-03T17:18:28.632Z{"msg":"[1]R flush_numbers[0..12]: [314, 318, 318, 318, 317, 317, 317, 318, 299, 315, 315, 304]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.206834532Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201552024-02-03T17:18:28.632Z{"msg":"[1]R generation[0..12]: [181, 185, 185, 185, 184, 184, 184, 185, 174, 182, 182, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.206942598Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201562024-02-03T17:18:28.632Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.206997228Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201572024-02-03T17:18:28.632Z{"msg":"[2]R flush_numbers[0..12]: [314, 318, 318, 318, 317, 317, 317, 318, 299, 315, 315, 304]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.207024161Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201582024-02-03T17:18:28.632Z{"msg":"[2]R generation[0..12]: [181, 185, 185, 185, 184, 184, 184, 185, 174, 182, 182, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.207109356Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201592024-02-03T17:18:28.632Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.207144941Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201602024-02-03T17:18:28.632Z{"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.207169553Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201612024-02-03T17:18:28.633Z{"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.207219733Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201622024-02-03T17:18:28.633Z{"msg":"Next flush: 319","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.207243852Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201632024-02-03T17:18:28.633Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.207265307Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201642024-02-03T17:18:28.633Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.207287178Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201652024-02-03T17:18:28.633Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.207307068Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201662024-02-03T17:18:28.633Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.207363537Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201672024-02-03T17:18:28.633Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.20738879Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201682024-02-03T17:18:28.633Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.207436889Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201692024-02-03T17:18:28.633Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.207459845Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201702024-02-03T17:18:28.633Z{"msg":"6b566869-ff3a-4605-8ef4-f8369f21e57a is now active with session: 9f41bb01-5ea8-419f-b769-812d76fab0ce","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.207481445Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201712024-02-03T17:18:28.633Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.207500894Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879,"session_id":"9f41bb01-5ea8-419f-b769-812d76fab0ce"}
201722024-02-03T17:18:28.633Z{"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:28.207565092Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879}
201732024-02-03T17:18:28.633ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
201742024-02-03T17:18:28.634ZLoading write count information from file "/tmp/test_repair_verify.data"
201752024-02-03T17:18:28.634ZRun Repair workload
201762024-02-03T17:18:28.634Z01/30 Write block 226 len 40960 data: 8 9 13 9 9 11 14 14 17 16
201772024-02-03T17:18:28.638Z02/30 Read block 274 len 12288
201782024-02-03T17:18:28.638Z03/30 Write block 235 len 8192 data: 17 14
201792024-02-03T17:18:28.638Z04/30 Read block 572 len 32768
201802024-02-03T17:18:28.638Z05/30 Write block 58 len 32768 data: 12 13 12 12 14 12 10 9
201812024-02-03T17:18:28.638Z06/30 Read block 472 len 16384
201822024-02-03T17:18:28.642Z07/30 Write block 565 len 24576 data: 14 13 13 18 14 15
201832024-02-03T17:18:28.642Z08/30 Read block 33 len 12288
201842024-02-03T17:18:28.645Z09/30 Write block 566 len 12288 data: 14 14 19
201852024-02-03T17:18:28.645Z10/30 Read block 558 len 24576
201862024-02-03T17:18:28.645Z11/30 Write block 400 len 28672 data: 13 10 9 9 9 13 16
201872024-02-03T17:18:28.648Z12/30 Flush
201882024-02-03T17:18:28.652Z13/30 Flush
201892024-02-03T17:18:28.652Z14/30 Write block 216 len 40960 data: 13 12 11 10 8 8 6 7 6 6
201902024-02-03T17:18:28.656Z15/30 Read block 280 len 12288
201912024-02-03T17:18:28.656Z16/30 Read block 265 len 40960
201922024-02-03T17:18:28.659Z17/30 Write block 361 len 24576 data: 14 12 12 13 15 13
201932024-02-03T17:18:28.659Z18/30 Read block 467 len 8192
201942024-02-03T17:18:28.659Z19/30 Read block 540 len 24576
201952024-02-03T17:18:28.663Z20/30 Write block 426 len 8192 data: 11 14
201962024-02-03T17:18:28.663Z21/30 Read block 180 len 20480
201972024-02-03T17:18:28.667Z22/30 Flush
201982024-02-03T17:18:28.671Z23/30 Write block 64 len 40960 data: 11 10 9 8 9 7 8 7 6 7
201992024-02-03T17:18:28.671Z24/30 Write block 32 len 16384 data: 15 13 14 14
202002024-02-03T17:18:28.671Z25/30 Read block 367 len 12288
202012024-02-03T17:18:28.674Z26/30 Flush
202022024-02-03T17:18:28.674Z27/30 Write block 47 len 12288 data: 13 15 17
202032024-02-03T17:18:28.684Z28/30 Write block 214 len 28672 data: 8 9 14 13 12 11 9
202042024-02-03T17:18:28.684Z29/30 Write block 216 len 4096 data: 15
202052024-02-03T17:18:28.684Z30/30 Write block 265 len 8192 data: 21 19
202062024-02-03T17:18:28.684Z----------------------------------------------------------------
202072024-02-03T17:18:28.684Z Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:30
202082024-02-03T17:18:28.684ZStates: Active Active Active
202092024-02-03T17:18:28.684ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
202102024-02-03T17:18:28.684Z 1 Acked 1000 Write 10 Done Done Done false
202112024-02-03T17:18:28.684Z 2 Acked 1001 Read 3 Done Done Done false
202122024-02-03T17:18:28.684Z 3 Acked 1002 Write 2 Done Done Sent false
202132024-02-03T17:18:28.685Z 4 Acked 1003 Read 8 Done Done Sent false
202142024-02-03T17:18:28.685Z 5 Acked 1004 Write 8 Done Done Sent false
202152024-02-03T17:18:28.685Z 6 Acked 1005 Read 4 Done Done Sent false
202162024-02-03T17:18:28.685Z 7 Acked 1006 Write 6 Done Done Sent false
202172024-02-03T17:18:28.685Z 8 Acked 1007 Read 3 Done Done Sent false
202182024-02-03T17:18:28.685Z 9 Acked 1008 Write 3 Done Done Sent false
202192024-02-03T17:18:28.685Z 10 Acked 1009 Read 6 Done Done Sent false
202202024-02-03T17:18:28.685Z 11 Acked 1010 Write 7 Done Done Sent false
202212024-02-03T17:18:28.685Z 12 Acked 1011 Flush 0 Done Done Sent false
202222024-02-03T17:18:28.685Z 13 Acked 1012 Flush 0 Done Done Sent false
202232024-02-03T17:18:28.685Z 14 Acked 1013 Write 10 Done Done Sent false
202242024-02-03T17:18:28.685Z 15 Acked 1014 Read 3 Done Done Sent false
202252024-02-03T17:18:28.685Z 16 Acked 1015 Read 10 Done Done Sent false
202262024-02-03T17:18:28.685Z 17 Acked 1016 Write 6 Done Done Sent false
202272024-02-03T17:18:28.685Z 18 Acked 1017 Read 2 Done Done Sent false
202282024-02-03T17:18:28.685Z 19 Acked 1018 Read 6 Done Done Sent false
202292024-02-03T17:18:28.685Z 20 Acked 1019 Write 2 Done Done Sent false
202302024-02-03T17:18:28.685Z 21 Acked 1020 Read 5 Done Done Sent false
202312024-02-03T17:18:28.685Z 22 Acked 1021 Flush 0 Done Done Sent false
202322024-02-03T17:18:28.685Z 23 Acked 1022 Write 10 Done Done Sent false
202332024-02-03T17:18:28.685Z 24 Acked 1023 Write 4 Done Done Sent false
202342024-02-03T17:18:28.685Z 25 Acked 1024 Read 3 Done Done Sent false
202352024-02-03T17:18:28.685Z 26 Acked 1025 Flush 0 Done Done Sent false
202362024-02-03T17:18:28.685Z 27 Acked 1026 Write 3 Done Done Sent false
202372024-02-03T17:18:28.686Z 28 Acked 1027 Write 7 Done Done Sent false
202382024-02-03T17:18:28.686Z 29 Acked 1028 Write 1 Done Done Sent false
202392024-02-03T17:18:28.686Z 30 Acked 1029 Write 2 Sent Sent Sent false
202402024-02-03T17:18:28.686Z STATES DS:0 DS:1 DS:2 TOTAL
202412024-02-03T17:18:28.686Z New 0 0 0 0
202422024-02-03T17:18:28.686Z Sent 1 1 28 30
202432024-02-03T17:18:28.686Z Done 29 29 2 60
202442024-02-03T17:18:28.686Z Skipped 0 0 0 0
202452024-02-03T17:18:28.686Z Error 0 0 0 0
202462024-02-03T17:18:28.686ZLast Flush: 1025 1025 0
202472024-02-03T17:18:28.686ZDownstairs last five completed:
202482024-02-03T17:18:28.686ZUpstairs last five completed: 30 29 28 27 26
202492024-02-03T17:18:28.686Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:18:28.260205979Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1879}
202502024-02-03T17:18:28.690ZWrote out file "/tmp/test_repair_verify.data"
202512024-02-03T17:18:28.690Z
202522024-02-03T17:18:28.694ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
202532024-02-03T17:18:28.716ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
202542024-02-03T17:18:28.716Z 0 000-019 181 181 181 314 314 314 F F F
202552024-02-03T17:18:28.716Z 1 020-039 186 186 185 322 322 318 F F F <---
202562024-02-03T17:18:28.716Z 2 040-059 186 186 185 319 319 318 T T F <---
202572024-02-03T17:18:28.720Z 3 060-079 186 186 185 322 322 318 F F F <---
202582024-02-03T17:18:28.721Z 4 080-099 184 184 184 317 317 317 F F F
202592024-02-03T17:18:28.721Z 5 100-119 184 184 184 317 317 317 F F F
202602024-02-03T17:18:28.721Z 6 120-139 184 184 184 317 317 317 F F F
202612024-02-03T17:18:28.721Z 7 140-159 185 185 185 318 318 318 F F F
202622024-02-03T17:18:28.721Z 8 160-179 174 174 174 299 299 299 F F F
202632024-02-03T17:18:28.721Z 9 180-199 182 182 182 315 315 315 F F F
202642024-02-03T17:18:28.721Z 10 200-219 186 186 182 321 321 315 T T F <---
202652024-02-03T17:18:28.721Z 11 220-239 186 186 177 321 321 304 T T T <---
202662024-02-03T17:18:28.721Z 12 240-259 180 180 180 311 311 311 F F F
202672024-02-03T17:18:28.721Z 13 260-279 184 184 184 317 317 317 T T F <---
202682024-02-03T17:18:28.721Z 14 280-299 185 185 185 318 318 318 F F F
202692024-02-03T17:18:28.721Z 15 300-319 184 184 184 317 317 317 F F F
202702024-02-03T17:18:28.721Z 16 320-339 177 177 177 304 304 304 F F F
202712024-02-03T17:18:28.721Z 17 340-359 181 181 181 314 314 314 F F F
202722024-02-03T17:18:28.721Z 18 360-379 186 186 182 321 321 315 F F F <---
202732024-02-03T17:18:28.722Z 19 380-399 184 184 184 317 317 317 F F F
202742024-02-03T17:18:28.722Z 20 400-419 186 186 177 319 319 304 F F F <---
202752024-02-03T17:18:28.722Z 21 420-439 186 186 166 321 321 286 F F F <---
202762024-02-03T17:18:28.722Z 22 440-459 182 182 182 315 315 315 F F F
202772024-02-03T17:18:28.722Z 23 460-479 184 184 184 317 317 317 F F F
202782024-02-03T17:18:28.722Z 24 480-499 182 182 182 315 315 315 F F F
202792024-02-03T17:18:28.722Z 25 500-519 182 182 182 315 315 315 F F F
202802024-02-03T17:18:28.722Z 26 520-539 184 184 184 317 317 317 F F F
202812024-02-03T17:18:28.722Z 27 540-559 180 180 180 312 312 312 F F F
202822024-02-03T17:18:28.722Z 28 560-579 186 186 185 319 319 318 F F F <---
202832024-02-03T17:18:28.722Z 29 580-599 185 185 185 318 318 318 F F F
202842024-02-03T17:18:28.722ZMax gen: 186, Max flush: 322
202852024-02-03T17:18:28.722ZError: Difference in extent metadata found!
202862024-02-03T17:18:28.726ZA Difference in extent metadata is expected here
202872024-02-03T17:18:28.726ZOn loop 93
202882024-02-03T17:18:28.726Z
202892024-02-03T17:18:28.726ZVerifying data now
202902024-02-03T17:18:33.638ZNo change after verify
202912024-02-03T17:18:33.638ZLoop: 93 Downstairs dump after verify (and repair):
202922024-02-03T17:18:33.661ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
202932024-02-03T17:18:33.661Z 0 000-019 181 181 181 314 314 314 F F F
202942024-02-03T17:18:33.661Z 1 020-039 186 186 186 322 322 322 F F F
202952024-02-03T17:18:33.661Z 2 040-059 187 187 187 323 323 323 F F F
202962024-02-03T17:18:33.661Z 3 060-079 186 186 186 322 322 322 F F F
202972024-02-03T17:18:33.661Z 4 080-099 184 184 184 317 317 317 F F F
202982024-02-03T17:18:33.664Z 5 100-119 184 184 184 317 317 317 F F F
202992024-02-03T17:18:33.664Z 6 120-139 184 184 184 317 317 317 F F F
203002024-02-03T17:18:33.664Z 7 140-159 185 185 185 318 318 318 F F F
203012024-02-03T17:18:33.664Z 8 160-179 174 174 174 299 299 299 F F F
203022024-02-03T17:18:33.664Z 9 180-199 182 182 182 315 315 315 F F F
203032024-02-03T17:18:33.664Z 10 200-219 187 187 187 323 323 323 F F F
203042024-02-03T17:18:33.664Z 11 220-239 187 187 187 323 323 323 F F F
203052024-02-03T17:18:33.665Z 12 240-259 180 180 180 311 311 311 F F F
203062024-02-03T17:18:33.665Z 13 260-279 187 187 187 323 323 323 F F F
203072024-02-03T17:18:33.665Z 14 280-299 185 185 185 318 318 318 F F F
203082024-02-03T17:18:33.665Z 15 300-319 184 184 184 317 317 317 F F F
203092024-02-03T17:18:33.665Z 16 320-339 177 177 177 304 304 304 F F F
203102024-02-03T17:18:33.665Z 17 340-359 181 181 181 314 314 314 F F F
203112024-02-03T17:18:33.665Z 18 360-379 186 186 186 321 321 321 F F F
203122024-02-03T17:18:33.665Z 19 380-399 184 184 184 317 317 317 F F F
203132024-02-03T17:18:33.665Z 20 400-419 186 186 186 319 319 319 F F F
203142024-02-03T17:18:33.665Z 21 420-439 186 186 186 321 321 321 F F F
203152024-02-03T17:18:33.665Z 22 440-459 182 182 182 315 315 315 F F F
203162024-02-03T17:18:33.665Z 23 460-479 184 184 184 317 317 317 F F F
203172024-02-03T17:18:33.665Z 24 480-499 182 182 182 315 315 315 F F F
203182024-02-03T17:18:33.665Z 25 500-519 182 182 182 315 315 315 F F F
203192024-02-03T17:18:33.665Z 26 520-539 184 184 184 317 317 317 F F F
203202024-02-03T17:18:33.665Z 27 540-559 180 180 180 312 312 312 F F F
203212024-02-03T17:18:33.665Z 28 560-579 186 186 186 319 319 319 F F F
203222024-02-03T17:18:33.665Z 29 580-599 185 185 185 318 318 318 F F F
203232024-02-03T17:18:33.665ZMax gen: 187, Max flush: 323
203242024-02-03T17:18:33.669Z
203252024-02-03T17:18:33.669ZBegin loop 94
203262024-02-03T17:18:33.669ZDownstairs to restart: 1
203272024-02-03T17:18:33.697Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.278985412Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887}
203282024-02-03T17:18:33.698ZCrucible runtime is spawned
203292024-02-03T17:18:33.698Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.279590053Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887}
203302024-02-03T17:18:33.698Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.279639382Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887}
203312024-02-03T17:18:33.698Z{"msg":"Crucible 9941d150-2665-41b5-96d3-719054cac957 has session id: 9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.279705529Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203322024-02-03T17:18:33.702Z{"msg":"Upstairs opts: Upstairs UUID: 9941d150-2665-41b5-96d3-719054cac957, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.279746128Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203332024-02-03T17:18:33.702Z{"msg":"Crucible stats registered with UUID: 9941d150-2665-41b5-96d3-719054cac957","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.279823131Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203342024-02-03T17:18:33.702Z{"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.279896089Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887}
203352024-02-03T17:18:33.702Z{"msg":"9941d150-2665-41b5-96d3-719054cac957 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.279921595Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203362024-02-03T17:18:33.702Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.279942192Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203372024-02-03T17:18:33.702Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.279973115Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203382024-02-03T17:18:33.702Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.280023702Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203392024-02-03T17:18:33.702Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.280170387Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203402024-02-03T17:18:33.702Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.280216488Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203412024-02-03T17:18:33.702Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.280256324Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203422024-02-03T17:18:33.702Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.28033684Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"":"io task","client":"0","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203432024-02-03T17:18:33.702Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.280386578Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"":"io task","client":"2","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203442024-02-03T17:18:33.703Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.280429779Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"":"io task","client":"1","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203452024-02-03T17:18:33.703Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.280471594Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"":"io task","client":"0","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203462024-02-03T17:18:33.703Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.280516796Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"":"io task","client":"1","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203472024-02-03T17:18:33.703Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.280557342Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"":"io task","client":"2","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203482024-02-03T17:18:33.703Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.281718749Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203492024-02-03T17:18:33.703Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.281913837Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203502024-02-03T17:18:33.703Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.2820752Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203512024-02-03T17:18:33.703Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.282216736Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203522024-02-03T17:18:33.703Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.282639478Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203532024-02-03T17:18:33.703Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.282850783Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203542024-02-03T17:18:33.703Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.283133477Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203552024-02-03T17:18:33.703Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.283215436Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203562024-02-03T17:18:33.703Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.28341929Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203572024-02-03T17:18:33.703Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.283478015Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203582024-02-03T17:18:33.708Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.283524704Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203592024-02-03T17:18:33.708Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.283726252Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203602024-02-03T17:18:33.708Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.2837974Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203612024-02-03T17:18:33.708Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.283825255Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203622024-02-03T17:18:33.708Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.284170382Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203632024-02-03T17:18:33.708Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.284954072Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203642024-02-03T17:18:33.708Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.285034793Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203652024-02-03T17:18:33.708Z{"msg":"[0]R flush_numbers[0..12]: [314, 322, 323, 322, 317, 317, 317, 318, 299, 315, 323, 323]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.285070831Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203662024-02-03T17:18:33.708Z{"msg":"[0]R generation[0..12]: [181, 186, 187, 186, 184, 184, 184, 185, 174, 182, 187, 187]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.285139881Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203672024-02-03T17:18:33.708Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.285173701Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203682024-02-03T17:18:33.708Z{"msg":"[1]R flush_numbers[0..12]: [314, 322, 323, 322, 317, 317, 317, 318, 299, 315, 323, 323]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.285238267Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203692024-02-03T17:18:33.708Z{"msg":"[1]R generation[0..12]: [181, 186, 187, 186, 184, 184, 184, 185, 174, 182, 187, 187]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.285270014Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203702024-02-03T17:18:33.708Z{"msg":"Wait for a query_work_queue command to finish before sending IO
203712024-02-03T17:18:33.708Z[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.285338296Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203722024-02-03T17:18:33.708Z{"msg":"[2]R flush_numbers[0..12]: [314, 322, 323, 322, 317, 317, 317, 318, 299, 315, 323, 323]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.285373313Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203732024-02-03T17:18:33.708Z{"msg":"[2]R generation[0..12]: [181, 186, 187, 186, 184, 184, 184, 185, 174, 182, 187, 187]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.285437096Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203742024-02-03T17:18:33.708Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.285467532Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203752024-02-03T17:18:33.709Z{"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.285531305Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203762024-02-03T17:18:33.709Z{"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.285559542Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203772024-02-03T17:18:33.709Z{"msg":"Next flush: 324","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.285635763Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203782024-02-03T17:18:33.709Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.285675824Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203792024-02-03T17:18:33.709Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.285735879Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203802024-02-03T17:18:33.709Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.285760347Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203812024-02-03T17:18:33.709Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.285825957Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203822024-02-03T17:18:33.709Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.285854016Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203832024-02-03T17:18:33.709Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.285923706Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203842024-02-03T17:18:33.709Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.285949989Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203852024-02-03T17:18:33.709Z{"msg":"9941d150-2665-41b5-96d3-719054cac957 is now active with session: 9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.286016061Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203862024-02-03T17:18:33.709Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.286041695Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887,"session_id":"9a1b41c7-429a-4028-bec0-9b2d3f8ec2d0"}
203872024-02-03T17:18:33.709ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
203882024-02-03T17:18:33.709Z{"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:33.286106748Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887}
203892024-02-03T17:18:33.709ZLoading write count information from file "/tmp/test_repair_verify.data"
203902024-02-03T17:18:33.709ZRun Repair workload
203912024-02-03T17:18:33.709Z01/30 Write block 361 len 4096 data: 15
203922024-02-03T17:18:33.713Z02/30 Read block 169 len 28672
203932024-02-03T17:18:33.713Z03/30 Read block 98 len 40960
203942024-02-03T17:18:33.713Z04/30 Flush
203952024-02-03T17:18:33.716Z05/30 Write block 551 len 16384 data: 16 16 18 18
203962024-02-03T17:18:33.716Z06/30 Read block 270 len 4096
203972024-02-03T17:18:33.719Z07/30 Write block 534 len 36864 data: 11 11 8 10 12 11 9 11 14
203982024-02-03T17:18:33.719Z08/30 Write block 34 len 8192 data: 15 15
203992024-02-03T17:18:33.719Z09/30 Read block 582 len 24576
204002024-02-03T17:18:33.722Z10/30 Read block 147 len 32768
204012024-02-03T17:18:33.722Z11/30 Write block 12 len 4096 data: 14
204022024-02-03T17:18:33.727Z12/30 Write block 231 len 40960 data: 12 15 15 18 18 15 14 13 14 11
204032024-02-03T17:18:33.727Z13/30 Read block 220 len 20480
204042024-02-03T17:18:33.730Z14/30 Write block 178 len 16384 data: 13 13 13 17
204052024-02-03T17:18:33.730Z15/30 Read block 567 len 36864
204062024-02-03T17:18:33.730Z16/30 Write block 467 len 4096 data: 15
204072024-02-03T17:18:33.733Z17/30 Write block 34 len 24576 data: 16 16 17 15 14 14
204082024-02-03T17:18:33.733Z18/30 Flush
204092024-02-03T17:18:33.737Z19/30 Flush
204102024-02-03T17:18:33.740Z20/30 Flush
204112024-02-03T17:18:33.740Z21/30 Write block 320 len 32768 data: 18 17 14 15 15 14 14 19
204122024-02-03T17:18:33.740Z22/30 Read block 345 len 28672
204132024-02-03T17:18:33.743Z23/30 Read block 151 len 32768
204142024-02-03T17:18:33.748Z24/30 Read block 175 len 36864
204152024-02-03T17:18:33.748Z25/30 Read block 182 len 40960
204162024-02-03T17:18:33.758Z26/30 Read block 389 len 4096
204172024-02-03T17:18:33.758Z27/30 Read block 481 len 16384
204182024-02-03T17:18:33.758Z28/30 Write block 104 len 36864 data: 9 10 13 13 8 8 9 11 12
204192024-02-03T17:18:33.758Z29/30 Read block 597 len 8192
204202024-02-03T17:18:33.758Z30/30 Read block 275 len 12288
204212024-02-03T17:18:33.779Z----------------------------------------------------------------
204222024-02-03T17:18:33.779Z Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:26
204232024-02-03T17:18:33.779ZStates: Active Active Active
204242024-02-03T17:18:33.779ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
204252024-02-03T17:18:33.780Z 5 Acked 1004 Write 4 Done Done Done false
204262024-02-03T17:18:33.780Z 6 Acked 1005 Read 1 Done Done Done false
204272024-02-03T17:18:33.780Z 7 Acked 1006 Write 9 Done Sent Done false
204282024-02-03T17:18:33.780Z 8 Acked 1007 Write 2 Done Sent Done false
204292024-02-03T17:18:33.780Z 9 Acked 1008 Read 6 Done Sent Done false
204302024-02-03T17:18:33.780Z 10 Acked 1009 Read 8 Done Sent Done false
204312024-02-03T17:18:33.780Z 11 Acked 1010 Write 1 Done Sent Done false
204322024-02-03T17:18:33.780Z 12 Acked 1011 Write 10 Done Sent Done false
204332024-02-03T17:18:33.781Z 13 Acked 1012 Read 5 Done Sent Done false
204342024-02-03T17:18:33.781Z 14 Acked 1013 Write 4 Done Sent Done false
204352024-02-03T17:18:33.781Z 15 Acked 1014 Read 9 Done Sent Done false
204362024-02-03T17:18:33.781Z 16 Acked 1015 Write 1 Done Sent Done false
204372024-02-03T17:18:33.781Z 17 Acked 1016 Write 6 Done Sent Done false
204382024-02-03T17:18:33.781Z 18 Acked 1017 Flush 0 Done Sent Done false
204392024-02-03T17:18:33.781Z 19 Acked 1018 Flush 0 Done Sent Done false
204402024-02-03T17:18:33.781Z 20 Acked 1019 Flush 0 Done Sent Done false
204412024-02-03T17:18:33.781Z 21 Acked 1020 Write 8 Done Sent Done false
204422024-02-03T17:18:33.782Z 22 Acked 1021 Read 7 Done Sent Done false
204432024-02-03T17:18:33.782Z 23 Acked 1022 Read 8 Done Sent Done false
204442024-02-03T17:18:33.782Z 24 Acked 1023 Read 9 Done Sent Done false
204452024-02-03T17:18:33.782Z 25 Acked 1024 Read 10 Done Sent Done false
204462024-02-03T17:18:33.782Z 26 Acked 1025 Read 1 Done Sent Done false
204472024-02-03T17:18:33.782Z 27 Acked 1026 Read 4 Done Sent Done false
204482024-02-03T17:18:33.782Z 28 Acked 1027 Write 9 Done Sent Done false
204492024-02-03T17:18:33.782Z 29 Acked 1028 Read 2 Done Sent Done false
204502024-02-03T17:18:33.782Z 30 Acked 1029 Read 3 Done Sent Done false
204512024-02-03T17:18:33.782Z STATES DS:0 DS:1 DS:2 TOTAL
204522024-02-03T17:18:33.783Z New 0 0 0 0
204532024-02-03T17:18:33.783Z Sent 0 24 0 24
204542024-02-03T17:18:33.783Z Done 26 2 26 54
204552024-02-03T17:18:33.783Z Skipped 0 0 0 0
204562024-02-03T17:18:33.783Z Error 0 0 0 0
204572024-02-03T17:18:33.783ZLast Flush: 1019 1003 1019
204582024-02-03T17:18:33.783ZDownstairs last five completed: 1003 1002 1001 1000
204592024-02-03T17:18:33.783ZUpstairs last five completed: 30 29 28 27 26
204602024-02-03T17:18:33.783Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:18:33.336735555Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1887}
204612024-02-03T17:18:33.783ZWrote out file "/tmp/test_repair_verify.data"
204622024-02-03T17:18:33.783Z
204632024-02-03T17:18:33.784ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
204642024-02-03T17:18:33.791ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
204652024-02-03T17:18:33.792Z 0 000-019 188 181 188 325 314 325 F F F <---
204662024-02-03T17:18:33.795Z 1 020-039 188 186 188 325 322 325 F F F <---
204672024-02-03T17:18:33.795Z 2 040-059 187 187 187 323 323 323 F F F
204682024-02-03T17:18:33.795Z 3 060-079 186 186 186 322 322 322 F F F
204692024-02-03T17:18:33.795Z 4 080-099 184 184 184 317 317 317 F F F
204702024-02-03T17:18:33.795Z 5 100-119 184 184 184 317 317 317 T F T <---
204712024-02-03T17:18:33.795Z 6 120-139 184 184 184 317 317 317 F F F
204722024-02-03T17:18:33.795Z 7 140-159 185 185 185 318 318 318 F F F
204732024-02-03T17:18:33.795Z 8 160-179 188 174 188 325 299 325 F F F <---
204742024-02-03T17:18:33.795Z 9 180-199 188 182 188 325 315 325 F F F <---
204752024-02-03T17:18:33.795Z 10 200-219 187 187 187 323 323 323 F F F
204762024-02-03T17:18:33.795Z 11 220-239 188 187 188 325 323 325 F F F <---
204772024-02-03T17:18:33.795Z 12 240-259 188 180 188 325 311 325 F F F <---
204782024-02-03T17:18:33.795Z 13 260-279 187 187 187 323 323 323 F F F
204792024-02-03T17:18:33.796Z 14 280-299 185 185 185 318 318 318 F F F
204802024-02-03T17:18:33.796Z 15 300-319 184 184 184 317 317 317 F F F
204812024-02-03T17:18:33.796Z 16 320-339 177 177 177 304 304 304 T F T <---
204822024-02-03T17:18:33.796Z 17 340-359 181 181 181 314 314 314 F F F
204832024-02-03T17:18:33.796Z 18 360-379 188 188 188 324 324 324 F F F
204842024-02-03T17:18:33.796Z 19 380-399 184 184 184 317 317 317 F F F
204852024-02-03T17:18:33.796Z 20 400-419 186 186 186 319 319 319 F F F
204862024-02-03T17:18:33.796Z 21 420-439 186 186 186 321 321 321 F F F
204872024-02-03T17:18:33.796Z 22 440-459 182 182 182 315 315 315 F F F
204882024-02-03T17:18:33.796Z 23 460-479 188 184 188 325 317 325 F F F <---
204892024-02-03T17:18:33.796Z 24 480-499 182 182 182 315 315 315 F F F
204902024-02-03T17:18:33.796Z 25 500-519 182 182 182 315 315 315 F F F
204912024-02-03T17:18:33.796Z 26 520-539 188 184 188 325 317 325 F F F <---
204922024-02-03T17:18:33.796Z 27 540-559 188 180 188 325 312 325 F T F <---
204932024-02-03T17:18:33.796Z 28 560-579 186 186 186 319 319 319 F F F
204942024-02-03T17:18:33.796Z 29 580-599 185 185 185 318 318 318 F F F
204952024-02-03T17:18:33.796ZMax gen: 188, Max flush: 325
204962024-02-03T17:18:33.796ZError: Difference in extent metadata found!
204972024-02-03T17:18:33.799ZA Difference in extent metadata is expected here
204982024-02-03T17:18:33.800ZOn loop 94
204992024-02-03T17:18:33.800Z
205002024-02-03T17:18:33.802ZVerifying data now
205012024-02-03T17:18:38.766ZNo change after verify
205022024-02-03T17:18:38.766ZLoop: 94 Downstairs dump after verify (and repair):
205032024-02-03T17:18:38.791ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
205042024-02-03T17:18:38.791Z 0 000-019 188 188 188 325 325 325 F F F
205052024-02-03T17:18:38.791Z 1 020-039 188 188 188 325 325 325 F F F
205062024-02-03T17:18:38.791Z 2 040-059 187 187 187 323 323 323 F F F
205072024-02-03T17:18:38.795Z 3 060-079 186 186 186 322 322 322 F F F
205082024-02-03T17:18:38.795Z 4 080-099 184 184 184 317 317 317 F F F
205092024-02-03T17:18:38.795Z 5 100-119 189 189 189 326 326 326 F F F
205102024-02-03T17:18:38.795Z 6 120-139 184 184 184 317 317 317 F F F
205112024-02-03T17:18:38.795Z 7 140-159 185 185 185 318 318 318 F F F
205122024-02-03T17:18:38.795Z 8 160-179 188 188 188 325 325 325 F F F
205132024-02-03T17:18:38.795Z 9 180-199 188 188 188 325 325 325 F F F
205142024-02-03T17:18:38.795Z 10 200-219 187 187 187 323 323 323 F F F
205152024-02-03T17:18:38.795Z 11 220-239 188 188 188 325 325 325 F F F
205162024-02-03T17:18:38.796Z 12 240-259 188 188 188 325 325 325 F F F
205172024-02-03T17:18:38.796Z 13 260-279 187 187 187 323 323 323 F F F
205182024-02-03T17:18:38.796Z 14 280-299 185 185 185 318 318 318 F F F
205192024-02-03T17:18:38.796Z 15 300-319 184 184 184 317 317 317 F F F
205202024-02-03T17:18:38.796Z 16 320-339 189 189 189 326 326 326 F F F
205212024-02-03T17:18:38.796Z 17 340-359 181 181 181 314 314 314 F F F
205222024-02-03T17:18:38.796Z 18 360-379 188 188 188 324 324 324 F F F
205232024-02-03T17:18:38.796Z 19 380-399 184 184 184 317 317 317 F F F
205242024-02-03T17:18:38.796Z 20 400-419 186 186 186 319 319 319 F F F
205252024-02-03T17:18:38.796Z 21 420-439 186 186 186 321 321 321 F F F
205262024-02-03T17:18:38.796Z 22 440-459 182 182 182 315 315 315 F F F
205272024-02-03T17:18:38.796Z 23 460-479 188 188 188 325 325 325 F F F
205282024-02-03T17:18:38.796Z 24 480-499 182 182 182 315 315 315 F F F
205292024-02-03T17:18:38.797Z 25 500-519 182 182 182 315 315 315 F F F
205302024-02-03T17:18:38.797Z 26 520-539 188 188 188 325 325 325 F F F
205312024-02-03T17:18:38.797Z 27 540-559 188 188 188 325 325 325 F F F
205322024-02-03T17:18:38.797Z 28 560-579 186 186 186 319 319 319 F F F
205332024-02-03T17:18:38.797Z 29 580-599 185 185 185 318 318 318 F F F
205342024-02-03T17:18:38.797ZMax gen: 189, Max flush: 326
205352024-02-03T17:18:38.797Z
205362024-02-03T17:18:38.797ZBegin loop 95
205372024-02-03T17:18:38.797ZDownstairs to restart: 0
205382024-02-03T17:18:38.828Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.407827765Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895}
205392024-02-03T17:18:38.832Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.40842632Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895}
205402024-02-03T17:18:38.833Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.408520521Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895}
205412024-02-03T17:18:38.833Z{"msg":"Crucible 396cb012-67ca-45f9-9530-320997c6742a has session id: a82fcf23-aeb6-41c2-845e-3e0d59c15948","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.408575873Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205422024-02-03T17:18:38.833Z{"msg":"Upstairs opts: Upstairs UUID: 396cb012-67ca-45f9-9530-320997c6742a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.408643708Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205432024-02-03T17:18:38.833Z{"msg":"Crucible stats registered with UUID: 396cb012-67ca-45f9-9530-320997c6742a","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.408682739Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205442024-02-03T17:18:38.833ZCrucible runtime is spawned
205452024-02-03T17:18:38.833Z{"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.408976203Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895}
205462024-02-03T17:18:38.833Z{"msg":"396cb012-67ca-45f9-9530-320997c6742a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.409240525Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205472024-02-03T17:18:38.833Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.409376798Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205482024-02-03T17:18:38.833Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.409468347Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205492024-02-03T17:18:38.833Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.409549897Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205502024-02-03T17:18:38.833Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.409580019Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205512024-02-03T17:18:38.833Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.409638654Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205522024-02-03T17:18:38.834Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.40968476Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205532024-02-03T17:18:38.834Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.40978554Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"":"io task","client":"2","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205542024-02-03T17:18:38.834Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.409895475Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"":"io task","client":"0","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205552024-02-03T17:18:38.834Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.409942707Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"":"io task","client":"1","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205562024-02-03T17:18:38.834Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.41009477Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"":"io task","client":"1","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205572024-02-03T17:18:38.834Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.4105166Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"":"io task","client":"0","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205582024-02-03T17:18:38.834Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.410648862Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"":"io task","client":"2","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205592024-02-03T17:18:38.834Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.411415058Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205602024-02-03T17:18:38.834Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.411486385Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205612024-02-03T17:18:38.834Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.411513763Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205622024-02-03T17:18:38.834Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.411536375Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205632024-02-03T17:18:38.834Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.412074961Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205642024-02-03T17:18:38.834Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.412188258Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205652024-02-03T17:18:38.835Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.412238363Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205662024-02-03T17:18:38.835Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.412633414Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205672024-02-03T17:18:38.835Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.413139089Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205682024-02-03T17:18:38.835Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.413265405Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205692024-02-03T17:18:38.835Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.413324084Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205702024-02-03T17:18:38.835Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.413367681Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205712024-02-03T17:18:38.840Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.413726782Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205722024-02-03T17:18:38.840Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.413829015Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205732024-02-03T17:18:38.841Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.413914079Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205742024-02-03T17:18:38.841Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.414244868Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205752024-02-03T17:18:38.841Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.414336061Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205762024-02-03T17:18:38.841Z{"msg":"[0]R flush_numbers[0..12]: [325, 325, 323, 322, 317, 326, 317, 318, 325, 325, 323, 325]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.414372736Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205772024-02-03T17:18:38.841Z{"msg":"[0]R generation[0..12]: [188, 188, 187, 186, 184, 189, 184, 185, 188, 188, 187, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.414440779Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205782024-02-03T17:18:38.841Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.414476414Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205792024-02-03T17:18:38.841Z{"msg":"[1]R flush_numbers[0..12]: [325, 325, 323, 322, 317, 326, 317, 318, 325, 325, 323, 325]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.414526913Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205802024-02-03T17:18:38.841Z{"msg":"[1]R generation[0..12]: [188, 188, 187, 186, 184, 189, 184, 185, 188, 188, 187, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.414558026Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205812024-02-03T17:18:38.841Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.414646112Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205822024-02-03T17:18:38.841Z{"msg":"[2]R flush_numbers[0..12]: [325, 325, 323, 322, 317, 326, 317, 318, 325, 325, 323, 325]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.414675225Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205832024-02-03T17:18:38.841Z{"msg":"Wait for a query_work_queue command to finish before sending IO
205842024-02-03T17:18:38.841Z[2]R generation[0..12]: [188, 188, 187, 186, 184, 189, 184, 185, 188, 188, 187, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.414715252Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205852024-02-03T17:18:38.841Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.414751358Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205862024-02-03T17:18:38.842Z{"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.414814447Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205872024-02-03T17:18:38.842Z{"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.414841248Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205882024-02-03T17:18:38.842Z{"msg":"Next flush: 327","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.414908809Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205892024-02-03T17:18:38.842Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.414936967Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205902024-02-03T17:18:38.842Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.414978051Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205912024-02-03T17:18:38.842Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.415014576Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205922024-02-03T17:18:38.842Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.415050567Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205932024-02-03T17:18:38.842Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.415088893Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205942024-02-03T17:18:38.842Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.415129653Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205952024-02-03T17:18:38.842Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.415160282Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205962024-02-03T17:18:38.842Z{"msg":"396cb012-67ca-45f9-9530-320997c6742a is now active with session: a82fcf23-aeb6-41c2-845e-3e0d59c15948","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.415182854Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205972024-02-03T17:18:38.842Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.415245995Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895,"session_id":"a82fcf23-aeb6-41c2-845e-3e0d59c15948"}
205982024-02-03T17:18:38.842Z{"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:38.415278208Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895}
205992024-02-03T17:18:38.843ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
206002024-02-03T17:18:38.843ZLoading write count information from file "/tmp/test_repair_verify.data"
206012024-02-03T17:18:38.843ZRun Repair workload
206022024-02-03T17:18:38.843Z01/30 Write block 394 len 28672 data: 19 18 18 14 16 16 14
206032024-02-03T17:18:38.843Z02/30 Read block 200 len 12288
206042024-02-03T17:18:38.843Z03/30 Read block 131 len 20480
206052024-02-03T17:18:38.847Z04/30 Read block 29 len 4096
206062024-02-03T17:18:38.847Z05/30 Read block 572 len 40960
206072024-02-03T17:18:38.847Z06/30 Write block 99 len 36864 data: 14 15 14 10 10 10 11 14 14
206082024-02-03T17:18:38.847Z07/30 Write block 145 len 20480 data: 13 13 10 15 16
206092024-02-03T17:18:38.847Z08/30 Read block 134 len 4096
206102024-02-03T17:18:38.851Z09/30 Write block 140 len 28672 data: 14 17 18 16 12 14 14
206112024-02-03T17:18:38.851Z10/30 Read block 324 len 36864
206122024-02-03T17:18:38.856Z11/30 Write block 264 len 24576 data: 22 22 20 19 23 16
206132024-02-03T17:18:38.856Z12/30 Write block 138 len 28672 data: 11 13 15 18 19 17 13
206142024-02-03T17:18:38.856Z13/30 Write block 255 len 20480 data: 9 13 14 15 16
206152024-02-03T17:18:38.860Z14/30 Write block 300 len 36864 data: 10 13 13 13 16 16 19 16 17
206162024-02-03T17:18:38.860Z15/30 Write block 554 len 20480 data: 19 15 16 15 12
206172024-02-03T17:18:38.860Z16/30 Write block 437 len 24576 data: 16 16 17 19 19 22
206182024-02-03T17:18:38.860Z17/30 Read block 355 len 28672
206192024-02-03T17:18:38.864Z18/30 Read block 502 len 36864
206202024-02-03T17:18:38.864Z19/30 Write block 355 len 12288 data: 15 12 13
206212024-02-03T17:18:38.864Z20/30 Read block 479 len 16384
206222024-02-03T17:18:38.868Z21/30 Read block 299 len 32768
206232024-02-03T17:18:38.868Z22/30 Read block 396 len 16384
206242024-02-03T17:18:38.871Z23/30 Flush
206252024-02-03T17:18:38.877Z24/30 Write block 499 len 12288 data: 11 9 10
206262024-02-03T17:18:38.877Z25/30 Write block 376 len 24576 data: 16 19 17 19 18 16
206272024-02-03T17:18:38.877Z26/30 Read block 201 len 20480
206282024-02-03T17:18:38.881Z27/30 Write block 349 len 32768 data: 14 13 12 14 13 14 16 13
206292024-02-03T17:18:38.881Z28/30 Write block 165 len 8192 data: 18 19
206302024-02-03T17:18:38.881Z29/30 Read block 288 len 24576
206312024-02-03T17:18:38.885Z30/30 Read block 358 len 36864
206322024-02-03T17:18:38.888Z----------------------------------------------------------------
206332024-02-03T17:18:38.889Z Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:30
206342024-02-03T17:18:38.889ZStates: Active Active Active
206352024-02-03T17:18:38.889ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
206362024-02-03T17:18:38.889Z 1 Acked 1000 Write 7 Done Done Done false
206372024-02-03T17:18:38.889Z 2 Acked 1001 Read 3 Done Done Done false
206382024-02-03T17:18:38.889Z 3 Acked 1002 Read 5 Sent Done Done false
206392024-02-03T17:18:38.889Z 4 Acked 1003 Read 1 Sent Done Done false
206402024-02-03T17:18:38.889Z 5 Acked 1004 Read 10 Sent Done Done false
206412024-02-03T17:18:38.889Z 6 Acked 1005 Write 9 Sent Done Done false
206422024-02-03T17:18:38.889Z 7 Acked 1006 Write 5 Sent Done Done false
206432024-02-03T17:18:38.889Z 8 Acked 1007 Read 1 Sent Done Done false
206442024-02-03T17:18:38.889Z 9 Acked 1008 Write 7 Sent Done Done false
206452024-02-03T17:18:38.890Z 10 Acked 1009 Read 9 Sent Done Done false
206462024-02-03T17:18:38.890Z 11 Acked 1010 Write 6 Sent Done Done false
206472024-02-03T17:18:38.890Z 12 Acked 1011 Write 7 Sent Done Done false
206482024-02-03T17:18:38.890Z 13 Acked 1012 Write 5 Sent Done Done false
206492024-02-03T17:18:38.890Z 14 Acked 1013 Write 9 Sent Done Done false
206502024-02-03T17:18:38.890Z 15 Acked 1014 Write 5 Sent Done Done false
206512024-02-03T17:18:38.890Z 16 Acked 1015 Write 6 Sent Done Done false
206522024-02-03T17:18:38.890Z 17 Acked 1016 Read 7 Sent Done Done false
206532024-02-03T17:18:38.890Z 18 Acked 1017 Read 9 Sent Done Done false
206542024-02-03T17:18:38.890Z 19 Acked 1018 Write 3 Sent Done Done false
206552024-02-03T17:18:38.890Z 20 Acked 1019 Read 4 Sent Done Done false
206562024-02-03T17:18:38.890Z 21 Acked 1020 Read 8 Sent Done Done false
206572024-02-03T17:18:38.890Z 22 Acked 1021 Read 4 Sent Done Done false
206582024-02-03T17:18:38.891Z 23 Acked 1022 Flush 0 Sent Done Done false
206592024-02-03T17:18:38.891Z 24 Acked 1023 Write 3 Sent Done Done false
206602024-02-03T17:18:38.891Z 25 Acked 1024 Write 6 Sent Done Done false
206612024-02-03T17:18:38.891Z 26 Acked 1025 Read 5 Sent Done Done false
206622024-02-03T17:18:38.891Z 27 Acked 1026 Write 8 Sent Done Done false
206632024-02-03T17:18:38.891Z 28 Acked 1027 Write 2 Sent Done Done false
206642024-02-03T17:18:38.891Z 29 Acked 1028 Read 6 Sent Done Done false
206652024-02-03T17:18:38.891Z 30 Acked 1029 Read 9 Sent Sent Done false
206662024-02-03T17:18:38.891Z STATES DS:0 DS:1 DS:2 TOTAL
206672024-02-03T17:18:38.891Z New 0 0 0 0
206682024-02-03T17:18:38.891Z Sent 28 1 0 29
206692024-02-03T17:18:38.891Z Done 2 29 30 61
206702024-02-03T17:18:38.891Z Skipped 0 0 0 0
206712024-02-03T17:18:38.892Z Error 0 0 0 0
206722024-02-03T17:18:38.892ZLast Flush: 0 1022 1022
206732024-02-03T17:18:38.892ZDownstairs last five completed:
206742024-02-03T17:18:38.892ZUpstairs last five completed: 30 29 28 27 26
206752024-02-03T17:18:38.892Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:18:38.468279113Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1895}
206762024-02-03T17:18:38.892ZWrote out file "/tmp/test_repair_verify.data"
206772024-02-03T17:18:38.896Z
206782024-02-03T17:18:38.900ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
206792024-02-03T17:18:38.927ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
206802024-02-03T17:18:38.927Z 0 000-019 188 188 188 325 325 325 F F F
206812024-02-03T17:18:38.927Z 1 020-039 188 188 188 325 325 325 F F F
206822024-02-03T17:18:38.927Z 2 040-059 187 187 187 323 323 323 F F F
206832024-02-03T17:18:38.928Z 3 060-079 186 186 186 322 322 322 F F F
206842024-02-03T17:18:38.928Z 4 080-099 184 190 190 317 327 327 F F F <---
206852024-02-03T17:18:38.928Z 5 100-119 189 190 190 326 327 327 F F F <---
206862024-02-03T17:18:38.928Z 6 120-139 184 190 190 317 327 327 F F F <---
206872024-02-03T17:18:38.928Z 7 140-159 185 190 190 318 327 327 F F F <---
206882024-02-03T17:18:38.932Z 8 160-179 188 188 188 325 325 325 F T T <---
206892024-02-03T17:18:38.932Z 9 180-199 188 188 188 325 325 325 F F F
206902024-02-03T17:18:38.932Z 10 200-219 187 187 187 323 323 323 F F F
206912024-02-03T17:18:38.932Z 11 220-239 188 188 188 325 325 325 F F F
206922024-02-03T17:18:38.932Z 12 240-259 188 190 190 325 327 327 F F F <---
206932024-02-03T17:18:38.932Z 13 260-279 187 190 190 323 327 327 F F F <---
206942024-02-03T17:18:38.932Z 14 280-299 185 185 185 318 318 318 F F F
206952024-02-03T17:18:38.932Z 15 300-319 184 190 190 317 327 327 F F F <---
206962024-02-03T17:18:38.932Z 16 320-339 189 189 189 326 326 326 F F F
206972024-02-03T17:18:38.932Z 17 340-359 181 190 190 314 327 327 F T T <---
206982024-02-03T17:18:38.932Z 18 360-379 188 188 188 324 324 324 F T T <---
206992024-02-03T17:18:38.932Z 19 380-399 184 190 190 317 327 327 T T T <---
207002024-02-03T17:18:38.933Z 20 400-419 186 190 190 319 327 327 T F F <---
207012024-02-03T17:18:38.933Z 21 420-439 186 190 190 321 327 327 F F F <---
207022024-02-03T17:18:38.933Z 22 440-459 182 190 190 315 327 327 F F F <---
207032024-02-03T17:18:38.933Z 23 460-479 188 188 188 325 325 325 F F F
207042024-02-03T17:18:38.933Z 24 480-499 182 182 182 315 315 315 F T T <---
207052024-02-03T17:18:38.933Z 25 500-519 182 182 182 315 315 315 F T T <---
207062024-02-03T17:18:38.933Z 26 520-539 188 188 188 325 325 325 F F F
207072024-02-03T17:18:38.933Z 27 540-559 188 190 190 325 327 327 F F F <---
207082024-02-03T17:18:38.933Z 28 560-579 186 186 186 319 319 319 F F F
207092024-02-03T17:18:38.933Z 29 580-599 185 185 185 318 318 318 F F F
207102024-02-03T17:18:38.933ZMax gen: 190, Max flush: 327
207112024-02-03T17:18:38.933ZError: Difference in extent metadata found!
207122024-02-03T17:18:38.937ZA Difference in extent metadata is expected here
207132024-02-03T17:18:38.937ZOn loop 95
207142024-02-03T17:18:38.937Z
207152024-02-03T17:18:38.937ZVerifying data now
207162024-02-03T17:18:45.353ZNo change after verify
207172024-02-03T17:18:45.354ZLoop: 95 Downstairs dump after verify (and repair):
207182024-02-03T17:18:45.376ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
207192024-02-03T17:18:45.376Z 0 000-019 188 188 188 325 325 325 F F F
207202024-02-03T17:18:45.376Z 1 020-039 188 188 188 325 325 325 F F F
207212024-02-03T17:18:45.376Z 2 040-059 187 187 187 323 323 323 F F F
207222024-02-03T17:18:45.379Z 3 060-079 186 186 186 322 322 322 F F F
207232024-02-03T17:18:45.380Z 4 080-099 190 190 190 327 327 327 F F F
207242024-02-03T17:18:45.380Z 5 100-119 190 190 190 327 327 327 F F F
207252024-02-03T17:18:45.380Z 6 120-139 190 190 190 327 327 327 F F F
207262024-02-03T17:18:45.380Z 7 140-159 190 190 190 327 327 327 F F F
207272024-02-03T17:18:45.380Z 8 160-179 191 191 191 328 328 328 F F F
207282024-02-03T17:18:45.380Z 9 180-199 188 188 188 325 325 325 F F F
207292024-02-03T17:18:45.380Z 10 200-219 187 187 187 323 323 323 F F F
207302024-02-03T17:18:45.380Z 11 220-239 188 188 188 325 325 325 F F F
207312024-02-03T17:18:45.380Z 12 240-259 190 190 190 327 327 327 F F F
207322024-02-03T17:18:45.380Z 13 260-279 190 190 190 327 327 327 F F F
207332024-02-03T17:18:45.380Z 14 280-299 185 185 185 318 318 318 F F F
207342024-02-03T17:18:45.380Z 15 300-319 190 190 190 327 327 327 F F F
207352024-02-03T17:18:45.380Z 16 320-339 189 189 189 326 326 326 F F F
207362024-02-03T17:18:45.380Z 17 340-359 191 191 191 328 328 328 F F F
207372024-02-03T17:18:45.380Z 18 360-379 191 191 191 328 328 328 F F F
207382024-02-03T17:18:45.380Z 19 380-399 191 191 191 328 328 328 F F F
207392024-02-03T17:18:45.380Z 20 400-419 190 190 190 327 327 327 F F F
207402024-02-03T17:18:45.380Z 21 420-439 190 190 190 327 327 327 F F F
207412024-02-03T17:18:45.380Z 22 440-459 190 190 190 327 327 327 F F F
207422024-02-03T17:18:45.380Z 23 460-479 188 188 188 325 325 325 F F F
207432024-02-03T17:18:45.380Z 24 480-499 191 191 191 328 328 328 F F F
207442024-02-03T17:18:45.380Z 25 500-519 191 191 191 328 328 328 F F F
207452024-02-03T17:18:45.380Z 26 520-539 188 188 188 325 325 325 F F F
207462024-02-03T17:18:45.380Z 27 540-559 190 190 190 327 327 327 F F F
207472024-02-03T17:18:45.381Z 28 560-579 186 186 186 319 319 319 F F F
207482024-02-03T17:18:45.381Z 29 580-599 185 185 185 318 318 318 F F F
207492024-02-03T17:18:45.381ZMax gen: 191, Max flush: 328
207502024-02-03T17:18:45.384Z
207512024-02-03T17:18:45.384ZBegin loop 96
207522024-02-03T17:18:45.384ZDownstairs to restart: 2
207532024-02-03T17:18:45.413Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:44.995694868Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903}
207542024-02-03T17:18:45.413Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:44.996210371Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903}
207552024-02-03T17:18:45.417Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
207562024-02-03T17:18:45.417Z,"time":"2024-02-03T17:18:44.996254197Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903}
207572024-02-03T17:18:45.417Z{"msg":"Crucible 941c216e-3f75-4cab-bc08-f3369c5612fc has session id: 1baf8615-dd07-4a37-8984-a87a495e69a4","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:44.996414322Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207582024-02-03T17:18:45.417Z{"msg":"Upstairs opts: Upstairs UUID: 941c216e-3f75-4cab-bc08-f3369c5612fc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:44.996524324Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207592024-02-03T17:18:45.417Z{"msg":"Crucible stats registered with UUID: 941c216e-3f75-4cab-bc08-f3369c5612fc","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:44.996610851Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207602024-02-03T17:18:45.417Z{"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:44.996690303Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903}
207612024-02-03T17:18:45.417Z{"msg":"941c216e-3f75-4cab-bc08-f3369c5612fc active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:44.996727943Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207622024-02-03T17:18:45.417Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:44.996803799Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207632024-02-03T17:18:45.417Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:44.996903333Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207642024-02-03T17:18:45.417Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:44.99701709Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207652024-02-03T17:18:45.417Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:44.997060278Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207662024-02-03T17:18:45.417Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:44.997130982Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207672024-02-03T17:18:45.417Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:44.997212204Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207682024-02-03T17:18:45.417Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:44.997252388Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"":"io task","client":"2","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207692024-02-03T17:18:45.417Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:44.997292901Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"":"io task","client":"0","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207702024-02-03T17:18:45.417Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:44.997330947Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"":"io task","client":"1","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207712024-02-03T17:18:45.417Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:44.997370878Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"":"io task","client":"2","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207722024-02-03T17:18:45.417Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:44.997452277Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"":"io task","client":"0","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207732024-02-03T17:18:45.417Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:44.997676312Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"":"io task","client":"1","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207742024-02-03T17:18:45.417Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:44.998671418Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207752024-02-03T17:18:45.417Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:44.99877537Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207762024-02-03T17:18:45.417Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:44.998965111Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207772024-02-03T17:18:45.417Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:44.999061943Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207782024-02-03T17:18:45.422Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:44.999632591Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207792024-02-03T17:18:45.422Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:44.999937028Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207802024-02-03T17:18:45.422Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.000114967Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207812024-02-03T17:18:45.423Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.000208773Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207822024-02-03T17:18:45.423Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.00060261Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207832024-02-03T17:18:45.423Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.000716571Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207842024-02-03T17:18:45.423Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.000794456Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207852024-02-03T17:18:45.423Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.0010628Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207862024-02-03T17:18:45.423Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.00113524Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207872024-02-03T17:18:45.423Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.001162488Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207882024-02-03T17:18:45.423Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.001220447Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207892024-02-03T17:18:45.423Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.001911473Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207902024-02-03T17:18:45.423Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.001974103Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207912024-02-03T17:18:45.423Z{"msg":"[0]R flush_numbers[0..12]: [325, 325, 323, 322, 327, 327, 327, 327, 328, 325, 323, 325]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.002022056Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207922024-02-03T17:18:45.423Z{"msg":"[0]R generation[0..12]: [188, 188, 187, 186, 190, 190, 190, 190, 191, 188, 187, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.002062551Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207932024-02-03T17:18:45.423Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.002107251Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207942024-02-03T17:18:45.423Z{"msg":"[1]R flush_numbers[0..12]: [325, 325, 323, 322, 327, 327, 327, 327, 328, 325, 323, 325]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.002182833Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207952024-02-03T17:18:45.423Z{"msg":"[1]R generation[0..12]: [188, 188, 187, 186, 190, 190, 190, 190, 191, 188, 187, 188]","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO
207962024-02-03T17:18:45.423Zlevel":30,"time":"2024-02-03T17:18:45.002215478Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207972024-02-03T17:18:45.423Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.002267597Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207982024-02-03T17:18:45.423Z{"msg":"[2]R flush_numbers[0..12]: [325, 325, 323, 322, 327, 327, 327, 327, 328, 325, 323, 325]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.002293996Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
207992024-02-03T17:18:45.423Z{"msg":"[2]R generation[0..12]: [188, 188, 187, 186, 190, 190, 190, 190, 191, 188, 187, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.002361822Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
208002024-02-03T17:18:45.423Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.002387929Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
208012024-02-03T17:18:45.423Z{"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.002409978Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
208022024-02-03T17:18:45.423Z{"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.002432328Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
208032024-02-03T17:18:45.423Z{"msg":"Next flush: 329","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.002455127Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
208042024-02-03T17:18:45.424Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.0024769Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
208052024-02-03T17:18:45.424Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.002525572Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
208062024-02-03T17:18:45.424Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.002548235Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
208072024-02-03T17:18:45.424Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.002570184Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
208082024-02-03T17:18:45.424Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.002592645Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
208092024-02-03T17:18:45.424Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.00264154Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
208102024-02-03T17:18:45.424Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.002664806Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
208112024-02-03T17:18:45.424Z{"msg":"941c216e-3f75-4cab-bc08-f3369c5612fc is now active with session: 1baf8615-dd07-4a37-8984-a87a495e69a4","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.002711505Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
208122024-02-03T17:18:45.424Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.002733887Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903,"session_id":"1baf8615-dd07-4a37-8984-a87a495e69a4"}
208132024-02-03T17:18:45.424Z{"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:45.002754372Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903}
208142024-02-03T17:18:45.424ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
208152024-02-03T17:18:45.424ZLoading write count information from file "/tmp/test_repair_verify.data"
208162024-02-03T17:18:45.424ZRun Repair workload
208172024-02-03T17:18:45.427Z01/30 Write block 366 len 28672 data: 14 15 13 13 13 9 12
208182024-02-03T17:18:45.427Z02/30 Read block 250 len 20480
208192024-02-03T17:18:45.428Z03/30 Write block 441 len 40960 data: 20 23 20 15 17 18 18 17 12 11
208202024-02-03T17:18:45.430Z04/30 Read block 196 len 20480
208212024-02-03T17:18:45.430Z05/30 Write block 297 len 12288 data: 13 13 12
208222024-02-03T17:18:45.433Z06/30 Write block 355 len 20480 data: 17 14 14 12 13
208232024-02-03T17:18:45.434Z07/30 Write block 410 len 4096 data: 15
208242024-02-03T17:18:45.434Z08/30 Read block 55 len 16384
208252024-02-03T17:18:45.437Z09/30 Read block 338 len 4096
208262024-02-03T17:18:45.437Z10/30 Read block 361 len 12288
208272024-02-03T17:18:45.437Z11/30 Read block 353 len 36864
208282024-02-03T17:18:45.440Z12/30 Write block 572 len 36864 data: 13 13 15 15 16 13 13 14 13
208292024-02-03T17:18:45.445Z13/30 Read block 18 len 12288
208302024-02-03T17:18:45.449Z14/30 Read block 545 len 28672
208312024-02-03T17:18:45.449Z15/30 Write block 108 len 12288 data: 9 9 10
208322024-02-03T17:18:45.449Z16/30 Read block 544 len 40960
208332024-02-03T17:18:45.449Z17/30 Write block 383 len 40960 data: 16 14 12 11 11 10 8 16 17 22
208342024-02-03T17:18:45.452Z18/30 Write block 551 len 4096 data: 17
208352024-02-03T17:18:45.452Z19/30 Read block 190 len 32768
208362024-02-03T17:18:45.452Z20/30 Read block 136 len 16384
208372024-02-03T17:18:45.455Z21/30 Write block 320 len 24576 data: 19 18 15 16 16 15
208382024-02-03T17:18:45.455Z22/30 Read block 38 len 32768
208392024-02-03T17:18:45.458Z23/30 Write block 60 len 12288 data: 13 13 15
208402024-02-03T17:18:45.458Z24/30 Write block 366 len 20480 data: 15 16 14 14 14
208412024-02-03T17:18:45.461Z25/30 Write block 459 len 20480 data: 16 20 18 16 18
208422024-02-03T17:18:45.461Z26/30 Read block 312 len 36864
208432024-02-03T17:18:45.464Z27/30 Write block 433 len 28672 data: 14 13 17 15 17 17 18
208442024-02-03T17:18:45.464Z28/30 Write block 477 len 16384 data: 12 14 16 14
208452024-02-03T17:18:45.467Z29/30 Write block 465 len 32768 data: 13 16 16 14 14 14 13 12
208462024-02-03T17:18:45.468Z30/30 Write block 328 len 16384 data: 18 17 17 17
208472024-02-03T17:18:45.468Z----------------------------------------------------------------
208482024-02-03T17:18:45.468Z Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:30
208492024-02-03T17:18:45.468ZStates: Active Active Active
208502024-02-03T17:18:45.468ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
208512024-02-03T17:18:45.468Z 1 Acked 1000 Write 7 Done Done Sent false
208522024-02-03T17:18:45.468Z 2 Acked 1001 Read 5 Done Done Sent false
208532024-02-03T17:18:45.468Z 3 Acked 1002 Write 10 Done Done Sent false
208542024-02-03T17:18:45.468Z 4 Acked 1003 Read 5 Done Done Sent false
208552024-02-03T17:18:45.468Z 5 Acked 1004 Write 3 Done Done Sent false
208562024-02-03T17:18:45.468Z 6 Acked 1005 Write 5 Done Done Sent false
208572024-02-03T17:18:45.468Z 7 Acked 1006 Write 1 Done Done Sent false
208582024-02-03T17:18:45.468Z 8 Acked 1007 Read 4 Done Done Sent false
208592024-02-03T17:18:45.468Z 9 Acked 1008 Read 1 Done Done Sent false
208602024-02-03T17:18:45.468Z 10 Acked 1009 Read 3 Done Done Sent false
208612024-02-03T17:18:45.468Z 11 Acked 1010 Read 9 Done Done Sent false
208622024-02-03T17:18:45.468Z 12 Acked 1011 Write 9 Done Done Sent false
208632024-02-03T17:18:45.468Z 13 Acked 1012 Read 3 Done Done Sent false
208642024-02-03T17:18:45.468Z 14 Acked 1013 Read 7 Done Done Sent false
208652024-02-03T17:18:45.468Z 15 Acked 1014 Write 3 Done Done Sent false
208662024-02-03T17:18:45.468Z 16 Acked 1015 Read 10 Done Done Sent false
208672024-02-03T17:18:45.468Z 17 Acked 1016 Write 10 Done Done Sent false
208682024-02-03T17:18:45.468Z 18 Acked 1017 Write 1 Done Done Sent false
208692024-02-03T17:18:45.468Z 19 Acked 1018 Read 8 Done Done Sent false
208702024-02-03T17:18:45.469Z 20 Acked 1019 Read 4 Done Done Sent false
208712024-02-03T17:18:45.469Z 21 Acked 1020 Write 6 Done Done Sent false
208722024-02-03T17:18:45.472Z 22 Acked 1021 Read 8 Done Done Sent false
208732024-02-03T17:18:45.472Z 23 Acked 1022 Write 3 Done Done Sent false
208742024-02-03T17:18:45.472Z 24 Acked 1023 Write 5 Done Done Sent false
208752024-02-03T17:18:45.472Z 25 Acked 1024 Write 5 Done Done Sent false
208762024-02-03T17:18:45.472Z 26 Acked 1025 Read 9 Done Done Sent false
208772024-02-03T17:18:45.472Z 27 Acked 1026 Write 7 Done Done Sent false
208782024-02-03T17:18:45.472Z 28 Acked 1027 Write 4 Done Done Sent false
208792024-02-03T17:18:45.472Z 29 Acked 1028 Write 8 Sent Sent Sent false
208802024-02-03T17:18:45.472Z 30 Acked 1029 Write 4 Sent Sent Sent false
208812024-02-03T17:18:45.472Z STATES DS:0 DS:1 DS:2 TOTAL
208822024-02-03T17:18:45.472Z New 0 0 0 0
208832024-02-03T17:18:45.472Z Sent 2 2 30 34
208842024-02-03T17:18:45.472Z Done 28 28 0 56
208852024-02-03T17:18:45.472Z Skipped 0 0 0 0
208862024-02-03T17:18:45.472Z Error 0 0 0 0
208872024-02-03T17:18:45.473ZLast Flush: 0 0 0
208882024-02-03T17:18:45.473ZDownstairs last five completed:
208892024-02-03T17:18:45.473ZUpstairs last five completed: 30 29 28 27 26
208902024-02-03T17:18:45.473Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:18:45.051141452Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1903}
208912024-02-03T17:18:45.473ZWrote out file "/tmp/test_repair_verify.data"
208922024-02-03T17:18:45.476Z
208932024-02-03T17:18:45.481ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
208942024-02-03T17:18:45.513ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
208952024-02-03T17:18:45.513Z 0 000-019 188 188 188 325 325 325 F F F
208962024-02-03T17:18:45.513Z 1 020-039 188 188 188 325 325 325 F F F
208972024-02-03T17:18:45.513Z 2 040-059 187 187 187 323 323 323 F F F
208982024-02-03T17:18:45.513Z 3 060-079 186 186 186 322 322 322 T T F <---
208992024-02-03T17:18:45.513Z 4 080-099 190 190 190 327 327 327 F F F
209002024-02-03T17:18:45.513Z 5 100-119 190 190 190 327 327 327 T T F <---
209012024-02-03T17:18:45.513Z 6 120-139 190 190 190 327 327 327 F F F
209022024-02-03T17:18:45.513Z 7 140-159 190 190 190 327 327 327 F F F
209032024-02-03T17:18:45.513Z 8 160-179 191 191 191 328 328 328 F F F
209042024-02-03T17:18:45.516Z 9 180-199 188 188 188 325 325 325 F F F
209052024-02-03T17:18:45.516Z 10 200-219 187 187 187 323 323 323 F F F
209062024-02-03T17:18:45.516Z 11 220-239 188 188 188 325 325 325 F F F
209072024-02-03T17:18:45.517Z 12 240-259 190 190 190 327 327 327 F F F
209082024-02-03T17:18:45.517Z 13 260-279 190 190 190 327 327 327 F F F
209092024-02-03T17:18:45.517Z 14 280-299 185 185 185 318 318 318 T T F <---
209102024-02-03T17:18:45.517Z 15 300-319 190 190 190 327 327 327 F F F
209112024-02-03T17:18:45.517Z 16 320-339 189 189 189 326 326 326 T T F <---
209122024-02-03T17:18:45.517Z 17 340-359 191 191 191 328 328 328 T T F <---
209132024-02-03T17:18:45.517Z 18 360-379 191 191 191 328 328 328 T T F <---
209142024-02-03T17:18:45.517Z 19 380-399 191 191 191 328 328 328 T T F <---
209152024-02-03T17:18:45.517Z 20 400-419 190 190 190 327 327 327 T T F <---
209162024-02-03T17:18:45.517Z 21 420-439 190 190 190 327 327 327 T T F <---
209172024-02-03T17:18:45.517Z 22 440-459 190 190 190 327 327 327 T T F <---
209182024-02-03T17:18:45.517Z 23 460-479 188 188 188 325 325 325 T T F <---
209192024-02-03T17:18:45.517Z 24 480-499 191 191 191 328 328 328 T T F <---
209202024-02-03T17:18:45.517Z 25 500-519 191 191 191 328 328 328 F F F
209212024-02-03T17:18:45.517Z 26 520-539 188 188 188 325 325 325 F F F
209222024-02-03T17:18:45.517Z 27 540-559 190 190 190 327 327 327 T T F <---
209232024-02-03T17:18:45.517Z 28 560-579 186 186 186 319 319 319 T T F <---
209242024-02-03T17:18:45.517Z 29 580-599 185 185 185 318 318 318 T T F <---
209252024-02-03T17:18:45.517ZMax gen: 191, Max flush: 328
209262024-02-03T17:18:45.517ZError: Difference in extent metadata found!
209272024-02-03T17:18:45.521ZA Difference in extent metadata is expected here
209282024-02-03T17:18:45.521ZOn loop 96
209292024-02-03T17:18:45.521Z
209302024-02-03T17:18:45.524ZVerifying data now
209312024-02-03T17:18:50.823ZNo change after verify
209322024-02-03T17:18:50.823ZLoop: 96 Downstairs dump after verify (and repair):
209332024-02-03T17:18:50.848ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
209342024-02-03T17:18:50.848Z 0 000-019 188 188 188 325 325 325 F F F
209352024-02-03T17:18:50.852Z 1 020-039 188 188 188 325 325 325 F F F
209362024-02-03T17:18:50.852Z 2 040-059 187 187 187 323 323 323 F F F
209372024-02-03T17:18:50.852Z 3 060-079 192 192 192 329 329 329 F F F
209382024-02-03T17:18:50.852Z 4 080-099 190 190 190 327 327 327 F F F
209392024-02-03T17:18:50.852Z 5 100-119 192 192 192 329 329 329 F F F
209402024-02-03T17:18:50.852Z 6 120-139 190 190 190 327 327 327 F F F
209412024-02-03T17:18:50.852Z 7 140-159 190 190 190 327 327 327 F F F
209422024-02-03T17:18:50.852Z 8 160-179 191 191 191 328 328 328 F F F
209432024-02-03T17:18:50.852Z 9 180-199 188 188 188 325 325 325 F F F
209442024-02-03T17:18:50.852Z 10 200-219 187 187 187 323 323 323 F F F
209452024-02-03T17:18:50.852Z 11 220-239 188 188 188 325 325 325 F F F
209462024-02-03T17:18:50.852Z 12 240-259 190 190 190 327 327 327 F F F
209472024-02-03T17:18:50.852Z 13 260-279 190 190 190 327 327 327 F F F
209482024-02-03T17:18:50.852Z 14 280-299 192 192 192 329 329 329 F F F
209492024-02-03T17:18:50.852Z 15 300-319 190 190 190 327 327 327 F F F
209502024-02-03T17:18:50.852Z 16 320-339 192 192 192 329 329 329 F F F
209512024-02-03T17:18:50.852Z 17 340-359 192 192 192 329 329 329 F F F
209522024-02-03T17:18:50.853Z 18 360-379 192 192 192 329 329 329 F F F
209532024-02-03T17:18:50.853Z 19 380-399 192 192 192 329 329 329 F F F
209542024-02-03T17:18:50.853Z 20 400-419 192 192 192 329 329 329 F F F
209552024-02-03T17:18:50.853Z 21 420-439 192 192 192 329 329 329 F F F
209562024-02-03T17:18:50.853Z 22 440-459 192 192 192 329 329 329 F F F
209572024-02-03T17:18:50.853Z 23 460-479 192 192 192 329 329 329 F F F
209582024-02-03T17:18:50.853Z 24 480-499 192 192 192 329 329 329 F F F
209592024-02-03T17:18:50.853Z 25 500-519 191 191 191 328 328 328 F F F
209602024-02-03T17:18:50.853Z 26 520-539 188 188 188 325 325 325 F F F
209612024-02-03T17:18:50.853Z 27 540-559 192 192 192 329 329 329 F F F
209622024-02-03T17:18:50.853Z 28 560-579 192 192 192 329 329 329 F F F
209632024-02-03T17:18:50.853Z 29 580-599 192 192 192 329 329 329 F F F
209642024-02-03T17:18:50.853ZMax gen: 192, Max flush: 329
209652024-02-03T17:18:50.853Z
209662024-02-03T17:18:50.853ZBegin loop 97
209672024-02-03T17:18:50.853ZDownstairs to restart: 2
209682024-02-03T17:18:50.882Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.465089328Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911}
209692024-02-03T17:18:50.882Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.465562385Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911}
209702024-02-03T17:18:50.887Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
209712024-02-03T17:18:50.887Z,"time":"2024-02-03T17:18:50.46572247Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911}
209722024-02-03T17:18:50.887Z{"msg":"Crucible 21c3f6bf-f30d-41cd-84db-741396d0af5e has session id: c33a5a09-15cb-4838-9490-8ddabec7b6a5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.465779642Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
209732024-02-03T17:18:50.887Z{"msg":"Upstairs opts: Upstairs UUID: 21c3f6bf-f30d-41cd-84db-741396d0af5e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.465878582Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
209742024-02-03T17:18:50.887Z{"msg":"Crucible stats registered with UUID: 21c3f6bf-f30d-41cd-84db-741396d0af5e","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.465922327Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
209752024-02-03T17:18:50.887Z{"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.466002222Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911}
209762024-02-03T17:18:50.887Z{"msg":"21c3f6bf-f30d-41cd-84db-741396d0af5e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.466042525Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
209772024-02-03T17:18:50.887Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.466118129Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
209782024-02-03T17:18:50.887Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.466207146Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
209792024-02-03T17:18:50.888Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.466429664Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
209802024-02-03T17:18:50.888Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.466530032Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
209812024-02-03T17:18:50.888Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.466609718Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
209822024-02-03T17:18:50.888Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.466655757Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
209832024-02-03T17:18:50.888Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.466734365Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"":"io task","client":"0","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
209842024-02-03T17:18:50.888Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.466778697Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"":"io task","client":"1","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
209852024-02-03T17:18:50.888Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.466819305Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"":"io task","client":"2","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
209862024-02-03T17:18:50.888Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.46689894Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"":"io task","client":"2","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
209872024-02-03T17:18:50.888Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.466942703Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"":"io task","client":"0","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
209882024-02-03T17:18:50.888Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.467021894Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"":"io task","client":"1","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
209892024-02-03T17:18:50.888Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.467795067Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
209902024-02-03T17:18:50.888Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.467918505Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
209912024-02-03T17:18:50.889Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.467973704Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
209922024-02-03T17:18:50.889Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.468021032Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
209932024-02-03T17:18:50.889Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.469248411Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
209942024-02-03T17:18:50.889Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.469328171Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
209952024-02-03T17:18:50.889Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.469591227Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
209962024-02-03T17:18:50.889Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.469686629Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
209972024-02-03T17:18:50.894Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.470178509Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
209982024-02-03T17:18:50.894Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.470259943Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
209992024-02-03T17:18:50.895Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.470305886Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
210002024-02-03T17:18:50.895Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.470534081Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
210012024-02-03T17:18:50.895Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.47060475Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
210022024-02-03T17:18:50.895Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.47063133Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
210032024-02-03T17:18:50.895Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.47065367Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
210042024-02-03T17:18:50.895Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.471441727Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
210052024-02-03T17:18:50.895Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.471530394Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
210062024-02-03T17:18:50.895Z{"msg":"[0]R flush_numbers[0..12]: [325, 325, 323, 329, 327, 329, 327, 327, 328, 325, 323, 325]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.471567784Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
210072024-02-03T17:18:50.895Z{"msg":"[0]R generation[0..12]: [188, 188, 187, 192, 190, 192, 190, 190, 191, 188, 187, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.471593073Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
210082024-02-03T17:18:50.895Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.471647498Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
210092024-02-03T17:18:50.895Z{"msg":"[1]R flush_numbers[0..12]: [325, 325, 323, 329, 327, 329, 327, 327, 328, 325, 323, 325]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.471673463Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
210102024-02-03T17:18:50.895Z{"msg":"[1]R generation[0..12]: [188, 188, 187, 192, 190, 192, 190, 190, 191, 188, 187, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.471696831Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
210112024-02-03T17:18:50.896Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.471747526Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
210122024-02-03T17:18:50.896Z{"msg":"[2]R flush_numbers[0..12]: [325, 325, 323, 329, 327, 329, 327, 327, 328, 325, 323, 325]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.471773608Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
210132024-02-03T17:18:50.896Z{"msg":"[2]R generation[0..12]: [188, 188, 187, 192, 190, 192, 190, 190, 191, 188, 187, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.471796707Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
210142024-02-03T17:18:50.896Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
210152024-02-03T17:18:50.896Z,"time":"2024-02-03T17:18:50.471872849Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
210162024-02-03T17:18:50.896Z{"msg":"Max found gen is 193","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.471897308Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
210172024-02-03T17:18:50.896Z{"msg":"Generation requested: 194 >= found:193","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.471947545Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
210182024-02-03T17:18:50.896Z{"msg":"Next flush: 330","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.471970698Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
210192024-02-03T17:18:50.896Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.471992172Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
210202024-02-03T17:18:50.896Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.472015495Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
210212024-02-03T17:18:50.896Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.472071792Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
210222024-02-03T17:18:50.896Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.472114434Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
210232024-02-03T17:18:50.896Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.472154106Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
210242024-02-03T17:18:50.897Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.472193392Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
210252024-02-03T17:18:50.897Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.472229815Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
210262024-02-03T17:18:50.897Z{"msg":"21c3f6bf-f30d-41cd-84db-741396d0af5e is now active with session: c33a5a09-15cb-4838-9490-8ddabec7b6a5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.472267567Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
210272024-02-03T17:18:50.897Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.472328149Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911,"session_id":"c33a5a09-15cb-4838-9490-8ddabec7b6a5"}
210282024-02-03T17:18:50.897Z{"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:50.472368251Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911}
210292024-02-03T17:18:50.897ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
210302024-02-03T17:18:50.897ZLoading write count information from file "/tmp/test_repair_verify.data"
210312024-02-03T17:18:50.897ZRun Repair workload
210322024-02-03T17:18:50.897Z01/30 Write block 333 len 12288 data: 14 15 15
210332024-02-03T17:18:50.897Z02/30 Read block 375 len 12288
210342024-02-03T17:18:50.901Z03/30 Read block 311 len 36864
210352024-02-03T17:18:50.901Z04/30 Read block 15 len 24576
210362024-02-03T17:18:50.901Z05/30 Read block 8 len 12288
210372024-02-03T17:18:50.901Z06/30 Read block 17 len 40960
210382024-02-03T17:18:50.905Z07/30 Write block 266 len 28672 data: 21 20 24 17 17 16 17
210392024-02-03T17:18:50.905Z08/30 Write block 357 len 8192 data: 15 13
210402024-02-03T17:18:50.905Z09/30 Read block 433 len 4096
210412024-02-03T17:18:50.908Z10/30 Read block 504 len 8192
210422024-02-03T17:18:50.908Z11/30 Write block 390 len 12288 data: 17 18 23
210432024-02-03T17:18:50.908Z12/30 Write block 345 len 32768 data: 10 10 10 12 15 14 13 15
210442024-02-03T17:18:50.912Z13/30 Write block 521 len 40960 data: 11 14 16 15 18 17 17 16 16 18
210452024-02-03T17:18:50.912Z14/30 Write block 296 len 16384 data: 14 14 14 13
210462024-02-03T17:18:50.934Z15/30 Write block 222 len 4096 data: 7
210472024-02-03T17:18:50.934Z16/30 Write block 181 len 40960 data: 18 16 17 15 19 16 21 22 24 23
210482024-02-03T17:18:50.934Z17/30 Read block 381 len 28672
210492024-02-03T17:18:50.937Z18/30 Write block 235 len 36864 data: 19 16 15 14 15 12 12 11 11
210502024-02-03T17:18:50.937Z19/30 Read block 137 len 24576
210512024-02-03T17:18:50.938Z20/30 Read block 25 len 16384
210522024-02-03T17:18:50.938Z21/30 Write block 541 len 24576 data: 12 15 14 12 12 14
210532024-02-03T17:18:50.938Z22/30 Flush
210542024-02-03T17:18:50.938Z23/30 Write block 428 len 20480 data: 15 18 18 15 16
210552024-02-03T17:18:50.938Z24/30 Write block 361 len 24576 data: 16 13 13 14 16 16
210562024-02-03T17:18:50.938Z25/30 Read block 58 len 16384
210572024-02-03T17:18:50.938Z26/30 Read block 414 len 28672
210582024-02-03T17:18:50.938Z27/30 Read block 452 len 20480
210592024-02-03T17:18:50.938Z28/30 Read block 216 len 8192
210602024-02-03T17:18:50.942Z29/30 Write block 167 len 32768 data: 23 18 20 21 19 16 16 14
210612024-02-03T17:18:50.942Z30/30 Write block 359 len 36864 data: 14 13 17 14 14 15 17 17 17
210622024-02-03T17:18:50.942Z----------------------------------------------------------------
210632024-02-03T17:18:50.942Z Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:30
210642024-02-03T17:18:50.942ZStates: Active Active Active
210652024-02-03T17:18:50.942ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
210662024-02-03T17:18:50.942Z 1 Acked 1000 Write 3 Done Done Sent false
210672024-02-03T17:18:50.942Z 2 Acked 1001 Read 3 Done Done Sent false
210682024-02-03T17:18:50.943Z 3 Acked 1002 Read 9 Done Done Sent false
210692024-02-03T17:18:50.943Z 4 Acked 1003 Read 6 Done Done Sent false
210702024-02-03T17:18:50.943Z 5 Acked 1004 Read 3 Done Done Sent false
210712024-02-03T17:18:50.943Z 6 Acked 1005 Read 10 Done Done Sent false
210722024-02-03T17:18:50.943Z 7 Acked 1006 Write 7 Done Done Sent false
210732024-02-03T17:18:50.943Z 8 Acked 1007 Write 2 Done Done Sent false
210742024-02-03T17:18:50.943Z 9 Acked 1008 Read 1 Done Done Sent false
210752024-02-03T17:18:50.943Z 10 Acked 1009 Read 2 Done Done Sent false
210762024-02-03T17:18:50.943Z 11 Acked 1010 Write 3 Done Done Sent false
210772024-02-03T17:18:50.943Z 12 Acked 1011 Write 8 Done Done Sent false
210782024-02-03T17:18:50.943Z 13 Acked 1012 Write 10 Done Done Sent false
210792024-02-03T17:18:50.943Z 14 Acked 1013 Write 4 Done Done Sent false
210802024-02-03T17:18:50.943Z 15 Acked 1014 Write 1 Done Done Sent false
210812024-02-03T17:18:50.944Z 16 Acked 1015 Write 10 Done Done Sent false
210822024-02-03T17:18:50.944Z 17 Acked 1016 Read 7 Done Done Sent false
210832024-02-03T17:18:50.944Z 18 Acked 1017 Write 9 Done Done Sent false
210842024-02-03T17:18:50.944Z 19 Acked 1018 Read 6 Done Done Sent false
210852024-02-03T17:18:50.944Z 20 Acked 1019 Read 4 Done Done Sent false
210862024-02-03T17:18:50.944Z 21 Acked 1020 Write 6 Done Done Sent false
210872024-02-03T17:18:50.944Z 22 Acked 1021 Flush 0 Done Done Sent false
210882024-02-03T17:18:50.944Z 23 Acked 1022 Write 5 Done Done Sent false
210892024-02-03T17:18:50.944Z 24 Acked 1023 Write 6 Done Done Sent false
210902024-02-03T17:18:50.944Z 25 Acked 1024 Read 4 Done Done Sent false
210912024-02-03T17:18:50.944Z 26 Acked 1025 Read 7 Done Done Sent false
210922024-02-03T17:18:50.944Z 27 Acked 1026 Read 5 Done Done Sent false
210932024-02-03T17:18:50.944Z 28 Acked 1027 Read 2 Done Done Sent false
210942024-02-03T17:18:50.945Z 29 Acked 1028 Write 8 Done Done Sent false
210952024-02-03T17:18:50.945Z 30 Acked 1029 Write 9 Sent Sent Sent false
210962024-02-03T17:18:50.945Z STATES DS:0 DS:1 DS:2 TOTAL
210972024-02-03T17:18:50.945Z New 0 0 0 0
210982024-02-03T17:18:50.945Z Sent 1 1 30 32
210992024-02-03T17:18:50.945Z Done 29 29 0 58
211002024-02-03T17:18:50.945Z Skipped 0 0 0 0
211012024-02-03T17:18:50.945Z Error 0 0 0 0
211022024-02-03T17:18:50.945ZLast Flush: 1021 1021 0
211032024-02-03T17:18:50.945ZDownstairs last five completed:
211042024-02-03T17:18:50.945ZUpstairs last five completed: 30 29 28 27 26
211052024-02-03T17:18:50.945Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:18:50.524549389Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1911}
211062024-02-03T17:18:50.945ZWrote out file "/tmp/test_repair_verify.data"
211072024-02-03T17:18:50.949Z
211082024-02-03T17:18:50.955ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
211092024-02-03T17:18:50.980ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
211102024-02-03T17:18:50.980Z 0 000-019 188 188 188 325 325 325 F F F
211112024-02-03T17:18:50.984Z 1 020-039 188 188 188 325 325 325 F F F
211122024-02-03T17:18:50.984Z 2 040-059 187 187 187 323 323 323 F F F
211132024-02-03T17:18:50.984Z 3 060-079 192 192 192 329 329 329 F F F
211142024-02-03T17:18:50.984Z 4 080-099 190 190 190 327 327 327 F F F
211152024-02-03T17:18:50.984Z 5 100-119 192 192 192 329 329 329 F F F
211162024-02-03T17:18:50.984Z 6 120-139 190 190 190 327 327 327 F F F
211172024-02-03T17:18:50.984Z 7 140-159 190 190 190 327 327 327 F F F
211182024-02-03T17:18:50.984Z 8 160-179 191 191 191 328 328 328 T T F <---
211192024-02-03T17:18:50.984Z 9 180-199 194 194 188 330 330 325 F F F <---
211202024-02-03T17:18:50.984Z 10 200-219 187 187 187 323 323 323 F F F
211212024-02-03T17:18:50.984Z 11 220-239 194 194 188 330 330 325 F F F <---
211222024-02-03T17:18:50.984Z 12 240-259 194 194 190 330 330 327 F F F <---
211232024-02-03T17:18:50.985Z 13 260-279 194 194 190 330 330 327 F F F <---
211242024-02-03T17:18:50.985Z 14 280-299 194 194 192 330 330 329 F F F <---
211252024-02-03T17:18:50.985Z 15 300-319 190 190 190 327 327 327 F F F
211262024-02-03T17:18:50.985Z 16 320-339 194 194 192 330 330 329 F F F <---
211272024-02-03T17:18:50.985Z 17 340-359 194 194 192 330 330 329 T T F <---
211282024-02-03T17:18:50.985Z 18 360-379 192 192 192 329 329 329 T T F <---
211292024-02-03T17:18:50.985Z 19 380-399 194 194 192 330 330 329 F F F <---
211302024-02-03T17:18:50.985Z 20 400-419 192 192 192 329 329 329 F F F
211312024-02-03T17:18:50.985Z 21 420-439 192 192 192 329 329 329 T T F <---
211322024-02-03T17:18:50.985Z 22 440-459 192 192 192 329 329 329 F F F
211332024-02-03T17:18:50.985Z 23 460-479 192 192 192 329 329 329 F F F
211342024-02-03T17:18:50.985Z 24 480-499 192 192 192 329 329 329 F F F
211352024-02-03T17:18:50.985Z 25 500-519 191 191 191 328 328 328 F F F
211362024-02-03T17:18:50.986Z 26 520-539 194 194 188 330 330 325 F F F <---
211372024-02-03T17:18:50.986Z 27 540-559 194 194 192 330 330 329 F F F <---
211382024-02-03T17:18:50.986Z 28 560-579 192 192 192 329 329 329 F F F
211392024-02-03T17:18:50.986Z 29 580-599 192 192 192 329 329 329 F F F
211402024-02-03T17:18:50.986ZMax gen: 194, Max flush: 330
211412024-02-03T17:18:50.986ZError: Difference in extent metadata found!
211422024-02-03T17:18:50.989ZA Difference in extent metadata is expected here
211432024-02-03T17:18:50.989ZOn loop 97
211442024-02-03T17:18:50.990Z
211452024-02-03T17:18:50.990ZVerifying data now
211462024-02-03T17:18:56.103ZNo change after verify
211472024-02-03T17:18:56.103ZLoop: 97 Downstairs dump after verify (and repair):
211482024-02-03T17:18:56.126ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
211492024-02-03T17:18:56.126Z 0 000-019 188 188 188 325 325 325 F F F
211502024-02-03T17:18:56.126Z 1 020-039 188 188 188 325 325 325 F F F
211512024-02-03T17:18:56.126Z 2 040-059 187 187 187 323 323 323 F F F
211522024-02-03T17:18:56.126Z 3 060-079 192 192 192 329 329 329 F F F
211532024-02-03T17:18:56.130Z 4 080-099 190 190 190 327 327 327 F F F
211542024-02-03T17:18:56.130Z 5 100-119 192 192 192 329 329 329 F F F
211552024-02-03T17:18:56.130Z 6 120-139 190 190 190 327 327 327 F F F
211562024-02-03T17:18:56.130Z 7 140-159 190 190 190 327 327 327 F F F
211572024-02-03T17:18:56.130Z 8 160-179 195 195 195 331 331 331 F F F
211582024-02-03T17:18:56.130Z 9 180-199 194 194 194 330 330 330 F F F
211592024-02-03T17:18:56.130Z 10 200-219 187 187 187 323 323 323 F F F
211602024-02-03T17:18:56.130Z 11 220-239 194 194 194 330 330 330 F F F
211612024-02-03T17:18:56.130Z 12 240-259 194 194 194 330 330 330 F F F
211622024-02-03T17:18:56.130Z 13 260-279 194 194 194 330 330 330 F F F
211632024-02-03T17:18:56.131Z 14 280-299 194 194 194 330 330 330 F F F
211642024-02-03T17:18:56.131Z 15 300-319 190 190 190 327 327 327 F F F
211652024-02-03T17:18:56.131Z 16 320-339 194 194 194 330 330 330 F F F
211662024-02-03T17:18:56.131Z 17 340-359 195 195 195 331 331 331 F F F
211672024-02-03T17:18:56.131Z 18 360-379 195 195 195 331 331 331 F F F
211682024-02-03T17:18:56.131Z 19 380-399 194 194 194 330 330 330 F F F
211692024-02-03T17:18:56.131Z 20 400-419 192 192 192 329 329 329 F F F
211702024-02-03T17:18:56.131Z 21 420-439 195 195 195 331 331 331 F F F
211712024-02-03T17:18:56.131Z 22 440-459 192 192 192 329 329 329 F F F
211722024-02-03T17:18:56.131Z 23 460-479 192 192 192 329 329 329 F F F
211732024-02-03T17:18:56.131Z 24 480-499 192 192 192 329 329 329 F F F
211742024-02-03T17:18:56.131Z 25 500-519 191 191 191 328 328 328 F F F
211752024-02-03T17:18:56.131Z 26 520-539 194 194 194 330 330 330 F F F
211762024-02-03T17:18:56.131Z 27 540-559 194 194 194 330 330 330 F F F
211772024-02-03T17:18:56.131Z 28 560-579 192 192 192 329 329 329 F F F
211782024-02-03T17:18:56.131Z 29 580-599 192 192 192 329 329 329 F F F
211792024-02-03T17:18:56.131ZMax gen: 195, Max flush: 331
211802024-02-03T17:18:56.135Z
211812024-02-03T17:18:56.135ZBegin loop 98
211822024-02-03T17:18:56.135ZDownstairs to restart: 2
211832024-02-03T17:18:56.162Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.745502035Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919}
211842024-02-03T17:18:56.162Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.746042033Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919}
211852024-02-03T17:18:56.165Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
211862024-02-03T17:18:56.165Z,"time":"2024-02-03T17:18:55.746125112Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919}
211872024-02-03T17:18:56.166Z{"msg":"Crucible 9cb2a1e9-690a-43da-9c46-2445b227a05d has session id: 31639baf-bcf9-4d65-9a4d-46dcc81afd5b","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.746400131Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
211882024-02-03T17:18:56.166Z{"msg":"Upstairs opts: Upstairs UUID: 9cb2a1e9-690a-43da-9c46-2445b227a05d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.746457318Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
211892024-02-03T17:18:56.166Z{"msg":"Crucible stats registered with UUID: 9cb2a1e9-690a-43da-9c46-2445b227a05d","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.746522332Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
211902024-02-03T17:18:56.166Z{"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.746547411Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919}
211912024-02-03T17:18:56.166Z{"msg":"9cb2a1e9-690a-43da-9c46-2445b227a05d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.746738144Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
211922024-02-03T17:18:56.166Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.746813658Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
211932024-02-03T17:18:56.166Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.746842347Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
211942024-02-03T17:18:56.166Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.746865192Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
211952024-02-03T17:18:56.166Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.74688801Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
211962024-02-03T17:18:56.166Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.746910807Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
211972024-02-03T17:18:56.166Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.746941308Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
211982024-02-03T17:18:56.166Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.746981607Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"":"io task","client":"2","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
211992024-02-03T17:18:56.166Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.747170544Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"":"io task","client":"1","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212002024-02-03T17:18:56.166Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.747407844Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"":"io task","client":"0","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212012024-02-03T17:18:56.166Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.747540803Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"":"io task","client":"1","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212022024-02-03T17:18:56.166Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.747591249Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"":"io task","client":"0","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212032024-02-03T17:18:56.166Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.747683536Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"":"io task","client":"2","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212042024-02-03T17:18:56.166Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.74858568Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212052024-02-03T17:18:56.166Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.748728027Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212062024-02-03T17:18:56.166Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.748845042Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212072024-02-03T17:18:56.166Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.749202075Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212082024-02-03T17:18:56.171Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.749512359Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212092024-02-03T17:18:56.171Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.749866105Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212102024-02-03T17:18:56.171Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.750054216Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212112024-02-03T17:18:56.171Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.75037947Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212122024-02-03T17:18:56.172Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.750842355Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212132024-02-03T17:18:56.172Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.751112121Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212142024-02-03T17:18:56.172Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.751186789Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212152024-02-03T17:18:56.172Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.751214504Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212162024-02-03T17:18:56.172Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.751235212Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212172024-02-03T17:18:56.172Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.751303292Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212182024-02-03T17:18:56.172Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.751430041Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212192024-02-03T17:18:56.172Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.751624458Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212202024-02-03T17:18:56.172Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.751703471Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212212024-02-03T17:18:56.172Z{"msg":"[0]R flush_numbers[0..12]: [325, 325, 323, 329, 327, 329, 327, 327, 331, 330, 323, 330]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.751735396Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212222024-02-03T17:18:56.172Z{"msg":"[0]R generation[0..12]: [188, 188, 187, 192, 190, 192, 190, 190, 195, 194, 187, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.751803405Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212232024-02-03T17:18:56.172Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.75183845Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212242024-02-03T17:18:56.172Z{"msg":"[1]R flush_numbers[0..12]: [325, 325, 323, 329, 327, 329, 327, 327, 331, 330, 323, 330]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.751905129Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"Wait for a query_work_queue command to finish before sending IO
212252024-02-03T17:18:56.172Z":"downstairs","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212262024-02-03T17:18:56.172Z{"msg":"[1]R generation[0..12]: [188, 188, 187, 192, 190, 192, 190, 190, 195, 194, 187, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.751942661Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212272024-02-03T17:18:56.172Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.752077241Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212282024-02-03T17:18:56.172Z{"msg":"[2]R flush_numbers[0..12]: [325, 325, 323, 329, 327, 329, 327, 327, 331, 330, 323, 330]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.752106159Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212292024-02-03T17:18:56.172Z{"msg":"[2]R generation[0..12]: [188, 188, 187, 192, 190, 192, 190, 190, 195, 194, 187, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.752133675Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212302024-02-03T17:18:56.172Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.752200166Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212312024-02-03T17:18:56.172Z{"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.752227498Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212322024-02-03T17:18:56.172Z{"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.752256818Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212332024-02-03T17:18:56.172Z{"msg":"Next flush: 332","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.752322567Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212342024-02-03T17:18:56.173Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.752349408Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212352024-02-03T17:18:56.173Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.752376775Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212362024-02-03T17:18:56.173Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.752436566Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212372024-02-03T17:18:56.173Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.752463666Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212382024-02-03T17:18:56.173Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.752494261Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212392024-02-03T17:18:56.173Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.752555913Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212402024-02-03T17:18:56.173Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.752581148Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212412024-02-03T17:18:56.173Z{"msg":"9cb2a1e9-690a-43da-9c46-2445b227a05d is now active with session: 31639baf-bcf9-4d65-9a4d-46dcc81afd5b","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.752604209Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212422024-02-03T17:18:56.173ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
212432024-02-03T17:18:56.173Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.75267129Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919,"session_id":"31639baf-bcf9-4d65-9a4d-46dcc81afd5b"}
212442024-02-03T17:18:56.173Z{"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:55.752698025Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919}
212452024-02-03T17:18:56.173ZLoading write count information from file "/tmp/test_repair_verify.data"
212462024-02-03T17:18:56.173ZRun Repair workload
212472024-02-03T17:18:56.173Z01/30 Flush
212482024-02-03T17:18:56.176Z02/30 Write block 98 len 40960 data: 13 15 16 15 11 11 11 12 15 15
212492024-02-03T17:18:56.176Z03/30 Write block 67 len 40960 data: 9 10 8 9 8 7 8 10 13 15
212502024-02-03T17:18:56.176Z04/30 Read block 68 len 4096
212512024-02-03T17:18:56.179Z05/30 Flush
212522024-02-03T17:18:56.182Z06/30 Write block 571 len 16384 data: 14 14 14 16
212532024-02-03T17:18:56.182Z07/30 Read block 337 len 28672
212542024-02-03T17:18:56.185Z08/30 Write block 359 len 24576 data: 15 14 18 15 15 16
212552024-02-03T17:18:56.185Z09/30 Read block 423 len 12288
212562024-02-03T17:18:56.190Z10/30 Write block 442 len 24576 data: 24 21 16 18 19 19
212572024-02-03T17:18:56.190Z11/30 Write block 352 len 4096 data: 16
212582024-02-03T17:18:56.190Z12/30 Read block 408 len 36864
212592024-02-03T17:18:56.196Z13/30 Read block 119 len 36864
212602024-02-03T17:18:56.196Z14/30 Flush
212612024-02-03T17:18:56.202Z15/30 Write block 69 len 20480 data: 9 10 9 8 9
212622024-02-03T17:18:56.202Z16/30 Write block 588 len 12288 data: 18 15 16
212632024-02-03T17:18:56.202Z17/30 Read block 405 len 32768
212642024-02-03T17:18:56.202Z18/30 Write block 316 len 4096 data: 20
212652024-02-03T17:18:56.202Z19/30 Flush
212662024-02-03T17:18:56.206Z20/30 Write block 196 len 16384 data: 18 17 17 11
212672024-02-03T17:18:56.206Z21/30 Flush
212682024-02-03T17:18:56.209Z22/30 Write block 283 len 20480 data: 20 16 17 17 16
212692024-02-03T17:18:56.209Z23/30 Read block 486 len 36864
212702024-02-03T17:18:56.214Z24/30 Flush
212712024-02-03T17:18:56.214Z25/30 Write block 564 len 16384 data: 16 15 15 15
212722024-02-03T17:18:56.214Z26/30 Read block 175 len 8192
212732024-02-03T17:18:56.220Z27/30 Write block 128 len 32768 data: 15 16 15 15 14 13 13 11
212742024-02-03T17:18:56.221Z28/30 Read block 100 len 20480
212752024-02-03T17:18:56.221Z29/30 Read block 366 len 36864
212762024-02-03T17:18:56.225Z30/30 Read block 370 len 28672
212772024-02-03T17:18:56.225Z----------------------------------------------------------------
212782024-02-03T17:18:56.225Z Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:30
212792024-02-03T17:18:56.225ZStates: Active Active Active
212802024-02-03T17:18:56.225ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
212812024-02-03T17:18:56.225Z 1 Acked 1000 Flush 0 Done Done Sent false
212822024-02-03T17:18:56.225Z 2 Acked 1001 Write 10 Done Done Sent false
212832024-02-03T17:18:56.225Z 3 Acked 1002 Write 10 Done Done Sent false
212842024-02-03T17:18:56.225Z 4 Acked 1003 Read 1 Done Done Sent false
212852024-02-03T17:18:56.225Z 5 Acked 1004 Flush 0 Done Done Sent false
212862024-02-03T17:18:56.225Z 6 Acked 1005 Write 4 Done Done Sent false
212872024-02-03T17:18:56.225Z 7 Acked 1006 Read 7 Done Done Sent false
212882024-02-03T17:18:56.226Z 8 Acked 1007 Write 6 Done Done Sent false
212892024-02-03T17:18:56.226Z 9 Acked 1008 Read 3 Done Done Sent false
212902024-02-03T17:18:56.226Z 10 Acked 1009 Write 6 Done Done Sent false
212912024-02-03T17:18:56.226Z 11 Acked 1010 Write 1 Done Done Sent false
212922024-02-03T17:18:56.226Z 12 Acked 1011 Read 9 Done Done Sent false
212932024-02-03T17:18:56.226Z 13 Acked 1012 Read 9 Done Done Sent false
212942024-02-03T17:18:56.226Z 14 Acked 1013 Flush 0 Done Done Sent false
212952024-02-03T17:18:56.226Z 15 Acked 1014 Write 5 Done Done Sent false
212962024-02-03T17:18:56.226Z 16 Acked 1015 Write 3 Done Done Sent false
212972024-02-03T17:18:56.226Z 17 Acked 1016 Read 8 Done Done Sent false
212982024-02-03T17:18:56.226Z 18 Acked 1017 Write 1 Done Done Sent false
212992024-02-03T17:18:56.226Z 19 Acked 1018 Flush 0 Done Done Sent false
213002024-02-03T17:18:56.226Z 20 Acked 1019 Write 4 Done Done Sent false
213012024-02-03T17:18:56.226Z 21 Acked 1020 Flush 0 Done Done Sent false
213022024-02-03T17:18:56.226Z 22 Acked 1021 Write 5 Done Done Sent false
213032024-02-03T17:18:56.226Z 23 Acked 1022 Read 9 Done Done Sent false
213042024-02-03T17:18:56.226Z 24 Acked 1023 Flush 0 Done Done Sent false
213052024-02-03T17:18:56.226Z 25 Acked 1024 Write 4 Done Done Sent false
213062024-02-03T17:18:56.226Z 26 Acked 1025 Read 2 Done Done Sent false
213072024-02-03T17:18:56.226Z 27 Acked 1026 Write 8 Done Done Sent false
213082024-02-03T17:18:56.226Z 28 Acked 1027 Read 5 Done Done Sent false
213092024-02-03T17:18:56.226Z 29 Acked 1028 Read 9 Done Done Sent false
213102024-02-03T17:18:56.226Z 30 Acked 1029 Read 7 Done Sent Sent false
213112024-02-03T17:18:56.226Z STATES DS:0 DS:1 DS:2 TOTAL
213122024-02-03T17:18:56.226Z New 0 0 0 0
213132024-02-03T17:18:56.227Z Sent 0 1 30 31
213142024-02-03T17:18:56.227Z Done 30 29 0 59
213152024-02-03T17:18:56.227Z Skipped 0 0 0 0
213162024-02-03T17:18:56.227Z Error 0 0 0 0
213172024-02-03T17:18:56.227ZLast Flush: 1023 1023 0
213182024-02-03T17:18:56.227ZDownstairs last five completed:
213192024-02-03T17:18:56.227ZUpstairs last five completed: 30 29 28 27 26
213202024-02-03T17:18:56.227Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:18:55.80555893Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1919}
213212024-02-03T17:18:56.227ZWrote out file "/tmp/test_repair_verify.data"
213222024-02-03T17:18:56.232Z
213232024-02-03T17:18:56.235ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
213242024-02-03T17:18:56.259ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
213252024-02-03T17:18:56.259Z 0 000-019 188 188 188 325 325 325 F F F
213262024-02-03T17:18:56.259Z 1 020-039 188 188 188 325 325 325 F F F
213272024-02-03T17:18:56.263Z 2 040-059 187 187 187 323 323 323 F F F
213282024-02-03T17:18:56.263Z 3 060-079 196 196 192 335 335 329 F F F <---
213292024-02-03T17:18:56.264Z 4 080-099 196 196 190 333 333 327 F F F <---
213302024-02-03T17:18:56.264Z 5 100-119 196 196 192 333 333 329 F F F <---
213312024-02-03T17:18:56.264Z 6 120-139 190 190 190 327 327 327 T T F <---
213322024-02-03T17:18:56.264Z 7 140-159 190 190 190 327 327 327 F F F
213332024-02-03T17:18:56.264Z 8 160-179 195 195 195 331 331 331 F F F
213342024-02-03T17:18:56.264Z 9 180-199 196 196 194 336 336 330 F F F <---
213352024-02-03T17:18:56.264Z 10 200-219 187 187 187 323 323 323 F F F
213362024-02-03T17:18:56.264Z 11 220-239 194 194 194 330 330 330 F F F
213372024-02-03T17:18:56.264Z 12 240-259 194 194 194 330 330 330 F F F
213382024-02-03T17:18:56.264Z 13 260-279 194 194 194 330 330 330 F F F
213392024-02-03T17:18:56.264Z 14 280-299 196 196 194 337 337 330 F F F <---
213402024-02-03T17:18:56.264Z 15 300-319 196 196 190 335 335 327 F F F <---
213412024-02-03T17:18:56.264Z 16 320-339 194 194 194 330 330 330 F F F
213422024-02-03T17:18:56.264Z 17 340-359 196 196 195 334 334 331 F F F <---
213432024-02-03T17:18:56.264Z 18 360-379 196 196 195 334 334 331 F F F <---
213442024-02-03T17:18:56.264Z 19 380-399 194 194 194 330 330 330 F F F
213452024-02-03T17:18:56.264Z 20 400-419 192 192 192 329 329 329 F F F
213462024-02-03T17:18:56.264Z 21 420-439 195 195 195 331 331 331 F F F
213472024-02-03T17:18:56.264Z 22 440-459 196 196 192 334 334 329 F F F <---
213482024-02-03T17:18:56.264Z 23 460-479 192 192 192 329 329 329 F F F
213492024-02-03T17:18:56.264Z 24 480-499 192 192 192 329 329 329 F F F
213502024-02-03T17:18:56.264Z 25 500-519 191 191 191 328 328 328 F F F
213512024-02-03T17:18:56.264Z 26 520-539 194 194 194 330 330 330 F F F
213522024-02-03T17:18:56.264Z 27 540-559 194 194 194 330 330 330 F F F
213532024-02-03T17:18:56.265Z 28 560-579 196 196 192 334 334 329 T T F <---
213542024-02-03T17:18:56.265Z 29 580-599 196 196 192 335 335 329 F F F <---
213552024-02-03T17:18:56.265ZMax gen: 196, Max flush: 337
213562024-02-03T17:18:56.265ZError: Difference in extent metadata found!
213572024-02-03T17:18:56.265ZA Difference in extent metadata is expected here
213582024-02-03T17:18:56.265ZOn loop 98
213592024-02-03T17:18:56.265Z
213602024-02-03T17:18:56.271ZVerifying data now
213612024-02-03T17:18:57.319ZNo change after verify
213622024-02-03T17:18:57.319ZLoop: 98 Downstairs dump after verify (and repair):
213632024-02-03T17:18:57.331ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
213642024-02-03T17:18:57.331Z 0 000-019 188 188 188 325 325 325 F F F
213652024-02-03T17:18:57.331Z 1 020-039 188 188 188 325 325 325 F F F
213662024-02-03T17:18:57.331Z 2 040-059 187 187 187 323 323 323 F F F
213672024-02-03T17:18:57.331Z 3 060-079 196 196 196 335 335 335 F F F
213682024-02-03T17:18:57.335Z 4 080-099 196 196 196 333 333 333 F F F
213692024-02-03T17:18:57.335Z 5 100-119 196 196 196 333 333 333 F F F
213702024-02-03T17:18:57.335Z 6 120-139 197 197 197 338 338 338 F F F
213712024-02-03T17:18:57.335Z 7 140-159 190 190 190 327 327 327 F F F
213722024-02-03T17:18:57.335Z 8 160-179 195 195 195 331 331 331 F F F
213732024-02-03T17:18:57.335Z 9 180-199 196 196 196 336 336 336 F F F
213742024-02-03T17:18:57.335Z 10 200-219 187 187 187 323 323 323 F F F
213752024-02-03T17:18:57.335Z 11 220-239 194 194 194 330 330 330 F F F
213762024-02-03T17:18:57.335Z 12 240-259 194 194 194 330 330 330 F F F
213772024-02-03T17:18:57.336Z 13 260-279 194 194 194 330 330 330 F F F
213782024-02-03T17:18:57.336Z 14 280-299 196 196 196 337 337 337 F F F
213792024-02-03T17:18:57.336Z 15 300-319 196 196 196 335 335 335 F F F
213802024-02-03T17:18:57.336Z 16 320-339 194 194 194 330 330 330 F F F
213812024-02-03T17:18:57.336Z 17 340-359 196 196 196 334 334 334 F F F
213822024-02-03T17:18:57.336Z 18 360-379 196 196 196 334 334 334 F F F
213832024-02-03T17:18:57.336Z 19 380-399 194 194 194 330 330 330 F F F
213842024-02-03T17:18:57.336Z 20 400-419 192 192 192 329 329 329 F F F
213852024-02-03T17:18:57.336Z 21 420-439 195 195 195 331 331 331 F F F
213862024-02-03T17:18:57.336Z 22 440-459 196 196 196 334 334 334 F F F
213872024-02-03T17:18:57.336Z 23 460-479 192 192 192 329 329 329 F F F
213882024-02-03T17:18:57.336Z 24 480-499 192 192 192 329 329 329 F F F
213892024-02-03T17:18:57.336Z 25 500-519 191 191 191 328 328 328 F F F
213902024-02-03T17:18:57.337Z 26 520-539 194 194 194 330 330 330 F F F
213912024-02-03T17:18:57.337Z 27 540-559 194 194 194 330 330 330 F F F
213922024-02-03T17:18:57.337Z 28 560-579 197 197 197 338 338 338 F F F
213932024-02-03T17:18:57.337Z 29 580-599 196 196 196 335 335 335 F F F
213942024-02-03T17:18:57.337ZMax gen: 197, Max flush: 338
213952024-02-03T17:18:57.341Z
213962024-02-03T17:18:57.341ZBegin loop 99
213972024-02-03T17:18:57.341ZDownstairs to restart: 0
213982024-02-03T17:18:57.367Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.950596027Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927}
213992024-02-03T17:18:57.367Z{Crucible runtime is spawned
214002024-02-03T17:18:57.374Z"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"85ad6bd3639073b418f2b23f55877174588f0947\",\n git_commit_timestamp: \"2024-02-03T15:18:54.000000000Z\",\n git_branch: \"renovate/async-trait-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.951393407Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927}
214012024-02-03T17:18:57.374Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.951488688Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927}
214022024-02-03T17:18:57.374Z{"msg":"Crucible c4771a43-4173-418a-ab0a-c1e59ed2e4f2 has session id: 6ff9ee6b-dadd-43b2-bd7e-c797950f6af9","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.95153088Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214032024-02-03T17:18:57.374Z{"msg":"Upstairs opts: Upstairs UUID: c4771a43-4173-418a-ab0a-c1e59ed2e4f2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.951622031Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214042024-02-03T17:18:57.374Z{"msg":"Crucible stats registered with UUID: c4771a43-4173-418a-ab0a-c1e59ed2e4f2","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.951700957Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214052024-02-03T17:18:57.374Z{"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.951738517Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927}
214062024-02-03T17:18:57.374Z{"msg":"c4771a43-4173-418a-ab0a-c1e59ed2e4f2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.951771613Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214072024-02-03T17:18:57.374Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.951843454Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214082024-02-03T17:18:57.374Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.95188649Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214092024-02-03T17:18:57.374Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.951923514Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214102024-02-03T17:18:57.374Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.952054851Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214112024-02-03T17:18:57.374Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.952100981Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214122024-02-03T17:18:57.375Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.952142104Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214132024-02-03T17:18:57.375Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.952180219Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"":"io task","client":"2","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214142024-02-03T17:18:57.375Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.952259253Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"":"io task","client":"0","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214152024-02-03T17:18:57.375Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.952303825Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"":"io task","client":"1","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214162024-02-03T17:18:57.375Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.95234369Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"":"io task","client":"2","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214172024-02-03T17:18:57.375Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.952386557Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"":"io task","client":"0","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214182024-02-03T17:18:57.375Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.952468631Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"":"io task","client":"1","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214192024-02-03T17:18:57.375Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.953329029Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214202024-02-03T17:18:57.375Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.953464988Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214212024-02-03T17:18:57.375Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.953512887Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214222024-02-03T17:18:57.375Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.95355895Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214232024-02-03T17:18:57.375Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.954450743Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214242024-02-03T17:18:57.376Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.954750898Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214252024-02-03T17:18:57.376Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.95484518Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214262024-02-03T17:18:57.376Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.954891087Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214272024-02-03T17:18:57.376Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.955276457Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214282024-02-03T17:18:57.376Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.955374835Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214292024-02-03T17:18:57.376Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.955426248Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214302024-02-03T17:18:57.376Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.955649723Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214312024-02-03T17:18:57.376Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.955724229Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214322024-02-03T17:18:57.376Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.955811545Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214332024-02-03T17:18:57.376Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.955861374Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214342024-02-03T17:18:57.381Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.956573937Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214352024-02-03T17:18:57.381Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.956672316Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214362024-02-03T17:18:57.381Z{"msg":"[0]R flush_numbers[0..12]: [325, 325, 323, 335, 333, 333, 338, 327, 331, 336, 323, 330]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.956710096Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214372024-02-03T17:18:57.381Z{"msg":"[0]R generation[0..12]: [188, 188, 187, 196, 196, 196, 197, 190, 195, 196, 187, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.956734809Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214382024-02-03T17:18:57.381Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.956788394Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214392024-02-03T17:18:57.382Z{"msg":"[1]R flush_numbers[0..12]: [325, 325, 323, 335, 333, 333, 338, 327, 331, 336, 323, 330]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.956815007Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214402024-02-03T17:18:57.382Z{"msg":"[1]R generation[0..12]: [188, 188, 187, 196, 196, 196, 197, 190, 195, 196, 187, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.956839967Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214412024-02-03T17:18:57.382Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.956864325Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214422024-02-03T17:18:57.382Z{"msg":"[2]R flush_numbers[0..12]: [325, 325, 323, 335, 333, 333, 338, 327, 331, 336, 323, 330]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.956913145Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"":"Wait for a query_work_queue command to finish before sending IO
214432024-02-03T17:18:57.382Zdownstairs","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214442024-02-03T17:18:57.382Z{"msg":"[2]R generation[0..12]: [188, 188, 187, 196, 196, 196, 197, 190, 195, 196, 187, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.956943289Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214452024-02-03T17:18:57.382Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.956992108Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214462024-02-03T17:18:57.382Z{"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.957016719Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214472024-02-03T17:18:57.382Z{"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.957089707Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214482024-02-03T17:18:57.382Z{"msg":"Next flush: 339","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.957131657Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214492024-02-03T17:18:57.382Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.957208597Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214502024-02-03T17:18:57.382Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.957292868Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214512024-02-03T17:18:57.383Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.957375516Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214522024-02-03T17:18:57.383Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.957462837Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214532024-02-03T17:18:57.383Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.957547015Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214542024-02-03T17:18:57.383Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.957627592Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214552024-02-03T17:18:57.383Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.95770758Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214562024-02-03T17:18:57.383Z{"msg":"c4771a43-4173-418a-ab0a-c1e59ed2e4f2 is now active with session: 6ff9ee6b-dadd-43b2-bd7e-c797950f6af9","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.957789857Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"session_id":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
214572024-02-03T17:18:57.383Z6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214582024-02-03T17:18:57.383Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.95793501Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927,"session_id":"6ff9ee6b-dadd-43b2-bd7e-c797950f6af9"}
214592024-02-03T17:18:57.383Z{"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:18:56.957966718Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927}
214602024-02-03T17:18:57.383ZLoading write count information from file "/tmp/test_repair_verify.data"
214612024-02-03T17:18:57.383ZRun Repair workload
214622024-02-03T17:18:57.383Z01/30 Write block 4 len 32768 data: 12 12 14 12 17 20 18 12
214632024-02-03T17:18:57.384Z02/30 Read block 39 len 36864
214642024-02-03T17:18:57.387Z03/30 Read block 238 len 32768
214652024-02-03T17:18:57.387Z04/30 Write block 531 len 40960 data: 15 12 13 12 12 9 11 13 12 10
214662024-02-03T17:18:57.387Z05/30 Read block 483 len 28672
214672024-02-03T17:18:57.387Z06/30 Write block 460 len 32768 data: 21 19 17 19 16 14 17 17
214682024-02-03T17:18:57.391Z07/30 Read block 167 len 36864
214692024-02-03T17:18:57.391Z08/30 Read block 413 len 4096
214702024-02-03T17:18:57.391Z09/30 Write block 457 len 20480 data: 11 14 17 22 20
214712024-02-03T17:18:57.394Z10/30 Read block 556 len 20480
214722024-02-03T17:18:57.394Z11/30 Write block 470 len 20480 data: 15 14 13 9 9
214732024-02-03T17:18:57.397Z12/30 Flush
214742024-02-03T17:18:57.401Z13/30 Write block 544 len 24576 data: 13 13 15 18 15 17
214752024-02-03T17:18:57.401Z14/30 Read block 255 len 12288
214762024-02-03T17:18:57.404Z15/30 Read block 583 len 24576
214772024-02-03T17:18:57.404Z16/30 Read block 151 len 32768
214782024-02-03T17:18:57.407Z17/30 Flush
214792024-02-03T17:18:57.410Z18/30 Write block 519 len 40960 data: 12 13 12 15 17 16 19 18 18 17
214802024-02-03T17:18:57.410Z19/30 Write block 532 len 40960 data: 13 14 13 13 10 12 14 13 11 13
214812024-02-03T17:18:57.415Z20/30 Write block 534 len 40960 data: 14 14 11 13 15 14 12 14 16 15
214822024-02-03T17:18:57.415Z21/30 Write block 269 len 20480 data: 18 18 17 18 15
214832024-02-03T17:18:57.415Z22/30 Write block 573 len 4096 data: 15
214842024-02-03T17:18:57.415Z23/30 Read block 45 len 36864
214852024-02-03T17:18:57.418Z24/30 Flush
214862024-02-03T17:18:57.422Z25/30 Flush
214872024-02-03T17:18:57.425Z26/30 Write block 494 len 36864 data: 12 13 12 13 13 12 10 11 13
214882024-02-03T17:18:57.425Z27/30 Read block 248 len 4096
214892024-02-03T17:18:57.428Z28/30 Read block 174 len 16384
214902024-02-03T17:18:57.428Z29/30 Read block 107 len 28672
214912024-02-03T17:18:57.428Z30/30 Read block 396 len 12288
214922024-02-03T17:18:57.432Z----------------------------------------------------------------
214932024-02-03T17:18:57.432Z Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:30
214942024-02-03T17:18:57.432ZStates: Active Active Active
214952024-02-03T17:18:57.432ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
214962024-02-03T17:18:57.432Z 1 Acked 1000 Write 8 Done Done Done false
214972024-02-03T17:18:57.432Z 2 Acked 1001 Read 9 Done Done Done false
214982024-02-03T17:18:57.432Z 3 Acked 1002 Read 8 Sent Done Done false
214992024-02-03T17:18:57.432Z 4 Acked 1003 Write 10 Sent Done Done false
215002024-02-03T17:18:57.432Z 5 Acked 1004 Read 7 Sent Done Done false
215012024-02-03T17:18:57.433Z 6 Acked 1005 Write 8 Sent Done Done false
215022024-02-03T17:18:57.433Z 7 Acked 1006 Read 9 Sent Done Done false
215032024-02-03T17:18:57.433Z 8 Acked 1007 Read 1 Sent Done Done false
215042024-02-03T17:18:57.433Z 9 Acked 1008 Write 5 Sent Done Done false
215052024-02-03T17:18:57.433Z 10 Acked 1009 Read 5 Sent Done Done false
215062024-02-03T17:18:57.433Z 11 Acked 1010 Write 5 Sent Done Done false
215072024-02-03T17:18:57.433Z 12 Acked 1011 Flush 0 Sent Done Done false
215082024-02-03T17:18:57.433Z 13 Acked 1012 Write 6 Sent Done Done false
215092024-02-03T17:18:57.433Z 14 Acked 1013 Read 3 Sent Done Done false
215102024-02-03T17:18:57.433Z 15 Acked 1014 Read 6 Sent Done Done false
215112024-02-03T17:18:57.433Z 16 Acked 1015 Read 8 Sent Done Done false
215122024-02-03T17:18:57.433Z 17 Acked 1016 Flush 0 Sent Done Done false
215132024-02-03T17:18:57.433Z 18 Acked 1017 Write 10 Sent Done Done false
215142024-02-03T17:18:57.433Z 19 Acked 1018 Write 10 Sent Done Done false
215152024-02-03T17:18:57.434Z 20 Acked 1019 Write 10 Sent Done Done false
215162024-02-03T17:18:57.434Z 21 Acked 1020 Write 5 Sent Done Done false
215172024-02-03T17:18:57.434Z 22 Acked 1021 Write 1 Sent Done Done false
215182024-02-03T17:18:57.434Z 23 Acked 1022 Read 9 Sent Done Done false
215192024-02-03T17:18:57.434Z 24 Acked 1023 Flush 0 Sent Done Done false
215202024-02-03T17:18:57.434Z 25 Acked 1024 Flush 0 Sent Done Done false
215212024-02-03T17:18:57.434Z 26 Acked 1025 Write 9 Sent Done Done false
215222024-02-03T17:18:57.434Z 27 Acked 1026 Read 1 Sent Done Done false
215232024-02-03T17:18:57.434Z 28 Acked 1027 Read 4 Sent Done Done false
215242024-02-03T17:18:57.434Z 29 Acked 1028 Read 7 Sent Done Done false
215252024-02-03T17:18:57.434Z 30 Acked 1029 Read 3 Sent Done Done false
215262024-02-03T17:18:57.434Z STATES DS:0 DS:1 DS:2 TOTAL
215272024-02-03T17:18:57.434Z New 0 0 0 0
215282024-02-03T17:18:57.435Z Sent 28 0 0 28
215292024-02-03T17:18:57.435Z Done 2 30 30 62
215302024-02-03T17:18:57.435Z Skipped 0 0 0 0
215312024-02-03T17:18:57.435Z Error 0 0 0 0
215322024-02-03T17:18:57.435ZLast Flush: 0 1024 1024
215332024-02-03T17:18:57.435ZDownstairs last five completed:
215342024-02-03T17:18:57.435ZUpstairs last five completed: 30 29 28 27 26
215352024-02-03T17:18:57.435Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:18:57.014153801Z","hostname":"ip-10-150-1-9.us-west-2.compute.internal","pid":1927}
215362024-02-03T17:18:57.435ZWrote out file "/tmp/test_repair_verify.data"
215372024-02-03T17:18:57.439Z
215382024-02-03T17:18:57.444ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
215392024-02-03T17:18:57.479ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
215402024-02-03T17:18:57.479Z 0 000-019 188 198 198 325 339 339 T F F <---
215412024-02-03T17:18:57.479Z 1 020-039 188 188 188 325 325 325 F F F
215422024-02-03T17:18:57.479Z 2 040-059 187 187 187 323 323 323 F F F
215432024-02-03T17:18:57.479Z 3 060-079 196 196 196 335 335 335 F F F
215442024-02-03T17:18:57.485Z 4 080-099 196 196 196 333 333 333 F F F
215452024-02-03T17:18:57.485Z 5 100-119 196 196 196 333 333 333 F F F
215462024-02-03T17:18:57.485Z 6 120-139 197 197 197 338 338 338 F F F
215472024-02-03T17:18:57.485Z 7 140-159 190 190 190 327 327 327 F F F
215482024-02-03T17:18:57.485Z 8 160-179 195 195 195 331 331 331 F F F
215492024-02-03T17:18:57.485Z 9 180-199 196 196 196 336 336 336 F F F
215502024-02-03T17:18:57.485Z 10 200-219 187 187 187 323 323 323 F F F
215512024-02-03T17:18:57.485Z 11 220-239 194 194 194 330 330 330 F F F
215522024-02-03T17:18:57.485Z 12 240-259 194 194 194 330 330 330 F F F
215532024-02-03T17:18:57.485Z 13 260-279 194 198 198 330 341 341 F F F <---
215542024-02-03T17:18:57.485Z 14 280-299 196 196 196 337 337 337 F F F
215552024-02-03T17:18:57.486Z 15 300-319 196 196 196 335 335 335 F F F
215562024-02-03T17:18:57.486Z 16 320-339 194 194 194 330 330 330 F F F
215572024-02-03T17:18:57.486Z 17 340-359 196 196 196 334 334 334 F F F
215582024-02-03T17:18:57.486Z 18 360-379 196 196 196 334 334 334 F F F
215592024-02-03T17:18:57.486Z 19 380-399 194 194 194 330 330 330 F F F
215602024-02-03T17:18:57.486Z 20 400-419 192 192 192 329 329 329 F F F
215612024-02-03T17:18:57.486Z 21 420-439 195 195 195 331 331 331 F F F
215622024-02-03T17:18:57.486Z 22 440-459 196 198 198 334 339 339 F F F <---
215632024-02-03T17:18:57.486Z 23 460-479 192 198 198 329 339 339 F F F <---
215642024-02-03T17:18:57.486Z 24 480-499 192 192 192 329 329 329 F T T <---
215652024-02-03T17:18:57.486Z 25 500-519 191 198 198 328 341 341 F T T <---
215662024-02-03T17:18:57.486Z 26 520-539 194 198 198 330 341 341 F F F <---
215672024-02-03T17:18:57.486Z 27 540-559 194 198 198 330 341 341 F F F <---
215682024-02-03T17:18:57.486Z 28 560-579 197 198 198 338 341 341 F F F <---
215692024-02-03T17:18:57.486Z 29 580-599 196 196 196 335 335 335 F F F
215702024-02-03T17:18:57.486ZMax gen: 198, Max flush: 341
215712024-02-03T17:18:57.486ZError: Difference in extent metadata found!
215722024-02-03T17:18:57.486ZA Difference in extent metadata is expected here
215732024-02-03T17:18:57.486ZOn loop 99
215742024-02-03T17:18:57.486Z
215752024-02-03T17:18:57.486ZVerifying data now
215762024-02-03T17:19:02.298ZNo change after verify
215772024-02-03T17:19:02.298ZLoop: 99 Downstairs dump after verify (and repair):
215782024-02-03T17:19:02.321ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
215792024-02-03T17:19:02.321Z 0 000-019 198 198 198 339 339 339 F F F
215802024-02-03T17:19:02.322Z 1 020-039 188 188 188 325 325 325 F F F
215812024-02-03T17:19:02.322Z 2 040-059 187 187 187 323 323 323 F F F
215822024-02-03T17:19:02.322Z 3 060-079 196 196 196 335 335 335 F F F
215832024-02-03T17:19:02.322Z 4 080-099 196 196 196 333 333 333 F F F
215842024-02-03T17:19:02.322Z 5 100-119 196 196 196 333 333 333 F F F
215852024-02-03T17:19:02.322Z 6 120-139 197 197 197 338 338 338 F F F
215862024-02-03T17:19:02.322Z 7 140-159 190 190 190 327 327 327 F F F
215872024-02-03T17:19:02.322Z 8 160-179 195 195 195 331 331 331 F F F
215882024-02-03T17:19:02.326Z 9 180-199 196 196 196 336 336 336 F F F
215892024-02-03T17:19:02.326Z 10 200-219 187 187 187 323 323 323 F F F
215902024-02-03T17:19:02.326Z 11 220-239 194 194 194 330 330 330 F F F
215912024-02-03T17:19:02.326Z 12 240-259 194 194 194 330 330 330 F F F
215922024-02-03T17:19:02.326Z 13 260-279 198 198 198 341 341 341 F F F
215932024-02-03T17:19:02.326Z 14 280-299 196 196 196 337 337 337 F F F
215942024-02-03T17:19:02.326Z 15 300-319 196 196 196 335 335 335 F F F
215952024-02-03T17:19:02.326Z 16 320-339 194 194 194 330 330 330 F F F
215962024-02-03T17:19:02.326Z 17 340-359 196 196 196 334 334 334 F F F
215972024-02-03T17:19:02.326Z 18 360-379 196 196 196 334 334 334 F F F
215982024-02-03T17:19:02.326Z 19 380-399 194 194 194 330 330 330 F F F
215992024-02-03T17:19:02.326Z 20 400-419 192 192 192 329 329 329 F F F
216002024-02-03T17:19:02.326Z 21 420-439 195 195 195 331 331 331 F F F
216012024-02-03T17:19:02.326Z 22 440-459 198 198 198 339 339 339 F F F
216022024-02-03T17:19:02.326Z 23 460-479 198 198 198 339 339 339 F F F
216032024-02-03T17:19:02.326Z 24 480-499 199 199 199 342 342 342 F F F
216042024-02-03T17:19:02.326Z 25 500-519 199 199 199 342 342 342 F F F
216052024-02-03T17:19:02.327Z 26 520-539 198 198 198 341 341 341 F F F
216062024-02-03T17:19:02.327Z 27 540-559 198 198 198 341 341 341 F F F
216072024-02-03T17:19:02.327Z 28 560-579 198 198 198 341 341 341 F F F
216082024-02-03T17:19:02.327Z 29 580-599 196 196 196 335 335 335 F F F
216092024-02-03T17:19:02.327ZMax gen: 199, Max flush: 342
216102024-02-03T17:19:02.330Z9:33 Test duration
216112024-02-03T17:19:02.330ZTest completed
216122024-02-03T17:19:02.330Z
216132024-02-03T17:19:02.334Zreal 9:33.526869433
216142024-02-03T17:19:02.334Zuser 2:49.663308331
216152024-02-03T17:19:02.334Zsys 31.424744294
216162024-02-03T17:19:02.334Ztrap 0.147576580
216172024-02-03T17:19:02.334Ztflt 0.082045136
216182024-02-03T17:19:02.334Zdflt 0.156491057
216192024-02-03T17:19:02.334Zkflt 0.069628333
216202024-02-03T17:19:02.334Zlock 7:30:53.053419347
216212024-02-03T17:19:02.334Zslp 45:00.191649868
216222024-02-03T17:19:02.334Zlat 12.262821870
216232024-02-03T17:19:02.334Zstop 0.076463951
216242024-02-03T17:19:02.334Zprocess exited: duration 587483 ms, exit code 0
 
216252024-02-03T17:19:02.349Zfound 4 output files
216262024-02-03T17:19:02.349Zuploading: /tmp/test_repair_ds8810.txt (15887 bytes)
216272024-02-03T17:19:03.381Zuploaded: /tmp/test_repair_ds8810.txt
216282024-02-03T17:19:03.384Zuploading: /tmp/test_repair_ds8820.txt (101438 bytes)
216292024-02-03T17:19:04.815Zuploaded: /tmp/test_repair_ds8820.txt
216302024-02-03T17:19:04.816Zuploading: /tmp/test_repair_ds8830.txt (26737 bytes)
216312024-02-03T17:19:05.850Zuploaded: /tmp/test_repair_ds8830.txt
216322024-02-03T17:19:05.850Zuploading: /tmp/test_repair_out.txt (66616 bytes)
216332024-02-03T17:19:07.000Zuploaded: /tmp/test_repair_out.txt