01HRAQRCENMEGH3VNFV53N1SED: test-repair

BasicConfig {
    output_rules: [
        "/tmp/*.txt",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HRAQS5XHWDNCJHB8H1RYPCQP

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-03-06T20:53:40.583Zjob dependencies complete; ready to run (waiting for 15 m 49 s)
22024-03-06T20:55:29.812Zjob assigned to worker 01HRARQTS0TVE14DKZ437MF5TB (queued for 1 m 49 s)
32024-03-06T20:55:30.066Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-03-06T20:56:13.951Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-03-06T20:56:13.951Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-03-06T20:56:40.224Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-03-06T20:56:40.225Zdownloading input: /input/build/work/bins/crutest.gz
82024-03-06T20:57:09.210Zdownloaded input: /input/build/work/bins/crutest.gz
92024-03-06T20:57:09.210Zdownloading input: /input/build/work/bins/dsc.gz
102024-03-06T20:57:26.762Zdownloaded input: /input/build/work/bins/dsc.gz
112024-03-06T20:57:27.183Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-03-06T20:57:27.183Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-03-06T20:57:27.184Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-03-06T20:57:27.241Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-03-06T20:57:27.242Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-03-06T20:57:27.276Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-03-06T20:57:27.276Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-03-06T20:57:27.335Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-03-06T20:57:27.335Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-03-06T20:57:27.353Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-03-06T20:57:27.354Zdownloading input: /input/build/work/scripts/test_up.sh
222024-03-06T20:57:27.407Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-03-06T20:57:27.407Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-03-06T20:57:28.031Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-03-06T20:57:28.047Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-03-06T20:57:48.687Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-03-06T20:57:48.699Zstarting task 0: "setup"
282024-03-06T20:57:48.761Z++ uname -s
292024-03-06T20:57:48.773Z+ kern=SunOS
302024-03-06T20:57:48.774Z+ case "$kern" in
312024-03-06T20:57:48.774Z+ groupadd -g 12345 build
322024-03-06T20:57:48.774Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-03-06T20:57:50.769Z+ zfs create -o mountpoint=/work rpool/work
342024-03-06T20:57:50.855Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-03-06T20:57:50.859Z+ home_fs=zfs
362024-03-06T20:57:50.859Z+ [[ zfs == autofs ]]
372024-03-06T20:57:50.859Z+ mkdir -p /home/build
382024-03-06T20:57:50.859Z+ chown build:build /home/build /work
392024-03-06T20:57:52.864Z+ chmod 0700 /home/build /work
402024-03-06T20:57:52.867Zprocess exited: duration 4216 ms, exit code 0
 
412024-03-06T20:57:52.873Zstarting task 1: "authentication"
422024-03-06T20:57:52.893Zprocess exited: duration 19 ms, exit code 0
 
432024-03-06T20:57:52.900Zstarting task 2: "build"
442024-03-06T20:57:52.906Z+ banner cores
452024-03-06T20:57:52.961Z
462024-03-06T20:57:52.961Z #### #### ##### ###### ####
472024-03-06T20:57:52.961Z # # # # # # # #
482024-03-06T20:57:52.961Z # # # # # ##### ####
492024-03-06T20:57:52.961Z # # # ##### # #
502024-03-06T20:57:52.961Z # # # # # # # # #
512024-03-06T20:57:52.961Z #### #### # # ###### ####
522024-03-06T20:57:52.962Z
532024-03-06T20:57:52.968Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-03-06T20:57:52.979Z+ echo 'input bins dir contains:'
552024-03-06T20:57:52.980Zinput bins dir contains:
562024-03-06T20:57:52.980Z+ ls -ltr /input/build/work/bins
572024-03-06T20:57:52.980Ztotal 649113
582024-03-06T20:57:52.980Z-rw-r--r-- 1 root root 96182092 Mar 6 20:56 crucible-downstairs.gz
592024-03-06T20:57:52.980Z-rw-r--r-- 1 root root 80274145 Mar 6 20:56 crucible-hammer.gz
602024-03-06T20:57:52.980Z-rw-r--r-- 1 root root 92543568 Mar 6 20:57 crutest.gz
612024-03-06T20:57:52.980Z-rw-r--r-- 1 root root 63131242 Mar 6 20:57 dsc.gz
622024-03-06T20:57:52.980Z+ echo 'input script dir contains:'
632024-03-06T20:57:52.980Zinput script dir contains:
642024-03-06T20:57:52.981Z+ ls -ltr /input/build/work/scripts
652024-03-06T20:57:53.006Ztotal 39
662024-03-06T20:57:53.007Z-rw-r--r-- 1 root root 3056 Mar 6 20:57 perf-downstairs-tick.d
672024-03-06T20:57:53.007Z-rw-r--r-- 1 root root 1738 Mar 6 20:57 test_ds.sh
682024-03-06T20:57:53.007Z-rw-r--r-- 1 root root 3746 Mar 6 20:57 test_live_repair.sh
692024-03-06T20:57:53.008Z-rw-r--r-- 1 root root 6416 Mar 6 20:57 test_repair.sh
702024-03-06T20:57:53.008Z-rw-r--r-- 1 root root 2969 Mar 6 20:57 test_replay.sh
712024-03-06T20:57:53.008Z-rw-r--r-- 1 root root 9465 Mar 6 20:57 test_up.sh
722024-03-06T20:57:53.008Z-rw-r--r-- 1 root root 2824 Mar 6 20:57 upstairs_info.d
732024-03-06T20:57:53.008Z+ pfexec chmod +x /input/build/work/scripts/perf-downstairs-tick.d /input/build/work/scripts/test_ds.sh /input/build/work/scripts/test_live_repair.sh /input/build/work/scripts/test_repair.sh /input/build/work/scripts/test_replay.sh /input/build/work/scripts/test_up.sh /input/build/work/scripts/upstairs_info.d
742024-03-06T20:57:53.008Z+ echo ' chmod input script dir contains:'
752024-03-06T20:57:53.008Z chmod input script dir contains:
762024-03-06T20:57:53.008Z+ ls -ltr /input/build/work/scripts
772024-03-06T20:57:53.008Ztotal 39
782024-03-06T20:57:53.008Z-rwxr-xr-x 1 root root 3056 Mar 6 20:57 perf-downstairs-tick.d
792024-03-06T20:57:53.008Z-rwxr-xr-x 1 root root 1738 Mar 6 20:57 test_ds.sh
802024-03-06T20:57:53.008Z-rwxr-xr-x 1 root root 3746 Mar 6 20:57 test_live_repair.sh
812024-03-06T20:57:53.008Z-rwxr-xr-x 1 root root 6416 Mar 6 20:57 test_repair.sh
822024-03-06T20:57:53.008Z-rwxr-xr-x 1 root root 2969 Mar 6 20:57 test_replay.sh
832024-03-06T20:57:53.009Z-rwxr-xr-x 1 root root 9465 Mar 6 20:57 test_up.sh
842024-03-06T20:57:53.009Z-rwxr-xr-x 1 root root 2824 Mar 6 20:57 upstairs_info.d
852024-03-06T20:57:53.009Z+ banner unpack
862024-03-06T20:57:53.009Z
872024-03-06T20:57:53.009Z # # # # ##### ## #### # #
882024-03-06T20:57:53.009Z # # ## # # # # # # # # #
892024-03-06T20:57:53.009Z # # # # # # # # # # ####
902024-03-06T20:57:53.009Z # # # # # ##### ###### # # #
912024-03-06T20:57:53.009Z # # # ## # # # # # # #
922024-03-06T20:57:53.009Z #### # # # # # #### # #
932024-03-06T20:57:53.009Z
942024-03-06T20:57:53.009Z+ mkdir -p /var/tmp/bins
952024-03-06T20:57:53.009Z+ for t in "$input/bins/"*.gz
962024-03-06T20:57:53.009Z++ basename /input/build/work/bins/crucible-downstairs.gz
972024-03-06T20:57:53.009Z+ b=crucible-downstairs.gz
982024-03-06T20:57:53.010Z+ b=crucible-downstairs
992024-03-06T20:57:53.010Z+ gunzip
1002024-03-06T20:57:55.090Z+ chmod +x /var/tmp/bins/crucible-downstairs
1012024-03-06T20:57:55.093Z+ for t in "$input/bins/"*.gz
1022024-03-06T20:57:55.093Z++ basename /input/build/work/bins/crucible-hammer.gz
1032024-03-06T20:57:55.096Z+ b=crucible-hammer.gz
1042024-03-06T20:57:55.096Z+ b=crucible-hammer
1052024-03-06T20:57:55.096Z+ gunzip
1062024-03-06T20:58:01.114Z+ chmod +x /var/tmp/bins/crucible-hammer
1072024-03-06T20:58:01.141Z+ for t in "$input/bins/"*.gz
1082024-03-06T20:58:01.141Z++ basename /input/build/work/bins/crutest.gz
1092024-03-06T20:58:01.141Z+ b=crutest.gz
1102024-03-06T20:58:01.141Z+ b=crutest
1112024-03-06T20:58:01.141Z+ gunzip
1122024-03-06T20:58:01.141Z+ chmod +x /var/tmp/bins/crutest
1132024-03-06T20:58:01.141Z+ for t in "$input/bins/"*.gz
1142024-03-06T20:58:01.141Z++ basename /input/build/work/bins/dsc.gz
1152024-03-06T20:58:01.141Z+ b=dsc.gz
1162024-03-06T20:58:01.141Z+ b=dsc
1172024-03-06T20:58:01.141Z+ gunzip
1182024-03-06T20:58:01.141Z+ chmod +x /var/tmp/bins/dsc
1192024-03-06T20:58:01.141Z+ export BINDIR=/var/tmp/bins
1202024-03-06T20:58:01.141Z+ BINDIR=/var/tmp/bins
1212024-03-06T20:58:01.141Z+ banner repair
1222024-03-06T20:58:01.141Z
1232024-03-06T20:58:01.142Z ##### ###### ##### ## # #####
1242024-03-06T20:58:01.142Z # # # # # # # # # #
1252024-03-06T20:58:01.142Z # # ##### # # # # # # #
1262024-03-06T20:58:01.142Z ##### # ##### ###### # #####
1272024-03-06T20:58:01.142Z # # # # # # # # #
1282024-03-06T20:58:01.142Z # # ###### # # # # # #
1292024-03-06T20:58:01.142Z
1302024-03-06T20:58:01.142Z+ ptime -m bash /input/build/work/scripts/test_repair.sh -N
1312024-03-06T20:58:01.142ZTurn off color for downstairs dump
1322024-03-06T20:58:01.142ZCreating region directory at: /var/tmp/test_repair
1332024-03-06T20:58:01.142ZCreating dsc directory at: /tmp/dsc
1342024-03-06T20:58:01.302ZDownstairs region 0 created at /var/tmp/test_repair/8810 in 0.98620605
1352024-03-06T20:58:01.995ZDownstairs region 1 created at /var/tmp/test_repair/8820 in 0.6948144
1362024-03-06T20:58:02.307ZDownstairs region 2 created at /var/tmp/test_repair/8830 in 0.30457443
1372024-03-06T20:58:02.308ZRegion set with 3 regions was created
1382024-03-06T20:58:02.353Z/var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1138 1139 1140
1392024-03-06T20:58:02.353ZDump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
1402024-03-06T20:58:02.369Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.115213886Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142}
1412024-03-06T20:58:02.369Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.115932546Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142}
1422024-03-06T20:58:02.369Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.115957445Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142}
1432024-03-06T20:58:02.370Z{"msg":"Crucible 9e93b132-bafe-429e-874e-cec222ccbcb4 has session id: d381b88c-39ea-488e-b81a-8a6b41170ba7","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.115975954Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1442024-03-06T20:58:02.370Z{"msg":"Upstairs opts: Upstairs UUID: 9e93b132-bafe-429e-874e-cec222ccbcb4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.116006552Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1452024-03-06T20:58:02.370Z{"msg":"Crucible stats registered with UUID: 9e93b132-bafe-429e-874e-cec222ccbcb4","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.11602491Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1462024-03-06T20:58:02.370ZCrucible runtime is spawned
1472024-03-06T20:58:02.370Z{"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.116525986Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142}
1482024-03-06T20:58:02.370Z{"msg":"9e93b132-bafe-429e-874e-cec222ccbcb4 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.116803637Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1492024-03-06T20:58:02.370Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.116833364Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"client":"0","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1502024-03-06T20:58:02.370Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.116850543Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"client":"0","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1512024-03-06T20:58:02.370Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.116865002Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"client":"1","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1522024-03-06T20:58:02.370Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.116879391Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"client":"1","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1532024-03-06T20:58:02.371Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.11689555Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"client":"2","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1542024-03-06T20:58:02.371Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.116925548Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"client":"2","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1552024-03-06T20:58:02.371Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.117097936Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"":"io task","client":"0","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1562024-03-06T20:58:02.371Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.117139083Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"":"io task","client":"1","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1572024-03-06T20:58:02.371Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.117257355Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"":"io task","client":"2","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1582024-03-06T20:58:02.371Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.117421724Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"":"io task","client":"1","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1592024-03-06T20:58:02.371Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.117488779Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"":"io task","client":"2","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1602024-03-06T20:58:02.372Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.117542915Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"":"io task","client":"0","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1612024-03-06T20:58:02.372Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.118943979Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"client":"2","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1622024-03-06T20:58:02.372Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.118979236Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"client":"2","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1632024-03-06T20:58:02.372Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.118995835Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"client":"1","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1642024-03-06T20:58:02.372Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.119049372Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"client":"1","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1652024-03-06T20:58:02.372Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.119129466Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"client":"0","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1662024-03-06T20:58:02.372Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.119174273Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"client":"0","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1672024-03-06T20:58:02.373Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.120069541Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"client":"0","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1682024-03-06T20:58:02.373Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.120113738Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"client":"2","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1692024-03-06T20:58:02.373Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.120216421Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"client":"1","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1702024-03-06T20:58:02.373Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.120764684Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"client":"2","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1712024-03-06T20:58:02.373Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.12081537Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"client":"2","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1722024-03-06T20:58:02.373Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.120844438Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1732024-03-06T20:58:02.374Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.120868006Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"client":"1","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1742024-03-06T20:58:02.374Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.120890685Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"client":"1","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1752024-03-06T20:58:02.374Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.120979839Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1762024-03-06T20:58:02.374Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.121012996Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"client":"0","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1772024-03-06T20:58:02.374Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.121038115Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"client":"0","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1782024-03-06T20:58:02.374Z{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.121062213Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"":"downstairs","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1792024-03-06T20:58:02.374Z{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.121099401Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"":"downstairs","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1802024-03-06T20:58:02.374Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.121154957Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"":"downstairs","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1812024-03-06T20:58:02.375Z{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.121190824Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"":"downstairs","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1822024-03-06T20:58:02.375Z{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.121210403Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"":"downstairs","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1832024-03-06T20:58:02.375Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.121240231Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"":"downstairs","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1842024-03-06T20:58:02.375Z{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.12125947Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"":"downstairs","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1852024-03-06T20:58:02.375Z{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.121288128Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"":"downstairs","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1862024-03-06T20:58:02.375Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.121308486Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"":"downstairs","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1872024-03-06T20:58:02.375ZWait for a query_work_queue command to finish before sending IO
1882024-03-06T20:58:02.375Z{"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.121338954Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"":"downstairs","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1892024-03-06T20:58:02.376Z{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.121360332Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"":"downstairs","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1902024-03-06T20:58:02.376Z{"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.121387351Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"":"downstairs","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1912024-03-06T20:58:02.376Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.12140418Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"":"downstairs","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1922024-03-06T20:58:02.376Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.121432867Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1932024-03-06T20:58:02.376Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.121449996Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"client":"0","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1942024-03-06T20:58:02.376Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.121484264Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"client":"1","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1952024-03-06T20:58:02.376Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.121502503Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"client":"2","":"downstairs","session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1962024-03-06T20:58:02.376Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.121524281Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1972024-03-06T20:58:02.376Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.121557089Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1982024-03-06T20:58:02.376Z{"msg":"9e93b132-bafe-429e-874e-cec222ccbcb4 is now active with session: d381b88c-39ea-488e-b81a-8a6b41170ba7","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.121596596Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
1992024-03-06T20:58:02.376Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.121623084Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142,"session_id":"d381b88c-39ea-488e-b81a-8a6b41170ba7"}
2002024-03-06T20:58:02.376Z{"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:01.121654352Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142}
2012024-03-06T20:58:02.376ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2022024-03-06T20:58:02.377ZFill test
2032024-03-06T20:58:02.509ZRead and Verify all blocks (0..600 range:false)
2042024-03-06T20:58:02.579ZWrote out file "/tmp/test_repair_verify.data"
2052024-03-06T20:58:02.579ZCLIENT: Tests done. All submitted work has been ACK'd
2062024-03-06T20:58:02.595Z----------------------------------------------------------------
2072024-03-06T20:58:02.595Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:6
2082024-03-06T20:58:02.595ZStates: Active Active Active
2092024-03-06T20:58:02.596ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
2102024-03-06T20:58:02.596Z 8 Acked 1007 Read 100 Done Done Done false
2112024-03-06T20:58:02.596Z 9 Acked 1008 Read 100 Done Done Done false
2122024-03-06T20:58:02.596Z 10 Acked 1009 Read 100 Done Done Done false
2132024-03-06T20:58:02.596Z 11 Acked 1010 Read 100 Done Done Done false
2142024-03-06T20:58:02.597Z 12 Acked 1011 Read 100 Done Done Done false
2152024-03-06T20:58:02.597Z 13 Acked 1012 Read 100 Done Done Done false
2162024-03-06T20:58:02.597Z STATES DS:0 DS:1 DS:2 TOTAL
2172024-03-06T20:58:02.597Z New 0 0 0 0
2182024-03-06T20:58:02.597Z Sent 0 0 0 0
2192024-03-06T20:58:02.597Z Done 6 6 6 18
2202024-03-06T20:58:02.597Z Skipped 0 0 0 0
2212024-03-06T20:58:02.597Z Error 0 0 0 0
2222024-03-06T20:58:02.597ZLast Flush: 1006 1006 1006
2232024-03-06T20:58:02.597ZDownstairs last five completed: 1006 1005 1004 1003 1002
2242024-03-06T20:58:02.597ZUpstairs last five completed: 13 12 11 10 9
2252024-03-06T20:58:02.598ZCLIENT: Up:0 ds:6 act:3
2262024-03-06T20:58:06.578Z----------------------------------------------------------------
2272024-03-06T20:58:06.578Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0
2282024-03-06T20:58:06.578ZDownstairs last five completed: 1013 1012 1011 1010 1009
2292024-03-06T20:58:06.578ZUpstairs last five completed: 14 13 12 11 10
2302024-03-06T20:58:06.578ZCLIENT: Up:0 ds:0 act:3
2312024-03-06T20:58:06.578ZCLIENT: All crucible jobs finished, exiting program
2322024-03-06T20:58:06.581Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T20:58:05.354308781Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1142}
2332024-03-06T20:58:06.584Z
2342024-03-06T20:58:06.585ZBegin loop 1
2352024-03-06T20:58:06.585ZDownstairs to restart: 1
2362024-03-06T20:58:06.604Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.379532222Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144}
2372024-03-06T20:58:06.604Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.379808173Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144}
2382024-03-06T20:58:06.604Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.379827052Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144}
2392024-03-06T20:58:06.604Z{"msg":"Crucible 1c4186f7-26a2-4c0e-ad7e-7941e55840be has session id: 74f2ca58-92c3-4119-84f0-c656b7392678","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.379848171Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2402024-03-06T20:58:06.611Z{"msg":"Upstairs opts: Upstairs UUID: 1c4186f7-26a2-4c0e-ad7e-7941e55840be, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.379894278Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2412024-03-06T20:58:06.611Z{"msg":"Crucible stats registered with UUID: 1c4186f7-26a2-4c0e-ad7e-7941e55840be","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.379929055Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144Crucible runtime is spawned
2422024-03-06T20:58:06.611Z,"session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2432024-03-06T20:58:06.611Z{"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.380046077Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144}
2442024-03-06T20:58:06.611Z{"msg":"1c4186f7-26a2-4c0e-ad7e-7941e55840be active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.380085834Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2452024-03-06T20:58:06.611Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.380103233Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"client":"0","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2462024-03-06T20:58:06.611Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.380128851Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"client":"0","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2472024-03-06T20:58:06.611Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.38014602Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"client":"1","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2482024-03-06T20:58:06.611Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.380184367Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"client":"1","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2492024-03-06T20:58:06.611Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.380215265Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"client":"2","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2502024-03-06T20:58:06.611Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.380249973Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"client":"2","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2512024-03-06T20:58:06.611Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.380283511Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"":"io task","client":"2","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2522024-03-06T20:58:06.611Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.380311609Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"":"io task","client":"0","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2532024-03-06T20:58:06.611Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.380343696Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"":"io task","client":"1","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2542024-03-06T20:58:06.611Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.380511785Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"":"io task","client":"2","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2552024-03-06T20:58:06.611Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.380563771Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"":"io task","client":"1","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2562024-03-06T20:58:06.611Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.38059087Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"":"io task","client":"0","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2572024-03-06T20:58:06.612Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.381330759Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"client":"2","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2582024-03-06T20:58:06.612Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.381369076Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"client":"2","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2592024-03-06T20:58:06.612Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.381385845Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"client":"1","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2602024-03-06T20:58:06.612Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.381401183Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"client":"1","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2612024-03-06T20:58:06.612Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.381424102Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"client":"0","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2622024-03-06T20:58:06.612Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.381439131Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"client":"0","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2632024-03-06T20:58:06.612Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.382022761Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"client":"2","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2642024-03-06T20:58:06.612Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.382124454Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"client":"0","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2652024-03-06T20:58:06.612Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.382149152Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"client":"1","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2662024-03-06T20:58:06.612Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.382665347Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"client":"2","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2672024-03-06T20:58:06.612Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.382718793Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"client":"2","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2682024-03-06T20:58:06.612Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.382737271Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2692024-03-06T20:58:06.612Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.382877202Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"client":"1","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2702024-03-06T20:58:06.612Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.38290553Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"client":"1","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2712024-03-06T20:58:06.612Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.382920649Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2722024-03-06T20:58:06.612Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.382933748Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"client":"0","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2732024-03-06T20:58:06.612Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.382947317Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"client":"0","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2742024-03-06T20:58:06.612Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.382961576Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"":"downstairs","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2752024-03-06T20:58:06.612Z{"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.382976195Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"":"downstairs","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2762024-03-06T20:58:06.612Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.382991554Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"":"downstairs","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2772024-03-06T20:58:06.612Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.383005933Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"":"downstairs","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2782024-03-06T20:58:06.612Z{"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.383027511Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"":"downstairs","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2792024-03-06T20:58:06.612Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.383042021Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"":"downstairs","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2802024-03-06T20:58:06.613Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.383055809Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"":"downstairs","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2812024-03-06T20:58:06.613Z{"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.383069549Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"":"downstairs","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2822024-03-06T20:58:06.613Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.383091227Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"":"downstairs","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2832024-03-06T20:58:06.613Z{"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.383104856Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"":"downstairs","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2842024-03-06T20:58:06.613Z{"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.383118395Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"":"downstairs","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2852024-03-06T20:58:06.613Z{"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.383137964Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"":"downstairs","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2862024-03-06T20:58:06.613Z{"msg":"All extents match"Wait for a query_work_queue command to finish before sending IO
2872024-03-06T20:58:06.613Z,"v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.383154473Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"":"downstairs","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2882024-03-06T20:58:06.613Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.383167782Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2892024-03-06T20:58:06.613Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.383187931Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"client":"0","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2902024-03-06T20:58:06.613Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.38320243Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"client":"1","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2912024-03-06T20:58:06.613Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.383216349Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"client":"2","":"downstairs","session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2922024-03-06T20:58:06.613Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.383236287Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2932024-03-06T20:58:06.613Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.383248896Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2942024-03-06T20:58:06.613Z{"msg":"1c4186f7-26a2-4c0e-ad7e-7941e55840be is now active with session: 74f2ca58-92c3-4119-84f0-c656b7392678","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.383261785Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2952024-03-06T20:58:06.613Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.383273705Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144,"session_id":"74f2ca58-92c3-4119-84f0-c656b7392678"}
2962024-03-06T20:58:06.613Z{"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:05.383292803Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144}
2972024-03-06T20:58:06.613ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2982024-03-06T20:58:06.617ZLoading write count information from file "/tmp/test_repair_verify.data"
2992024-03-06T20:58:06.617ZRun Repair workload
3002024-03-06T20:58:06.617Z01/30 Write block 554 len 36864 data: 2 2 2 2 2 2 2 2 2
3012024-03-06T20:58:06.617Z02/30 Read block 348 len 40960
3022024-03-06T20:58:06.617Z03/30 Read block 123 len 12288
3032024-03-06T20:58:06.617Z04/30 Write block 66 len 28672 data: 2 2 2 2 2 2 2
3042024-03-06T20:58:06.617Z05/30 Read block 128 len 28672
3052024-03-06T20:58:06.617Z06/30 Read block 27 len 36864
3062024-03-06T20:58:06.618Z07/30 Write block 33 len 12288 data: 2 2 2
3072024-03-06T20:58:06.621Z08/30 Write block 284 len 36864 data: 2 2 2 2 2 2 2 2 2
3082024-03-06T20:58:06.621Z09/30 Read block 201 len 16384
3092024-03-06T20:58:06.621Z10/30 Read block 361 len 36864
3102024-03-06T20:58:06.621Z11/30 Read block 387 len 40960
3112024-03-06T20:58:06.628Z12/30 Read block 321 len 12288
3122024-03-06T20:58:06.629Z13/30 Write block 370 len 12288 data: 2 2 2
3132024-03-06T20:58:06.629Z14/30 Read block 496 len 16384
3142024-03-06T20:58:06.629Z15/30 Read block 294 len 24576
3152024-03-06T20:58:06.629Z16/30 Read block 118 len 20480
3162024-03-06T20:58:06.629Z17/30 Read block 589 len 16384
3172024-03-06T20:58:06.637Z18/30 Read block 500 len 4096
3182024-03-06T20:58:06.637Z19/30 Read block 343 len 28672
3192024-03-06T20:58:06.637Z20/30 Write block 481 len 36864 data: 2 2 2 2 2 2 2 2 2
3202024-03-06T20:58:06.638Z21/30 Read block 211 len 28672
3212024-03-06T20:58:06.638Z22/30 Read block 477 len 20480
3222024-03-06T20:58:06.638Z23/30 Read block 514 len 20480
3232024-03-06T20:58:06.638Z24/30 Read block 277 len 40960
3242024-03-06T20:58:06.638Z25/30 Read block 29 len 8192
3252024-03-06T20:58:06.652Z26/30 Write block 172 len 24576 data: 2 2 2 2 2 2
3262024-03-06T20:58:06.652Z27/30 Write block 540 len 20480 data: 2 2 2 2 2
3272024-03-06T20:58:06.652Z28/30 Read block 232 len 20480
3282024-03-06T20:58:06.652Z29/30 Write block 356 len 40960 data: 2 2 2 2 2 2 2 2 2 2
3292024-03-06T20:58:06.652Z30/30 Read block 71 len 32768
3302024-03-06T20:58:06.653Z----------------------------------------------------------------
3312024-03-06T20:58:06.653Z Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:30
3322024-03-06T20:58:06.653ZStates: Active Active Active
3332024-03-06T20:58:06.653ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
3342024-03-06T20:58:06.653Z 1 Acked 1000 Write 9 Done Done Done false
3352024-03-06T20:58:06.653Z 2 Acked 1001 Read 10 Done Done Done false
3362024-03-06T20:58:06.653Z 3 Acked 1002 Read 3 Done Sent Done false
3372024-03-06T20:58:06.653Z 4 Acked 1003 Write 7 Done Sent Done false
3382024-03-06T20:58:06.653Z 5 Acked 1004 Read 7 Done Sent Done false
3392024-03-06T20:58:06.653Z 6 Acked 1005 Read 9 Done Sent Done false
3402024-03-06T20:58:06.653Z 7 Acked 1006 Write 3 Done Sent Done false
3412024-03-06T20:58:06.653Z 8 Acked 1007 Write 9 Done Sent Done false
3422024-03-06T20:58:06.653Z 9 Acked 1008 Read 4 Done Sent Done false
3432024-03-06T20:58:06.653Z 10 Acked 1009 Read 9 Done Sent Done false
3442024-03-06T20:58:06.653Z 11 Acked 1010 Read 10 Done Sent Done false
3452024-03-06T20:58:06.653Z 12 Acked 1011 Read 3 Done Sent Done false
3462024-03-06T20:58:06.653Z 13 Acked 1012 Write 3 Done Sent Done false
3472024-03-06T20:58:06.654Z 14 Acked 1013 Read 4 Done Sent Done false
3482024-03-06T20:58:06.654Z 15 Acked 1014 Read 6 Done Sent Done false
3492024-03-06T20:58:06.654Z 16 Acked 1015 Read 5 Done Sent Done false
3502024-03-06T20:58:06.654Z 17 Acked 1016 Read 4 Done Sent Done false
3512024-03-06T20:58:06.654Z 18 Acked 1017 Read 1 Done Sent Done false
3522024-03-06T20:58:06.654Z 19 Acked 1018 Read 7 Done Sent Done false
3532024-03-06T20:58:06.654Z 20 Acked 1019 Write 9 Done Sent Done false
3542024-03-06T20:58:06.654Z 21 Acked 1020 Read 7 Done Sent Done false
3552024-03-06T20:58:06.654Z 22 Acked 1021 Read 5 Done Sent Done false
3562024-03-06T20:58:06.654Z 23 Acked 1022 Read 5 Done Sent Done false
3572024-03-06T20:58:06.654Z 24 Acked 1023 Read 10 Done Sent Done false
3582024-03-06T20:58:06.654Z 25 Acked 1024 Read 2 Done Sent Done false
3592024-03-06T20:58:06.654Z 26 Acked 1025 Write 6 Done Sent Done false
3602024-03-06T20:58:06.654Z 27 Acked 1026 Write 5 Done Sent Done false
3612024-03-06T20:58:06.654Z 28 Acked 1027 Read 5 Done Sent Done false
3622024-03-06T20:58:06.655Z 29 Acked 1028 Write 10 Done Sent Done false
3632024-03-06T20:58:06.655Z 30 Acked 1029 Read 8 Done Sent Sent false
3642024-03-06T20:58:06.655Z STATES DS:0 DS:1 DS:2 TOTAL
3652024-03-06T20:58:06.655Z New 0 0 0 0
3662024-03-06T20:58:06.655Z Sent 0 28 1 29
3672024-03-06T20:58:06.655Z Done 30 2 29 61
3682024-03-06T20:58:06.655Z Skipped 0 0 0 0
3692024-03-06T20:58:06.655Z Error 0 0 0 0
3702024-03-06T20:58:06.655ZLast Flush: 0 0 0
3712024-03-06T20:58:06.655ZDownstairs last five completed:
3722024-03-06T20:58:06.655ZUpstairs last five completed: 30 29 28 27 26
3732024-03-06T20:58:06.655Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T20:58:05.415666392Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1144}
3742024-03-06T20:58:06.655ZWrote out file "/tmp/test_repair_verify.data"
3752024-03-06T20:58:06.655Z
3762024-03-06T20:58:06.665ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
3772024-03-06T20:58:06.687ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3782024-03-06T20:58:06.687Z 0 000-019 1 1 1 1 1 1 F F F
3792024-03-06T20:58:06.687Z 1 020-039 1 1 1 1 1 1 T F T <---
3802024-03-06T20:58:06.687Z 2 040-059 1 1 1 1 1 1 F F F
3812024-03-06T20:58:06.687Z 3 060-079 1 1 1 1 1 1 T F T <---
3822024-03-06T20:58:06.687Z 4 080-099 1 1 1 1 1 1 F F F
3832024-03-06T20:58:06.687Z 5 100-119 1 1 1 1 1 1 F F F
3842024-03-06T20:58:06.687Z 6 120-139 1 1 1 1 1 1 F F F
3852024-03-06T20:58:06.687Z 7 140-159 1 1 1 1 1 1 F F F
3862024-03-06T20:58:06.687Z 8 160-179 1 1 1 1 1 1 T F T <---
3872024-03-06T20:58:06.687Z 9 180-199 1 1 1 1 1 1 F F F
3882024-03-06T20:58:06.687Z 10 200-219 1 1 1 1 1 1 F F F
3892024-03-06T20:58:06.687Z 11 220-239 1 1 1 1 1 1 F F F
3902024-03-06T20:58:06.687Z 12 240-259 1 1 1 1 1 1 F F F
3912024-03-06T20:58:06.687Z 13 260-279 1 1 1 1 1 1 F F F
3922024-03-06T20:58:06.687Z 14 280-299 1 1 1 1 1 1 T F T <---
3932024-03-06T20:58:06.687Z 15 300-319 1 1 1 1 1 1 F F F
3942024-03-06T20:58:06.687Z 16 320-339 1 1 1 1 1 1 F F F
3952024-03-06T20:58:06.687Z 17 340-359 1 1 1 1 1 1 T F T <---
3962024-03-06T20:58:06.687Z 18 360-379 1 1 1 1 1 1 T F T <---
3972024-03-06T20:58:06.687Z 19 380-399 1 1 1 1 1 1 F F F
3982024-03-06T20:58:06.688Z 20 400-419 1 1 1 1 1 1 F F F
3992024-03-06T20:58:06.688Z 21 420-439 1 1 1 1 1 1 F F F
4002024-03-06T20:58:06.688Z 22 440-459 1 1 1 1 1 1 F F F
4012024-03-06T20:58:06.688Z 23 460-479 1 1 1 1 1 1 F F F
4022024-03-06T20:58:06.688Z 24 480-499 1 1 1 1 1 1 T F T <---
4032024-03-06T20:58:06.688Z 25 500-519 1 1 1 1 1 1 F F F
4042024-03-06T20:58:06.688Z 26 520-539 1 1 1 1 1 1 F F F
4052024-03-06T20:58:06.688Z 27 540-559 1 1 1 1 1 1 T T T
4062024-03-06T20:58:06.688Z 28 560-579 1 1 1 1 1 1 T T T
4072024-03-06T20:58:06.688Z 29 580-599 1 1 1 1 1 1 F F F
4082024-03-06T20:58:06.688ZMax gen: 1, Max flush: 1
4092024-03-06T20:58:06.688ZError: Difference in extent metadata found!
4102024-03-06T20:58:06.688ZA Difference in extent metadata is expected here
4112024-03-06T20:58:06.688ZOn loop 1
4122024-03-06T20:58:06.688Z
4132024-03-06T20:58:06.688ZVerifying data now
4142024-03-06T20:58:13.550ZNo change after verify
4152024-03-06T20:58:13.550ZLoop: 1 Downstairs dump after verify (and repair):
4162024-03-06T20:58:13.560ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4172024-03-06T20:58:13.560Z 0 000-019 1 1 1 1 1 1 F F F
4182024-03-06T20:58:13.560Z 1 020-039 2 2 2 2 2 2 F F F
4192024-03-06T20:58:13.560Z 2 040-059 1 1 1 1 1 1 F F F
4202024-03-06T20:58:13.560Z 3 060-079 2 2 2 2 2 2 F F F
4212024-03-06T20:58:13.560Z 4 080-099 1 1 1 1 1 1 F F F
4222024-03-06T20:58:13.561Z 5 100-119 1 1 1 1 1 1 F F F
4232024-03-06T20:58:13.561Z 6 120-139 1 1 1 1 1 1 F F F
4242024-03-06T20:58:13.561Z 7 140-159 1 1 1 1 1 1 F F F
4252024-03-06T20:58:13.561Z 8 160-179 2 2 2 2 2 2 F F F
4262024-03-06T20:58:13.561Z 9 180-199 1 1 1 1 1 1 F F F
4272024-03-06T20:58:13.561Z 10 200-219 1 1 1 1 1 1 F F F
4282024-03-06T20:58:13.561Z 11 220-239 1 1 1 1 1 1 F F F
4292024-03-06T20:58:13.562Z 12 240-259 1 1 1 1 1 1 F F F
4302024-03-06T20:58:13.562Z 13 260-279 1 1 1 1 1 1 F F F
4312024-03-06T20:58:13.562Z 14 280-299 2 2 2 2 2 2 F F F
4322024-03-06T20:58:13.562Z 15 300-319 1 1 1 1 1 1 F F F
4332024-03-06T20:58:13.562Z 16 320-339 1 1 1 1 1 1 F F F
4342024-03-06T20:58:13.562Z 17 340-359 2 2 2 2 2 2 F F F
4352024-03-06T20:58:13.563Z 18 360-379 2 2 2 2 2 2 F F F
4362024-03-06T20:58:13.563Z 19 380-399 1 1 1 1 1 1 F F F
4372024-03-06T20:58:13.563Z 20 400-419 1 1 1 1 1 1 F F F
4382024-03-06T20:58:13.563Z 21 420-439 1 1 1 1 1 1 F F F
4392024-03-06T20:58:13.563Z 22 440-459 1 1 1 1 1 1 F F F
4402024-03-06T20:58:13.563Z 23 460-479 1 1 1 1 1 1 F F F
4412024-03-06T20:58:13.564Z 24 480-499 2 2 2 2 2 2 F F F
4422024-03-06T20:58:13.564Z 25 500-519 1 1 1 1 1 1 F F F
4432024-03-06T20:58:13.564Z 26 520-539 1 1 1 1 1 1 F F F
4442024-03-06T20:58:13.564Z 27 540-559 2 2 2 2 2 2 F F F
4452024-03-06T20:58:13.564Z 28 560-579 2 2 2 2 2 2 F F F
4462024-03-06T20:58:13.564Z 29 580-599 1 1 1 1 1 1 F F F
4472024-03-06T20:58:13.564ZMax gen: 2, Max flush: 2
4482024-03-06T20:58:13.572Z
4492024-03-06T20:58:13.572ZBegin loop 2
4502024-03-06T20:58:13.572ZDownstairs to restart: 2
4512024-03-06T20:58:13.585Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.352356486Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152}
4522024-03-06T20:58:13.585Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.35259009Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152}
4532024-03-06T20:58:13.586Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.352607659Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152}
4542024-03-06T20:58:13.586Z{"msg":"Crucible fce0db1c-8e97-430a-b3cc-739d28149f1e has session id: 1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.352629447Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4552024-03-06T20:58:13.586Z{"msg":"Upstairs opts: Upstairs UUID: fce0db1c-8e97-430a-b3cc-739d28149f1e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.352671764Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4562024-03-06T20:58:13.586ZCrucible runtime is spawned
4572024-03-06T20:58:13.586Z{"msg":"Crucible stats registered with UUID: fce0db1c-8e97-430a-b3cc-739d28149f1e","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.352702182Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4582024-03-06T20:58:13.587Z{"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.352743549Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152}
4592024-03-06T20:58:13.587Z{"msg":"fce0db1c-8e97-430a-b3cc-739d28149f1e active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.35287378Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4602024-03-06T20:58:13.587Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.352892499Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"client":"0","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4612024-03-06T20:58:13.587Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.352907818Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"client":"0","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4622024-03-06T20:58:13.588Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.352945835Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"client":"1","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4632024-03-06T20:58:13.588Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.352967214Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"client":"1","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4642024-03-06T20:58:13.588Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.352986482Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"client":"2","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4652024-03-06T20:58:13.588Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.353014841Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"client":"2","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4662024-03-06T20:58:13.588Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.35302977Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"":"io task","client":"2","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4672024-03-06T20:58:13.588Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.353058908Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"":"io task","client":"1","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4682024-03-06T20:58:13.588Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.353145592Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"":"io task","client":"0","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4692024-03-06T20:58:13.588Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.353187049Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"":"io task","client":"1","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4702024-03-06T20:58:13.589Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.353344238Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"":"io task","client":"0","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4712024-03-06T20:58:13.589Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.353371206Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"":"io task","client":"2","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4722024-03-06T20:58:13.589Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.353975805Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"client":"0","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4732024-03-06T20:58:13.589Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.354018722Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"client":"0","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4742024-03-06T20:58:13.589Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.35404694Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"client":"1","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4752024-03-06T20:58:13.589Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.354073658Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"client":"1","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4762024-03-06T20:58:13.605Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.354441752Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"client":"2","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4772024-03-06T20:58:13.605Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.35448301Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"client":"2","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4782024-03-06T20:58:13.606Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.354509728Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"client":"1","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4792024-03-06T20:58:13.606Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.354732542Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"client":"0","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4802024-03-06T20:58:13.606Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.355069639Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"client":"1","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4812024-03-06T20:58:13.606Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.355152503Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"client":"1","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4822024-03-06T20:58:13.606Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.355184291Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4832024-03-06T20:58:13.606Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.355233518Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"client":"2","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4842024-03-06T20:58:13.607Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.355253806Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"client":"0","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4852024-03-06T20:58:13.607Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.355283704Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"client":"0","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4862024-03-06T20:58:13.607Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.355301953Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4872024-03-06T20:58:13.607Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.355625641Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"client":"2","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4882024-03-06T20:58:13.607Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.355680217Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"client":"2","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4892024-03-06T20:58:13.608Z{"msg":"[0]R flush_numbers[0..12]: [1, 2, 1, 2, 1, 1, 1, 1, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.355715354Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"":"downstairs","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4902024-03-06T20:58:13.608Z{"msg":"[0]R generation[0..12]: [1, 2, 1, 2, 1, 1, 1, 1, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.355740503Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"":"downstairs","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4912024-03-06T20:58:13.608Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
4922024-03-06T20:58:13.608Z,"time":"2024-03-06T20:58:12.355769321Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"":"downstairs","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4932024-03-06T20:58:13.608Z{"msg":"[1]R flush_numbers[0..12]: [1, 2, 1, 2, 1, 1, 1, 1, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.355794869Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"":"downstairs","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4942024-03-06T20:58:13.608Z{"msg":"[1]R generation[0..12]: [1, 2, 1, 2, 1, 1, 1, 1, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.355822457Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"":"downstairs","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4952024-03-06T20:58:13.608Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.355846716Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"":"downstairs","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4962024-03-06T20:58:13.609Z{"msg":"[2]R flush_numbers[0..12]: [1, 2, 1, 2, 1, 1, 1, 1, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.355873694Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"":"downstairs","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4972024-03-06T20:58:13.609Z{"msg":"[2]R generation[0..12]: [1, 2, 1, 2, 1, 1, 1, 1, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.355896832Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"":"downstairs","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4982024-03-06T20:58:13.609Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.35592555Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"":"downstairs","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
4992024-03-06T20:58:13.609Z{"msg":"Max found gen is 3","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.355948889Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"":"downstairs","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
5002024-03-06T20:58:13.609Z{"msg":"Generation requested: 4 >= found:3","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.355980086Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"":"downstairs","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
5012024-03-06T20:58:13.610Z{"msg":"Next flush: 3","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.356030533Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"":"downstairs","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
5022024-03-06T20:58:13.610Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.356055931Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"":"downstairs","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
5032024-03-06T20:58:13.610Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.356079929Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
5042024-03-06T20:58:13.610Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.356104378Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"client":"0","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
5052024-03-06T20:58:13.610Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.356133836Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"client":"1","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
5062024-03-06T20:58:13.610Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.356154104Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"client":"2","":"downstairs","session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
5072024-03-06T20:58:13.610Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.356184662Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
5082024-03-06T20:58:13.611Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.356206481Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
5092024-03-06T20:58:13.611ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
5102024-03-06T20:58:13.611Z{"msg":"fce0db1c-8e97-430a-b3cc-739d28149f1e is now active with session: 1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.356238678Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
5112024-03-06T20:58:13.611Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.356298054Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152,"session_id":"1dd36ec9-cd53-4acc-bd13-c4c3bed3aae6"}
5122024-03-06T20:58:13.611Z{"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:12.356328392Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152}
5132024-03-06T20:58:13.611ZLoading write count information from file "/tmp/test_repair_verify.data"
5142024-03-06T20:58:13.612ZRun Repair workload
5152024-03-06T20:58:13.612Z01/30 Write block 390 len 20480 data: 2 2 2 2 2
5162024-03-06T20:58:13.612Z02/30 Write block 352 len 40960 data: 2 2 2 2 3 3 3 3 3 3
5172024-03-06T20:58:13.612Z03/30 Read block 315 len 20480
5182024-03-06T20:58:13.612Z04/30 Write block 524 len 40960 data: 2 2 2 2 2 2 2 2 2 2
5192024-03-06T20:58:13.612Z05/30 Read block 240 len 24576
5202024-03-06T20:58:13.613Z06/30 Write block 231 len 36864 data: 2 2 2 2 2 2 2 2 2
5212024-03-06T20:58:13.613Z07/30 Read block 300 len 20480
5222024-03-06T20:58:13.613Z08/30 Read block 390 len 40960
5232024-03-06T20:58:13.613Z09/30 Write block 384 len 16384 data: 2 2 2 2
5242024-03-06T20:58:13.613Z10/30 Read block 464 len 12288
5252024-03-06T20:58:13.613Z11/30 Read block 135 len 24576
5262024-03-06T20:58:14.056Z12/30 Read block 51 len 4096
5272024-03-06T20:58:14.057Z13/30 Read block 421 len 24576
5282024-03-06T20:58:14.057Z14/30 Read block 438 len 32768
5292024-03-06T20:58:14.057Z15/30 Write block 192 len 4096 data: 2
5302024-03-06T20:58:14.057Z16/30 Write block 179 len 28672 data: 2 2 2 2 2 2 2
5312024-03-06T20:58:14.057Z17/30 Read block 186 len 32768
5322024-03-06T20:58:14.057Z18/30 Read block 6 len 16384
5332024-03-06T20:58:14.057Z19/30 Read block 47 len 16384
5342024-03-06T20:58:14.057Z20/30 Read block 270 len 24576
5352024-03-06T20:58:14.058Z21/30 Read block 577 len 36864
5362024-03-06T20:58:14.058Z22/30 Write block 413 len 40960 data: 2 2 2 2 2 2 2 2 2 2
5372024-03-06T20:58:14.058Z23/30 Read block 144 len 40960
5382024-03-06T20:58:14.058Z24/30 Write block 265 len 16384 data: 2 2 2 2
5392024-03-06T20:58:14.058Z25/30 Read block 133 len 36864
5402024-03-06T20:58:14.058Z26/30 Read block 206 len 16384
5412024-03-06T20:58:14.058Z27/30 Read block 269 len 20480
5422024-03-06T20:58:14.059Z28/30 Read block 401 len 28672
5432024-03-06T20:58:14.059Z29/30 Write block 340 len 36864 data: 2 2 2 2 2 2 2 2 2
5442024-03-06T20:58:14.059Z30/30 Read block 16 len 40960
5452024-03-06T20:58:14.059Z----------------------------------------------------------------
5462024-03-06T20:58:14.059Z Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:30
5472024-03-06T20:58:14.059ZStates: Active Active Active
5482024-03-06T20:58:14.059ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
5492024-03-06T20:58:14.059Z 1 Acked 1000 Write 5 Done Done Sent false
5502024-03-06T20:58:14.059Z 2 Acked 1001 Write 10 Done Done Sent false
5512024-03-06T20:58:14.059Z 3 Acked 1002 Read 5 Done Done Sent false
5522024-03-06T20:58:14.060Z 4 Acked 1003 Write 10 Done Done Sent false
5532024-03-06T20:58:14.060Z 5 Acked 1004 Read 6 Done Done Sent false
5542024-03-06T20:58:14.060Z 6 Acked 1005 Write 9 Done Done Sent false
5552024-03-06T20:58:14.060Z 7 Acked 1006 Read 5 Done Done Sent false
5562024-03-06T20:58:14.060Z 8 Acked 1007 Read 10 Done Done Sent false
5572024-03-06T20:58:14.060Z 9 Acked 1008 Write 4 Done Done Sent false
5582024-03-06T20:58:14.060Z 10 Acked 1009 Read 3 Done Done Sent false
5592024-03-06T20:58:14.060Z 11 Acked 1010 Read 6 Done Done Sent false
5602024-03-06T20:58:14.060Z 12 Acked 1011 Read 1 Done Done Sent false
5612024-03-06T20:58:14.060Z 13 Acked 1012 Read 6 Done Done Sent false
5622024-03-06T20:58:14.060Z 14 Acked 1013 Read 8 Done Done Sent false
5632024-03-06T20:58:14.060Z 15 Acked 1014 Write 1 Done Done Sent false
5642024-03-06T20:58:14.061Z 16 Acked 1015 Write 7 Done Done Sent false
5652024-03-06T20:58:14.061Z 17 Acked 1016 Read 8 Done Done Sent false
5662024-03-06T20:58:14.061Z 18 Acked 1017 Read 4 Done Done Sent false
5672024-03-06T20:58:14.061Z 19 Acked 1018 Read 4 Done Done Sent false
5682024-03-06T20:58:14.061Z 20 Acked 1019 Read 6 Done Done Sent false
5692024-03-06T20:58:14.061Z 21 Acked 1020 Read 9 Done Done Sent false
5702024-03-06T20:58:14.061Z 22 Acked 1021 Write 10 Done Done Sent false
5712024-03-06T20:58:14.061Z 23 Acked 1022 Read 10 Done Done Sent false
5722024-03-06T20:58:14.061Z 24 Acked 1023 Write 4 Done Done Sent false
5732024-03-06T20:58:14.061Z 25 Acked 1024 Read 9 Done Done Sent false
5742024-03-06T20:58:14.061Z 26 Acked 1025 Read 4 Done Done Sent false
5752024-03-06T20:58:14.062Z 27 Acked 1026 Read 5 Done Done Sent false
5762024-03-06T20:58:14.062Z 28 Acked 1027 Read 7 Done Done Sent false
5772024-03-06T20:58:14.062Z 29 Acked 1028 Write 9 Done Sent Sent false
5782024-03-06T20:58:14.062Z 30 Acked 1029 Read 10 Done Sent Sent false
5792024-03-06T20:58:14.062Z STATES DS:0 DS:1 DS:2 TOTAL
5802024-03-06T20:58:14.062Z New 0 0 0 0
5812024-03-06T20:58:14.062Z Sent 0 2 30 32
5822024-03-06T20:58:14.062Z Done 30 28 0 58
5832024-03-06T20:58:14.062Z Skipped 0 0 0 0
5842024-03-06T20:58:14.062Z Error 0 0 0 0
5852024-03-06T20:58:14.062ZLast Flush: 0 0 0
5862024-03-06T20:58:14.063ZDownstairs last five completed:
5872024-03-06T20:58:14.063ZUpstairs last five completed: 30 29 28 27 26
5882024-03-06T20:58:14.063Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T20:58:12.388810654Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1152}
5892024-03-06T20:58:14.063ZWrote out file "/tmp/test_repair_verify.data"
5902024-03-06T20:58:14.063Z
5912024-03-06T20:58:14.086ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
5922024-03-06T20:58:14.086ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5932024-03-06T20:58:14.086Z 0 000-019 1 1 1 1 1 1 F F F
5942024-03-06T20:58:14.086Z 1 020-039 2 2 2 2 2 2 F F F
5952024-03-06T20:58:14.086Z 2 040-059 1 1 1 1 1 1 F F F
5962024-03-06T20:58:14.086Z 3 060-079 2 2 2 2 2 2 F F F
5972024-03-06T20:58:14.086Z 4 080-099 1 1 1 1 1 1 F F F
5982024-03-06T20:58:14.086Z 5 100-119 1 1 1 1 1 1 F F F
5992024-03-06T20:58:14.086Z 6 120-139 1 1 1 1 1 1 F F F
6002024-03-06T20:58:14.086Z 7 140-159 1 1 1 1 1 1 F F F
6012024-03-06T20:58:14.086Z 8 160-179 2 2 2 2 2 2 T T F <---
6022024-03-06T20:58:14.087Z 9 180-199 1 1 1 1 1 1 T T F <---
6032024-03-06T20:58:14.087Z 10 200-219 1 1 1 1 1 1 F F F
6042024-03-06T20:58:14.087Z 11 220-239 1 1 1 1 1 1 T T F <---
6052024-03-06T20:58:14.087Z 12 240-259 1 1 1 1 1 1 F F F
6062024-03-06T20:58:14.087Z 13 260-279 1 1 1 1 1 1 T T F <---
6072024-03-06T20:58:14.087Z 14 280-299 2 2 2 2 2 2 F F F
6082024-03-06T20:58:14.087Z 15 300-319 1 1 1 1 1 1 F F F
6092024-03-06T20:58:14.087Z 16 320-339 1 1 1 1 1 1 F F F
6102024-03-06T20:58:14.087Z 17 340-359 2 2 2 2 2 2 T T F <---
6112024-03-06T20:58:14.087Z 18 360-379 2 2 2 2 2 2 T T F <---
6122024-03-06T20:58:14.087Z 19 380-399 1 1 1 1 1 1 T T F <---
6132024-03-06T20:58:14.087Z 20 400-419 1 1 1 1 1 1 T T F <---
6142024-03-06T20:58:14.087Z 21 420-439 1 1 1 1 1 1 T T F <---
6152024-03-06T20:58:14.087Z 22 440-459 1 1 1 1 1 1 F F F
6162024-03-06T20:58:14.087Z 23 460-479 1 1 1 1 1 1 F F F
6172024-03-06T20:58:14.087Z 24 480-499 2 2 2 2 2 2 F F F
6182024-03-06T20:58:14.087Z 25 500-519 1 1 1 1 1 1 F F F
6192024-03-06T20:58:14.087Z 26 520-539 1 1 1 1 1 1 T T F <---
6202024-03-06T20:58:14.087Z 27 540-559 2 2 2 2 2 2 F F F
6212024-03-06T20:58:14.087Z 28 560-579 2 2 2 2 2 2 F F F
6222024-03-06T20:58:14.087Z 29 580-599 1 1 1 1 1 1 F F F
6232024-03-06T20:58:14.087ZMax gen: 2, Max flush: 2
6242024-03-06T20:58:14.088ZError: Difference in extent metadata found!
6252024-03-06T20:58:14.088ZA Difference in extent metadata is expected here
6262024-03-06T20:58:14.088ZOn loop 2
6272024-03-06T20:58:14.088Z
6282024-03-06T20:58:14.088ZVerifying data now
6292024-03-06T20:58:18.802ZNo change after verify
6302024-03-06T20:58:18.802ZLoop: 2 Downstairs dump after verify (and repair):
6312024-03-06T20:58:18.824ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6322024-03-06T20:58:18.825Z 0 000-019 1 1 1 1 1 1 F F F
6332024-03-06T20:58:18.825Z 1 020-039 2 2 2 2 2 2 F F F
6342024-03-06T20:58:18.825Z 2 040-059 1 1 1 1 1 1 F F F
6352024-03-06T20:58:18.825Z 3 060-079 2 2 2 2 2 2 F F F
6362024-03-06T20:58:18.825Z 4 080-099 1 1 1 1 1 1 F F F
6372024-03-06T20:58:18.825Z 5 100-119 1 1 1 1 1 1 F F F
6382024-03-06T20:58:18.825Z 6 120-139 1 1 1 1 1 1 F F F
6392024-03-06T20:58:18.825Z 7 140-159 1 1 1 1 1 1 F F F
6402024-03-06T20:58:18.825Z 8 160-179 3 3 3 3 3 3 F F F
6412024-03-06T20:58:18.826Z 9 180-199 3 3 3 3 3 3 F F F
6422024-03-06T20:58:18.826Z 10 200-219 1 1 1 1 1 1 F F F
6432024-03-06T20:58:18.826Z 11 220-239 3 3 3 3 3 3 F F F
6442024-03-06T20:58:18.826Z 12 240-259 1 1 1 1 1 1 F F F
6452024-03-06T20:58:18.826Z 13 260-279 3 3 3 3 3 3 F F F
6462024-03-06T20:58:18.826Z 14 280-299 2 2 2 2 2 2 F F F
6472024-03-06T20:58:18.826Z 15 300-319 1 1 1 1 1 1 F F F
6482024-03-06T20:58:18.826Z 16 320-339 1 1 1 1 1 1 F F F
6492024-03-06T20:58:18.827Z 17 340-359 3 3 3 3 3 3 F F F
6502024-03-06T20:58:18.827Z 18 360-379 3 3 3 3 3 3 F F F
6512024-03-06T20:58:18.827Z 19 380-399 3 3 3 3 3 3 F F F
6522024-03-06T20:58:18.827Z 20 400-419 3 3 3 3 3 3 F F F
6532024-03-06T20:58:18.827Z 21 420-439 3 3 3 3 3 3 F F F
6542024-03-06T20:58:18.827Z 22 440-459 1 1 1 1 1 1 F F F
6552024-03-06T20:58:18.827Z 23 460-479 1 1 1 1 1 1 F F F
6562024-03-06T20:58:18.827Z 24 480-499 2 2 2 2 2 2 F F F
6572024-03-06T20:58:18.827Z 25 500-519 1 1 1 1 1 1 F F F
6582024-03-06T20:58:18.828Z 26 520-539 3 3 3 3 3 3 F F F
6592024-03-06T20:58:18.828Z 27 540-559 2 2 2 2 2 2 F F F
6602024-03-06T20:58:18.828Z 28 560-579 2 2 2 2 2 2 F F F
6612024-03-06T20:58:18.828Z 29 580-599 1 1 1 1 1 1 F F F
6622024-03-06T20:58:18.828ZMax gen: 3, Max flush: 3
6632024-03-06T20:58:18.869Z
6642024-03-06T20:58:18.869ZBegin loop 3
6652024-03-06T20:58:18.869ZDownstairs to restart: 2
6662024-03-06T20:58:18.869Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.609185671Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160}
6672024-03-06T20:58:18.869Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.609475521Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160}
6682024-03-06T20:58:18.869Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.609496139Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160}
6692024-03-06T20:58:18.869Z{"msg":"Crucible 69da18bb-b2cd-40a8-b2ad-5cbe62672df0 has session id: 3ebdf750-77f4-44f1-a75e-312b0030d7fc","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.609512628Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
6702024-03-06T20:58:18.870Z{"Crucible runtime is spawned
6712024-03-06T20:58:18.870Zmsg":"Upstairs opts: Upstairs UUID: 69da18bb-b2cd-40a8-b2ad-5cbe62672df0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.609575604Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
6722024-03-06T20:58:18.870Z{"msg":"Crucible stats registered with UUID: 69da18bb-b2cd-40a8-b2ad-5cbe62672df0","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.60963221Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
6732024-03-06T20:58:18.870Z{"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.609666078Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160}
6742024-03-06T20:58:18.870Z{"msg":"69da18bb-b2cd-40a8-b2ad-5cbe62672df0 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.609814767Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
6752024-03-06T20:58:18.870Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.609851785Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"client":"0","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
6762024-03-06T20:58:18.870Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.609881243Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"client":"0","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
6772024-03-06T20:58:18.870Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.609909901Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"client":"1","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
6782024-03-06T20:58:18.870Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.609951928Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"client":"1","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
6792024-03-06T20:58:18.895Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.609983506Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"client":"2","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
6802024-03-06T20:58:18.895Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.610016494Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"client":"2","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
6812024-03-06T20:58:18.895Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.610100488Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"":"io task","client":"2","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
6822024-03-06T20:58:18.895Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.610148614Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"":"io task","client":"0","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
6832024-03-06T20:58:18.895Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.610290005Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"":"io task","client":"1","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
6842024-03-06T20:58:18.895Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.610329492Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"":"io task","client":"2","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
6852024-03-06T20:58:18.895Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.610365769Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"":"io task","client":"1","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
6862024-03-06T20:58:18.895Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.610392357Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"":"io task","client":"0","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
6872024-03-06T20:58:18.896Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.611006245Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"client":"1","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
6882024-03-06T20:58:18.896Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.611034013Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"client":"1","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
6892024-03-06T20:58:18.896Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.611049642Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"client":"0","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
6902024-03-06T20:58:18.896Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.611063561Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"client":"0","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
6912024-03-06T20:58:18.896Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.61165599Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"client":"0","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
6922024-03-06T20:58:18.896Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.61181109Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"client":"1","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
6932024-03-06T20:58:18.896Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.611965719Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"client":"2","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
6942024-03-06T20:58:18.896Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.612020755Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"client":"2","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
6952024-03-06T20:58:18.896Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.612199063Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"client":"0","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
6962024-03-06T20:58:18.896Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.61223789Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"client":"0","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
6972024-03-06T20:58:18.896Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.612261939Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
6982024-03-06T20:58:18.896Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.612314015Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"client":"1","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
6992024-03-06T20:58:18.897Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.612335434Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"client":"1","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
7002024-03-06T20:58:18.897Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.612357362Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
7012024-03-06T20:58:18.897Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.612622364Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"client":"2","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
7022024-03-06T20:58:18.897Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.613022546Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"client":"2","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
7032024-03-06T20:58:18.897Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.613068283Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"client":"2","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
7042024-03-06T20:58:18.897Z{"msg":"[0]R flush_numbers[0..12]: [1, 2, 1, 2, 1, 1, 1, 1, 3, 3, 1, 3]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.613096321Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"":"downstairs","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
7052024-03-06T20:58:18.897Z{"msg":"[0]R generation[0..12]: [1, 2, 1, 2, 1, 1, 1, 1, 3, 3, 1, 3]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.613128989Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"":"downstairs","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
7062024-03-06T20:58:18.897Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.613154987Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"":"downstairs","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
7072024-03-06T20:58:18.897Z{"msg":"[1]R flush_numbers[0..12]: [1, 2, 1, 2, 1, 1, 1, 1, 3, 3, 1, 3]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
7082024-03-06T20:58:18.897Z2024-03-06T20:58:17.613187755Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"":"downstairs","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
7092024-03-06T20:58:18.897Z{"msg":"[1]R generation[0..12]: [1, 2, 1, 2, 1, 1, 1, 1, 3, 3, 1, 3]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.613214563Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"":"downstairs","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
7102024-03-06T20:58:18.897Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.613242501Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"":"downstairs","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
7112024-03-06T20:58:18.897Z{"msg":"[2]R flush_numbers[0..12]: [1, 2, 1, 2, 1, 1, 1, 1, 3, 3, 1, 3]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.61326607Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"":"downstairs","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
7122024-03-06T20:58:18.897Z{"msg":"[2]R generation[0..12]: [1, 2, 1, 2, 1, 1, 1, 1, 3, 3, 1, 3]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.613294108Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"":"downstairs","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
7132024-03-06T20:58:18.897Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.613318506Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"":"downstairs","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
7142024-03-06T20:58:18.897Z{"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.613344874Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"":"downstairs","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
7152024-03-06T20:58:18.897Z{"msg":"Generation requested: 6 >= found:4","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.613367823Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"":"downstairs","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
7162024-03-06T20:58:18.897Z{"msg":"Next flush: 4","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.613394371Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"":"downstairs","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
7172024-03-06T20:58:18.898Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.613416809Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"":"downstairs","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
7182024-03-06T20:58:18.898Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
7192024-03-06T20:58:18.898Z,"time":"2024-03-06T20:58:17.613443727Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
7202024-03-06T20:58:18.898Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.613468576Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"client":"0","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
7212024-03-06T20:58:18.898Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.613495094Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"client":"1","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
7222024-03-06T20:58:18.898Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.613517182Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"client":"2","":"downstairs","session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
7232024-03-06T20:58:18.898Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.61354343Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
7242024-03-06T20:58:18.898Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.613564739Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
7252024-03-06T20:58:18.898Z{"msg":"69da18bb-b2cd-40a8-b2ad-5cbe62672df0 is now active with session: 3ebdf750-77f4-44f1-a75e-312b0030d7fc","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.613591757Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
7262024-03-06T20:58:18.898Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.613612286Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160,"session_id":"3ebdf750-77f4-44f1-a75e-312b0030d7fc"}
7272024-03-06T20:58:18.899Z{"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:17.613637224Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160}
7282024-03-06T20:58:18.899ZLoading write count information from file "/tmp/test_repair_verify.data"
7292024-03-06T20:58:18.899ZRun Repair workload
7302024-03-06T20:58:18.899Z01/30 Write block 238 len 8192 data: 3 3
7312024-03-06T20:58:18.899Z02/30 Read block 183 len 36864
7322024-03-06T20:58:18.899Z03/30 Read block 271 len 36864
7332024-03-06T20:58:18.899Z04/30 Write block 241 len 24576 data: 2 2 2 2 2 2
7342024-03-06T20:58:18.899Z05/30 Read block 179 len 20480
7352024-03-06T20:58:18.899Z06/30 Write block 286 len 24576 data: 3 3 3 3 3 3
7362024-03-06T20:58:18.899Z07/30 Read block 401 len 24576
7372024-03-06T20:58:18.899Z08/30 Write block 252 len 32768 data: 2 2 2 2 2 2 2 2
7382024-03-06T20:58:18.899Z09/30 Write block 356 len 4096 data: 4
7392024-03-06T20:58:18.899Z10/30 Write block 540 len 40960 data: 3 3 3 3 3 2 2 2 2 2
7402024-03-06T20:58:18.899Z11/30 Read block 463 len 40960
7412024-03-06T20:58:18.900Z12/30 Read block 140 len 20480
7422024-03-06T20:58:18.900Z13/30 Flush
7432024-03-06T20:58:18.912Z14/30 Write block 342 len 12288 data: 3 3 3
7442024-03-06T20:58:18.912Z15/30 Read block 480 len 12288
7452024-03-06T20:58:18.912Z16/30 Read block 163 len 16384
7462024-03-06T20:58:18.912Z17/30 Read block 494 len 16384
7472024-03-06T20:58:18.913Z18/30 Read block 561 len 40960
7482024-03-06T20:58:18.913Z19/30 Read block 71 len 40960
7492024-03-06T20:58:18.913Z20/30 Flush
7502024-03-06T20:58:18.913Z21/30 Flush
7512024-03-06T20:58:18.913Z22/30 Write block 231 len 8192 data: 3 3
7522024-03-06T20:58:18.913Z23/30 Flush
7532024-03-06T20:58:18.913Z24/30 Write block 426 len 40960 data: 2 2 2 2 2 2 2 2 2 2
7542024-03-06T20:58:18.913Z25/30 Read block 532 len 40960
7552024-03-06T20:58:18.914Z26/30 Read block 326 len 24576
7562024-03-06T20:58:18.914Z27/30 Read block 259 len 4096
7572024-03-06T20:58:18.914Z28/30 Read block 121 len 24576
7582024-03-06T20:58:18.914Z29/30 Read block 177 len 24576
7592024-03-06T20:58:18.914Z30/30 Read block 165 len 4096
7602024-03-06T20:58:18.914Z----------------------------------------------------------------
7612024-03-06T20:58:18.914Z Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:30
7622024-03-06T20:58:18.914ZStates: Active Active Active
7632024-03-06T20:58:18.914ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
7642024-03-06T20:58:18.914Z 1 Acked 1000 Write 2 Done Done Sent false
7652024-03-06T20:58:18.915Z 2 Acked 1001 Read 9 Done Done Sent false
7662024-03-06T20:58:18.915Z 3 Acked 1002 Read 9 Done Done Sent false
7672024-03-06T20:58:18.915Z 4 Acked 1003 Write 6 Done Done Sent false
7682024-03-06T20:58:18.915Z 5 Acked 1004 Read 5 Done Done Sent false
7692024-03-06T20:58:18.915Z 6 Acked 1005 Write 6 Done Done Sent false
7702024-03-06T20:58:18.915Z 7 Acked 1006 Read 6 Done Done Sent false
7712024-03-06T20:58:18.915Z 8 Acked 1007 Write 8 Done Done Sent false
7722024-03-06T20:58:18.915Z 9 Acked 1008 Write 1 Done Done Sent false
7732024-03-06T20:58:18.916Z 10 Acked 1009 Write 10 Done Done Sent false
7742024-03-06T20:58:18.916Z 11 Acked 1010 Read 10 Done Done Sent false
7752024-03-06T20:58:18.916Z 12 Acked 1011 Read 5 Done Done Sent false
7762024-03-06T20:58:18.916Z 13 Acked 1012 Flush 0 Done Done Sent false
7772024-03-06T20:58:18.916Z 14 Acked 1013 Write 3 Done Done Sent false
7782024-03-06T20:58:18.916Z 15 Acked 1014 Read 3 Done Done Sent false
7792024-03-06T20:58:18.916Z 16 Acked 1015 Read 4 Done Done Sent false
7802024-03-06T20:58:18.916Z 17 Acked 1016 Read 4 Done Done Sent false
7812024-03-06T20:58:18.916Z 18 Acked 1017 Read 10 Done Done Sent false
7822024-03-06T20:58:18.917Z 19 Acked 1018 Read 10 Done Done Sent false
7832024-03-06T20:58:18.917Z 20 Acked 1019 Flush 0 Done Done Sent false
7842024-03-06T20:58:18.917Z 21 Acked 1020 Flush 0 Done Done Sent false
7852024-03-06T20:58:18.917Z 22 Acked 1021 Write 2 Done Done Sent false
7862024-03-06T20:58:18.917Z 23 Acked 1022 Flush 0 Done Done Sent false
7872024-03-06T20:58:18.917Z 24 Acked 1023 Write 10 Done Done Sent false
7882024-03-06T20:58:18.917Z 25 Acked 1024 Read 10 Done Done Sent false
7892024-03-06T20:58:18.917Z 26 Acked 1025 Read 6 Done Done Sent false
7902024-03-06T20:58:18.917Z 27 Acked 1026 Read 1 Done Done Sent false
7912024-03-06T20:58:18.917Z 28 Acked 1027 Read 6 Done Done Sent false
7922024-03-06T20:58:18.917Z 29 Acked 1028 Read 6 Done Done Sent false
7932024-03-06T20:58:18.918Z 30 Acked 1029 Read 1 Done Done Sent false
7942024-03-06T20:58:18.918Z STATES DS:0 DS:1 DS:2 TOTAL
7952024-03-06T20:58:18.918Z New 0 0 0 0
7962024-03-06T20:58:18.918Z Sent 0 0 30 30
7972024-03-06T20:58:18.918Z Done 30 30 0 60
7982024-03-06T20:58:18.918Z Skipped 0 0 0 0
7992024-03-06T20:58:18.918Z Error 0 0 0 0
8002024-03-06T20:58:18.918ZLast Flush: 1022 1022 0
8012024-03-06T20:58:18.918ZDownstairs last five completed:
8022024-03-06T20:58:18.918ZUpstairs last five completed: 30 29 28 27 26
8032024-03-06T20:58:18.919Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T20:58:17.673726803Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1160}
8042024-03-06T20:58:18.919ZWrote out file "/tmp/test_repair_verify.data"
8052024-03-06T20:58:18.919Z
8062024-03-06T20:58:18.932ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
8072024-03-06T20:58:18.932ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8082024-03-06T20:58:18.932Z 0 000-019 1 1 1 1 1 1 F F F
8092024-03-06T20:58:18.932Z 1 020-039 2 2 2 2 2 2 F F F
8102024-03-06T20:58:18.932Z 2 040-059 1 1 1 1 1 1 F F F
8112024-03-06T20:58:18.932Z 3 060-079 2 2 2 2 2 2 F F F
8122024-03-06T20:58:18.933Z 4 080-099 1 1 1 1 1 1 F F F
8132024-03-06T20:58:18.933Z 5 100-119 1 1 1 1 1 1 F F F
8142024-03-06T20:58:18.933Z 6 120-139 1 1 1 1 1 1 F F F
8152024-03-06T20:58:18.933Z 7 140-159 1 1 1 1 1 1 F F F
8162024-03-06T20:58:18.933Z 8 160-179 3 3 3 3 3 3 F F F
8172024-03-06T20:58:18.933Z 9 180-199 3 3 3 3 3 3 F F F
8182024-03-06T20:58:18.934Z 10 200-219 1 1 1 1 1 1 F F F
8192024-03-06T20:58:18.934Z 11 220-239 6 6 3 7 7 3 F F F <---
8202024-03-06T20:58:18.934Z 12 240-259 6 6 1 4 4 1 F F F <---
8212024-03-06T20:58:18.934Z 13 260-279 3 3 3 3 3 3 F F F
8222024-03-06T20:58:18.934Z 14 280-299 6 6 2 4 4 2 F F F <---
8232024-03-06T20:58:18.934Z 15 300-319 1 1 1 1 1 1 F F F
8242024-03-06T20:58:18.934Z 16 320-339 1 1 1 1 1 1 F F F
8252024-03-06T20:58:18.934Z 17 340-359 6 6 3 5 5 3 F F F <---
8262024-03-06T20:58:18.934Z 18 360-379 3 3 3 3 3 3 F F F
8272024-03-06T20:58:18.934Z 19 380-399 3 3 3 3 3 3 F F F
8282024-03-06T20:58:18.934Z 20 400-419 3 3 3 3 3 3 F F F
8292024-03-06T20:58:18.934Z 21 420-439 3 3 3 3 3 3 T T F <---
8302024-03-06T20:58:18.934Z 22 440-459 1 1 1 1 1 1 F F F
8312024-03-06T20:58:18.935Z 23 460-479 1 1 1 1 1 1 F F F
8322024-03-06T20:58:18.935Z 24 480-499 2 2 2 2 2 2 F F F
8332024-03-06T20:58:18.935Z 25 500-519 1 1 1 1 1 1 F F F
8342024-03-06T20:58:18.935Z 26 520-539 3 3 3 3 3 3 F F F
8352024-03-06T20:58:18.935Z 27 540-559 6 6 2 4 4 2 F F F <---
8362024-03-06T20:58:18.935Z 28 560-579 2 2 2 2 2 2 F F F
8372024-03-06T20:58:18.935Z 29 580-599 1 1 1 1 1 1 F F F
8382024-03-06T20:58:18.935ZMax gen: 6, Max flush: 7
8392024-03-06T20:58:18.936ZError: Difference in extent metadata found!
8402024-03-06T20:58:18.945ZA Difference in extent metadata is expected here
8412024-03-06T20:58:18.945ZOn loop 3
8422024-03-06T20:58:18.945Z
8432024-03-06T20:58:18.945ZVerifying data now
8442024-03-06T20:58:23.503ZNo change after verify
8452024-03-06T20:58:23.503ZLoop: 3 Downstairs dump after verify (and repair):
8462024-03-06T20:58:23.517ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8472024-03-06T20:58:23.517Z 0 000-019 1 1 1 1 1 1 F F F
8482024-03-06T20:58:23.517Z 1 020-039 2 2 2 2 2 2 F F F
8492024-03-06T20:58:23.518Z 2 040-059 1 1 1 1 1 1 F F F
8502024-03-06T20:58:23.518Z 3 060-079 2 2 2 2 2 2 F F F
8512024-03-06T20:58:23.518Z 4 080-099 1 1 1 1 1 1 F F F
8522024-03-06T20:58:23.518Z 5 100-119 1 1 1 1 1 1 F F F
8532024-03-06T20:58:23.518Z 6 120-139 1 1 1 1 1 1 F F F
8542024-03-06T20:58:23.518Z 7 140-159 1 1 1 1 1 1 F F F
8552024-03-06T20:58:23.518Z 8 160-179 3 3 3 3 3 3 F F F
8562024-03-06T20:58:23.518Z 9 180-199 3 3 3 3 3 3 F F F
8572024-03-06T20:58:23.522Z 10 200-219 1 1 1 1 1 1 F F F
8582024-03-06T20:58:23.522Z 11 220-239 6 6 6 7 7 7 F F F
8592024-03-06T20:58:23.522Z 12 240-259 6 6 6 4 4 4 F F F
8602024-03-06T20:58:23.522Z 13 260-279 3 3 3 3 3 3 F F F
8612024-03-06T20:58:23.522Z 14 280-299 6 6 6 4 4 4 F F F
8622024-03-06T20:58:23.522Z 15 300-319 1 1 1 1 1 1 F F F
8632024-03-06T20:58:23.522Z 16 320-339 1 1 1 1 1 1 F F F
8642024-03-06T20:58:23.522Z 17 340-359 6 6 6 5 5 5 F F F
8652024-03-06T20:58:23.522Z 18 360-379 3 3 3 3 3 3 F F F
8662024-03-06T20:58:23.522Z 19 380-399 3 3 3 3 3 3 F F F
8672024-03-06T20:58:23.522Z 20 400-419 3 3 3 3 3 3 F F F
8682024-03-06T20:58:23.522Z 21 420-439 7 7 7 8 8 8 F F F
8692024-03-06T20:58:23.522Z 22 440-459 1 1 1 1 1 1 F F F
8702024-03-06T20:58:23.523Z 23 460-479 1 1 1 1 1 1 F F F
8712024-03-06T20:58:23.523Z 24 480-499 2 2 2 2 2 2 F F F
8722024-03-06T20:58:23.523Z 25 500-519 1 1 1 1 1 1 F F F
8732024-03-06T20:58:23.523Z 26 520-539 3 3 3 3 3 3 F F F
8742024-03-06T20:58:23.523Z 27 540-559 6 6 6 4 4 4 F F F
8752024-03-06T20:58:23.523Z 28 560-579 2 2 2 2 2 2 F F F
8762024-03-06T20:58:23.523Z 29 580-599 1 1 1 1 1 1 F F F
8772024-03-06T20:58:23.523ZMax gen: 7, Max flush: 8
8782024-03-06T20:58:23.523Z
8792024-03-06T20:58:23.523ZBegin loop 4
8802024-03-06T20:58:23.523ZDownstairs to restart: 2
8812024-03-06T20:58:23.539Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.312879177Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168}
8822024-03-06T20:58:23.539Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.313196035Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168}
8832024-03-06T20:58:23.539Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.313221913Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168}
8842024-03-06T20:58:23.539ZCrucible runtime is spawned
8852024-03-06T20:58:23.556Z{"msg":"Crucible 10c9b60b-4205-4088-b113-b247af91d829 has session id: e8e10c12-d8ba-4491-ad68-03ace8ba77af","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.313260631Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
8862024-03-06T20:58:23.556Z{"msg":"Upstairs opts: Upstairs UUID: 10c9b60b-4205-4088-b113-b247af91d829, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.313311627Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
8872024-03-06T20:58:23.557Z{"msg":"Crucible stats registered with UUID: 10c9b60b-4205-4088-b113-b247af91d829","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.313339485Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
8882024-03-06T20:58:23.557Z{"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.313361864Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168}
8892024-03-06T20:58:23.557Z{"msg":"10c9b60b-4205-4088-b113-b247af91d829 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.313381982Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
8902024-03-06T20:58:23.557Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.31341424Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"client":"0","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
8912024-03-06T20:58:23.557Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.313440958Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"client":"0","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
8922024-03-06T20:58:23.557Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.313474816Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"client":"1","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
8932024-03-06T20:58:23.557Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.313500504Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"client":"1","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
8942024-03-06T20:58:23.557Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.313532972Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"client":"2","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
8952024-03-06T20:58:23.557Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.31356784Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"client":"2","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
8962024-03-06T20:58:23.557Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.313606877Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"":"io task","client":"0","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
8972024-03-06T20:58:23.557Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.313651694Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"":"io task","client":"1","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
8982024-03-06T20:58:23.557Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.313679162Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"":"io task","client":"2","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
8992024-03-06T20:58:23.557Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.31370533Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"":"io task","client":"1","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9002024-03-06T20:58:23.557Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.313812803Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"":"io task","client":"0","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9012024-03-06T20:58:23.557Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.313870229Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"":"io task","client":"2","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9022024-03-06T20:58:23.557Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.314386463Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"client":"0","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9032024-03-06T20:58:23.557Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.31442369Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"client":"0","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9042024-03-06T20:58:23.557Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.314439899Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"client":"1","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9052024-03-06T20:58:23.558Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.314454168Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"client":"1","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9062024-03-06T20:58:23.558Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.314480247Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"client":"2","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9072024-03-06T20:58:23.558Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.314494166Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"client":"2","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9082024-03-06T20:58:23.558Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.315095274Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"client":"1","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9092024-03-06T20:58:23.558Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.315132542Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"client":"2","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9102024-03-06T20:58:23.558Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.31514856Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"client":"0","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9112024-03-06T20:58:23.558Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.315672375Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"client":"1","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9122024-03-06T20:58:23.558Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.315718901Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"client":"1","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9132024-03-06T20:58:23.558Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.31574193Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9142024-03-06T20:58:23.558Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.315772077Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"client":"2","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9152024-03-06T20:58:23.558Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.315788247Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"client":"2","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9162024-03-06T20:58:23.558Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.315819434Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9172024-03-06T20:58:23.558Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.315833983Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"client":"0","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9182024-03-06T20:58:23.558Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.315863611Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"client":"0","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9192024-03-06T20:58:23.558Z{"msg":"[0]R flush_numbers[0..12]: [1, 2, 1, 2, 1, 1, 1, 1, 3, 3, 1, 7]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.31587921Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"":"downstairs","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9202024-03-06T20:58:23.558Z{"msg":"[0]R generation[0..12]: [1, 2, 1, 2, 1, 1, 1, 1, 3, 3, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.315898379Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"":"downstairs","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9212024-03-06T20:58:23.558Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.315927757Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"":"downstairs","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9222024-03-06T20:58:23.558Z{"msg":"[1]R flush_numbers[0..12]: [1, 2, 1, 2, 1, 1, 1, 1, 3, 3, 1, 7]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.315944846Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"":"downstairs","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9232024-03-06T20:58:23.558Z{"msg":"[1]R generation[0..12]: [1, 2, 1, 2, 1, 1, 1, 1, 3, 3, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.315979033Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"":"downstairs","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9242024-03-06T20:58:23.558Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.316003362Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"":"downstairs","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9252024-03-06T20:58:23.558Z{"msg":"[2]R flush_numbers[0..12]: [1, 2, 1, 2, 1, 1, 1, 1, 3, 3, 1, 7]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
9262024-03-06T20:58:23.558Z,"time":"2024-03-06T20:58:22.31602491Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"":"downstairs","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9272024-03-06T20:58:23.559Z{"msg":"[2]R generation[0..12]: [1, 2, 1, 2, 1, 1, 1, 1, 3, 3, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.316062308Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"":"downstairs","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9282024-03-06T20:58:23.559Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.316081296Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"":"downstairs","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9292024-03-06T20:58:23.559Z{"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.316114084Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"":"downstairs","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9302024-03-06T20:58:23.559Z{"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.316138932Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"":"downstairs","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9312024-03-06T20:58:23.559Z{"msg":"Next flush: 9","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.31617166Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"":"downstairs","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9322024-03-06T20:58:23.559Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.316198168Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"":"downstairs","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9332024-03-06T20:58:23.559Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.316220817Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9342024-03-06T20:58:23.559Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.316234266Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"client":"0","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9352024-03-06T20:58:23.559Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.316250775Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"client":"1","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9362024-03-06T20:58:23.559Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.316285072Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"client":"2","":"downstairs","session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9372024-03-06T20:58:23.559Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.31631665Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9382024-03-06T20:58:23.559Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.316330869Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9392024-03-06T20:58:23.559Z{"msg":"10c9b60b-4205-4088-b113-b247af91d829 is now active with session: e8e10c12-d8ba-4491-ad68-03ace8ba77af","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.316357917Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9402024-03-06T20:58:23.559Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.316392155Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168,"session_id":"e8e10c12-d8ba-4491-ad68-03ace8ba77af"}
9412024-03-06T20:58:23.559Z{"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:22.316409954Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168}
9422024-03-06T20:58:23.559ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
9432024-03-06T20:58:23.559ZLoading write count information from file "/tmp/test_repair_verify.data"
9442024-03-06T20:58:23.559ZRun Repair workload
9452024-03-06T20:58:23.559Z01/30 Write block 457 len 40960 data: 2 2 2 2 2 2 2 2 2 2
9462024-03-06T20:58:23.570Z02/30 Write block 460 len 40960 data: 3 3 3 3 3 3 3 2 2 2
9472024-03-06T20:58:23.570Z03/30 Read block 570 len 24576
9482024-03-06T20:58:23.570Z04/30 Read block 354 len 16384
9492024-03-06T20:58:23.570Z05/30 Write block 217 len 8192 data: 2 2
9502024-03-06T20:58:23.570Z06/30 Write block 23 len 12288 data: 2 2 2
9512024-03-06T20:58:23.570Z07/30 Read block 23 len 40960
9522024-03-06T20:58:23.570Z08/30 Read block 289 len 32768
9532024-03-06T20:58:23.570Z09/30 Read block 252 len 24576
9542024-03-06T20:58:23.571Z10/30 Read block 431 len 40960
9552024-03-06T20:58:23.571Z11/30 Read block 582 len 12288
9562024-03-06T20:58:23.571Z12/30 Read block 502 len 40960
9572024-03-06T20:58:23.571Z13/30 Write block 286 len 40960 data: 4 4 4 4 4 4 3 2 2 2
9582024-03-06T20:58:23.571Z14/30 Write block 535 len 36864 data: 2 2 2 2 2 4 4 4 4
9592024-03-06T20:58:23.571Z15/30 Flush
9602024-03-06T20:58:24.027Z16/30 Write block 473 len 28672 data: 2 2 2 2 2 2 2
9612024-03-06T20:58:24.027Z17/30 Write block 369 len 8192 data: 2 3
9622024-03-06T20:58:24.028Z18/30 Read block 130 len 40960
9632024-03-06T20:58:24.028Z19/30 Write block 405 len 28672 data: 2 2 2 2 2 2 2
9642024-03-06T20:58:24.028Z20/30 Write block 524 len 40960 data: 3 3 3 3 3 3 3 3 3 3
9652024-03-06T20:58:24.028Z21/30 Read block 68 len 8192
9662024-03-06T20:58:24.028Z22/30 Read block 193 len 20480
9672024-03-06T20:58:24.028Z23/30 Write block 179 len 16384 data: 3 3 3 3
9682024-03-06T20:58:24.028Z24/30 Read block 489 len 28672
9692024-03-06T20:58:24.028Z25/30 Write block 349 len 24576 data: 2 2 2 3 3 3
9702024-03-06T20:58:24.029Z26/30 Read block 544 len 16384
9712024-03-06T20:58:24.029Z27/30 Read block 199 len 8192
9722024-03-06T20:58:24.029Z28/30 Read block 284 len 36864
9732024-03-06T20:58:24.029Z29/30 Read block 46 len 12288
9742024-03-06T20:58:24.029Z30/30 Write block 78 len 24576 data: 2 2 2 2 2 2
9752024-03-06T20:58:24.029Z----------------------------------------------------------------
9762024-03-06T20:58:24.029Z Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:30
9772024-03-06T20:58:24.029ZStates: Active Active Active
9782024-03-06T20:58:24.030ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
9792024-03-06T20:58:24.030Z 1 Acked 1000 Write 10 Done Done Done false
9802024-03-06T20:58:24.030Z 2 Acked 1001 Write 10 Done Done Done false
9812024-03-06T20:58:24.030Z 3 Acked 1002 Read 6 Done Done Done false
9822024-03-06T20:58:24.030Z 4 Acked 1003 Read 4 Done Done Sent false
9832024-03-06T20:58:24.030Z 5 Acked 1004 Write 2 Done Done Sent false
9842024-03-06T20:58:24.030Z 6 Acked 1005 Write 3 Done Done Sent false
9852024-03-06T20:58:24.030Z 7 Acked 1006 Read 10 Done Done Sent false
9862024-03-06T20:58:24.031Z 8 Acked 1007 Read 8 Done Done Sent false
9872024-03-06T20:58:24.031Z 9 Acked 1008 Read 6 Done Done Sent false
9882024-03-06T20:58:24.031Z 10 Acked 1009 Read 10 Done Done Sent false
9892024-03-06T20:58:24.031Z 11 Acked 1010 Read 3 Done Done Sent false
9902024-03-06T20:58:24.031Z 12 Acked 1011 Read 10 Done Done Sent false
9912024-03-06T20:58:24.031Z 13 Acked 1012 Write 10 Done Done Sent false
9922024-03-06T20:58:24.031Z 14 Acked 1013 Write 9 Done Done Sent false
9932024-03-06T20:58:24.032Z 15 Acked 1014 Flush 0 Done Done Sent false
9942024-03-06T20:58:24.032Z 16 Acked 1015 Write 7 Done Done Sent false
9952024-03-06T20:58:24.032Z 17 Acked 1016 Write 2 Done Done Sent false
9962024-03-06T20:58:24.032Z 18 Acked 1017 Read 10 Done Done Sent false
9972024-03-06T20:58:24.032Z 19 Acked 1018 Write 7 Done Done Sent false
9982024-03-06T20:58:24.032Z 20 Acked 1019 Write 10 Done Done Sent false
9992024-03-06T20:58:24.032Z 21 Acked 1020 Read 2 Done Done Sent false
10002024-03-06T20:58:24.032Z 22 Acked 1021 Read 5 Done Done Sent false
10012024-03-06T20:58:24.033Z 23 Acked 1022 Write 4 Done Done Sent false
10022024-03-06T20:58:24.033Z 24 Acked 1023 Read 7 Done Done Sent false
10032024-03-06T20:58:24.033Z 25 Acked 1024 Write 6 Done Done Sent false
10042024-03-06T20:58:24.033Z 26 Acked 1025 Read 4 Done Done Sent false
10052024-03-06T20:58:24.033Z 27 Acked 1026 Read 2 Done Done Sent false
10062024-03-06T20:58:24.033Z 28 Acked 1027 Read 9 Done Done Sent false
10072024-03-06T20:58:24.033Z 29 Acked 1028 Read 3 Done Done Sent false
10082024-03-06T20:58:24.034Z 30 Acked 1029 Write 6 Sent Sent Sent false
10092024-03-06T20:58:24.034Z STATES DS:0 DS:1 DS:2 TOTAL
10102024-03-06T20:58:24.034Z New 0 0 0 0
10112024-03-06T20:58:24.034Z Sent 1 1 27 29
10122024-03-06T20:58:24.034Z Done 29 29 3 61
10132024-03-06T20:58:24.034Z Skipped 0 0 0 0
10142024-03-06T20:58:24.035Z Error 0 0 0 0
10152024-03-06T20:58:24.035ZLast Flush: 1014 1014 0
10162024-03-06T20:58:24.035ZDownstairs last five completed:
10172024-03-06T20:58:24.035ZUpstairs last five completed: 30 29 28 27 26
10182024-03-06T20:58:24.035Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T20:58:22.375429816Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1168}
10192024-03-06T20:58:24.035ZWrote out file "/tmp/test_repair_verify.data"
10202024-03-06T20:58:24.035Z
10212024-03-06T20:58:24.035ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
10222024-03-06T20:58:24.035ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10232024-03-06T20:58:24.035Z 0 000-019 1 1 1 1 1 1 F F F
10242024-03-06T20:58:24.035Z 1 020-039 8 8 2 9 9 2 F F F <---
10252024-03-06T20:58:24.036Z 2 040-059 1 1 1 1 1 1 F F F
10262024-03-06T20:58:24.036Z 3 060-079 2 2 2 2 2 2 T T F <---
10272024-03-06T20:58:24.036Z 4 080-099 1 1 1 1 1 1 T T F <---
10282024-03-06T20:58:24.036Z 5 100-119 1 1 1 1 1 1 F F F
10292024-03-06T20:58:24.036Z 6 120-139 1 1 1 1 1 1 F F F
10302024-03-06T20:58:24.036Z 7 140-159 1 1 1 1 1 1 F F F
10312024-03-06T20:58:24.036Z 8 160-179 3 3 3 3 3 3 T T F <---
10322024-03-06T20:58:24.037Z 9 180-199 3 3 3 3 3 3 T T F <---
10332024-03-06T20:58:24.037Z 10 200-219 8 8 1 9 9 1 F F F <---
10342024-03-06T20:58:24.037Z 11 220-239 6 6 6 7 7 7 F F F
10352024-03-06T20:58:24.037Z 12 240-259 6 6 6 4 4 4 F F F
10362024-03-06T20:58:24.037Z 13 260-279 3 3 3 3 3 3 F F F
10372024-03-06T20:58:24.037Z 14 280-299 8 8 6 9 9 4 F F F <---
10382024-03-06T20:58:24.037Z 15 300-319 1 1 1 1 1 1 F F F
10392024-03-06T20:58:24.038Z 16 320-339 1 1 1 1 1 1 F F F
10402024-03-06T20:58:24.038Z 17 340-359 6 6 6 5 5 5 T T F <---
10412024-03-06T20:58:24.038Z 18 360-379 3 3 3 3 3 3 T T F <---
10422024-03-06T20:58:24.038Z 19 380-399 3 3 3 3 3 3 F F F
10432024-03-06T20:58:24.038Z 20 400-419 3 3 3 3 3 3 T T F <---
10442024-03-06T20:58:24.038Z 21 420-439 7 7 7 8 8 8 F F F
10452024-03-06T20:58:24.038Z 22 440-459 8 8 1 9 9 1 F F T <---
10462024-03-06T20:58:24.038Z 23 460-479 8 8 1 9 9 1 T T T <---
10472024-03-06T20:58:24.038Z 24 480-499 2 2 2 2 2 2 F F F
10482024-03-06T20:58:24.039Z 25 500-519 1 1 1 1 1 1 F F F
10492024-03-06T20:58:24.039Z 26 520-539 8 8 3 9 9 3 T T F <---
10502024-03-06T20:58:24.039Z 27 540-559 8 8 6 9 9 4 F F F <---
10512024-03-06T20:58:24.039Z 28 560-579 2 2 2 2 2 2 F F F
10522024-03-06T20:58:24.039Z 29 580-599 1 1 1 1 1 1 F F F
10532024-03-06T20:58:24.039ZMax gen: 8, Max flush: 9
10542024-03-06T20:58:24.039ZError: Difference in extent metadata found!
10552024-03-06T20:58:24.040ZA Difference in extent metadata is expected here
10562024-03-06T20:58:24.040ZOn loop 4
10572024-03-06T20:58:24.040Z
10582024-03-06T20:58:24.040ZVerifying data now
10592024-03-06T20:58:29.434ZNo change after verify
10602024-03-06T20:58:29.435ZLoop: 4 Downstairs dump after verify (and repair):
10612024-03-06T20:58:29.454ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10622024-03-06T20:58:29.454Z 0 000-019 1 1 1 1 1 1 F F F
10632024-03-06T20:58:29.454Z 1 020-039 8 8 8 9 9 9 F F F
10642024-03-06T20:58:29.454Z 2 040-059 1 1 1 1 1 1 F F F
10652024-03-06T20:58:29.454Z 3 060-079 9 9 9 10 10 10 F F F
10662024-03-06T20:58:29.454Z 4 080-099 9 9 9 10 10 10 F F F
10672024-03-06T20:58:29.454Z 5 100-119 1 1 1 1 1 1 F F F
10682024-03-06T20:58:29.455Z 6 120-139 1 1 1 1 1 1 F F F
10692024-03-06T20:58:29.455Z 7 140-159 1 1 1 1 1 1 F F F
10702024-03-06T20:58:29.455Z 8 160-179 9 9 9 10 10 10 F F F
10712024-03-06T20:58:29.455Z 9 180-199 9 9 9 10 10 10 F F F
10722024-03-06T20:58:29.462Z 10 200-219 8 8 8 9 9 9 F F F
10732024-03-06T20:58:29.462Z 11 220-239 6 6 6 7 7 7 F F F
10742024-03-06T20:58:29.463Z 12 240-259 6 6 6 4 4 4 F F F
10752024-03-06T20:58:29.463Z 13 260-279 3 3 3 3 3 3 F F F
10762024-03-06T20:58:29.463Z 14 280-299 8 8 8 9 9 9 F F F
10772024-03-06T20:58:29.463Z 15 300-319 1 1 1 1 1 1 F F F
10782024-03-06T20:58:29.463Z 16 320-339 1 1 1 1 1 1 F F F
10792024-03-06T20:58:29.463Z 17 340-359 9 9 9 10 10 10 F F F
10802024-03-06T20:58:29.463Z 18 360-379 9 9 9 10 10 10 F F F
10812024-03-06T20:58:29.463Z 19 380-399 3 3 3 3 3 3 F F F
10822024-03-06T20:58:29.463Z 20 400-419 9 9 9 10 10 10 F F F
10832024-03-06T20:58:29.463Z 21 420-439 7 7 7 8 8 8 F F F
10842024-03-06T20:58:29.463Z 22 440-459 8 8 8 9 9 9 F F F
10852024-03-06T20:58:29.463Z 23 460-479 9 9 9 10 10 10 F F F
10862024-03-06T20:58:29.463Z 24 480-499 2 2 2 2 2 2 F F F
10872024-03-06T20:58:29.463Z 25 500-519 1 1 1 1 1 1 F F F
10882024-03-06T20:58:29.463Z 26 520-539 9 9 9 10 10 10 F F F
10892024-03-06T20:58:29.463Z 27 540-559 8 8 8 9 9 9 F F F
10902024-03-06T20:58:29.463Z 28 560-579 2 2 2 2 2 2 F F F
10912024-03-06T20:58:29.463Z 29 580-599 1 1 1 1 1 1 F F F
10922024-03-06T20:58:29.463ZMax gen: 9, Max flush: 10
10932024-03-06T20:58:29.464Z
10942024-03-06T20:58:29.464ZBegin loop 5
10952024-03-06T20:58:29.464ZDownstairs to restart: 1
10962024-03-06T20:58:29.495Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.24722181Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176}
10972024-03-06T20:58:29.495Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.247522899Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176}
10982024-03-06T20:58:29.495ZCrucible runtime is spawned
10992024-03-06T20:58:29.531Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.247576655Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176}
11002024-03-06T20:58:29.531Z{"msg":"Crucible 97609499-6a97-4403-9073-cf173179f071 has session id: 2f9880f9-8031-4da8-9481-37603bb996b7","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.247602033Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11012024-03-06T20:58:29.531Z{"msg":"Upstairs opts: Upstairs UUID: 97609499-6a97-4403-9073-cf173179f071, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.247641051Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11022024-03-06T20:58:29.531Z{"msg":"Crucible stats registered with UUID: 97609499-6a97-4403-9073-cf173179f071","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.247663959Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11032024-03-06T20:58:29.532Z{"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.247692717Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176}
11042024-03-06T20:58:29.532Z{"msg":"97609499-6a97-4403-9073-cf173179f071 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.247815099Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11052024-03-06T20:58:29.532Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.247871185Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"client":"0","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11062024-03-06T20:58:29.532Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.247964628Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"client":"0","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11072024-03-06T20:58:29.532Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.248031474Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"client":"1","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11082024-03-06T20:58:29.532Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.248057672Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"client":"1","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11092024-03-06T20:58:29.532Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.24808306Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"client":"2","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11102024-03-06T20:58:29.532Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.248195273Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"client":"2","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11112024-03-06T20:58:29.532Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.24822909Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"":"io task","client":"0","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11122024-03-06T20:58:29.533Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.248259618Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"":"io task","client":"1","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11132024-03-06T20:58:29.533Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.248308765Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"":"io task","client":"2","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11142024-03-06T20:58:29.533Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.248337263Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"":"io task","client":"2","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11152024-03-06T20:58:29.533Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.248361811Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"":"io task","client":"1","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11162024-03-06T20:58:29.533Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.24838615Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"":"io task","client":"0","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11172024-03-06T20:58:29.533Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.248906464Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"client":"0","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11182024-03-06T20:58:29.533Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.248951Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"client":"0","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11192024-03-06T20:58:29.533Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.248986868Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"client":"2","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11202024-03-06T20:58:29.533Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.249009976Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"client":"2","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11212024-03-06T20:58:29.533Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.249473704Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"client":"2","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11222024-03-06T20:58:29.533Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.249514172Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"client":"0","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11232024-03-06T20:58:29.534Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.249591136Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"client":"1","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11242024-03-06T20:58:29.534Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.249672251Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"client":"1","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11252024-03-06T20:58:29.534Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.250111491Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"client":"2","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11262024-03-06T20:58:29.534Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.250161297Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"client":"2","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11272024-03-06T20:58:29.534Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.250202564Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11282024-03-06T20:58:29.534Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.250237542Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"client":"0","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11292024-03-06T20:58:29.534Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.250275179Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"client":"0","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11302024-03-06T20:58:29.534Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.250297998Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11312024-03-06T20:58:29.534Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.250435688Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"client":"1","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11322024-03-06T20:58:29.535Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.25084273Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"client":"1","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11332024-03-06T20:58:29.535Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.250888847Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"client":"1","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11342024-03-06T20:58:29.535Z{"msg":"[0]R flush_numbers[0..12]: [1, 9, 1, 10, 10, 1, 1, 1, 10, 10, 9, 7]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.250914245Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"":"downstairs","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11352024-03-06T20:58:29.535Z{"msg":"[0]R generation[0..12]: [1, 8, 1, 9, 9, 1, 1, 1, 9, 9, 8, 6]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.250948753Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"":"downstairs","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11362024-03-06T20:58:29.535Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.25098458Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"":"downstairs","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11372024-03-06T20:58:29.535Z{"msg":"[1]R flush_numbers[0..12]: [1, 9, 1, 10, 10, 1, 1, 1, 10, 10, 9, 7]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
11382024-03-06T20:58:29.535Z,"time":"2024-03-06T20:58:28.251037977Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"":"downstairs","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11392024-03-06T20:58:29.535Z{"msg":"[1]R generation[0..12]: [1, 8, 1, 9, 9, 1, 1, 1, 9, 9, 8, 6]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.251065905Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"":"downstairs","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11402024-03-06T20:58:29.536Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.251103722Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"":"downstairs","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11412024-03-06T20:58:29.536Z{"msg":"[2]R flush_numbers[0..12]: [1, 9, 1, 10, 10, 1, 1, 1, 10, 10, 9, 7]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.25113966Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"":"downstairs","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11422024-03-06T20:58:29.536Z{"msg":"[2]R generation[0..12]: [1, 8, 1, 9, 9, 1, 1, 1, 9, 9, 8, 6]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.251198906Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"":"downstairs","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11432024-03-06T20:58:29.536Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.251230734Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"":"downstairs","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11442024-03-06T20:58:29.536Z{"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.251265811Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"":"downstairs","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11452024-03-06T20:58:29.536Z{"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.251301709Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"":"downstairs","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11462024-03-06T20:58:29.536Z{"msg":"Next flush: 11","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.251338026Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"":"downstairs","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11472024-03-06T20:58:29.536Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.251374353Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"":"downstairs","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
11482024-03-06T20:58:29.536Z"}
11492024-03-06T20:58:29.536Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.251414551Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11502024-03-06T20:58:29.537Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.251449058Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"client":"0","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11512024-03-06T20:58:29.537Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.251465537Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"client":"1","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11522024-03-06T20:58:29.537Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.251487926Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"client":"2","":"downstairs","session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11532024-03-06T20:58:29.537Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.251502035Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11542024-03-06T20:58:29.537Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.251514344Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11552024-03-06T20:58:29.537Z{"msg":"97609499-6a97-4403-9073-cf173179f071 is now active with session: 2f9880f9-8031-4da8-9481-37603bb996b7","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.251527313Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11562024-03-06T20:58:29.537Z{"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.251546652Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176}
11572024-03-06T20:58:29.537Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:28.251557951Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176,"session_id":"2f9880f9-8031-4da8-9481-37603bb996b7"}
11582024-03-06T20:58:29.537ZLoading write count information from file "/tmp/test_repair_verify.data"
11592024-03-06T20:58:29.537ZRun Repair workload
11602024-03-06T20:58:29.538Z01/30 Write block 487 len 32768 data: 3 3 3 2 2 2 2 2
11612024-03-06T20:58:29.538Z02/30 Read block 250 len 16384
11622024-03-06T20:58:29.538Z03/30 Flush
11632024-03-06T20:58:29.538Z04/30 Write block 72 len 16384 data: 3 2 2 2
11642024-03-06T20:58:29.538Z05/30 Read block 40 len 8192
11652024-03-06T20:58:29.538Z06/30 Write block 240 len 8192 data: 2 3
11662024-03-06T20:58:29.538Z07/30 Write block 158 len 24576 data: 2 2 2 2 2 2
11672024-03-06T20:58:29.538Z08/30 Read block 13 len 24576
11682024-03-06T20:58:29.538Z09/30 Read block 155 len 8192
11692024-03-06T20:58:29.538Z10/30 Read block 58 len 20480
11702024-03-06T20:58:29.539Z11/30 Write block 261 len 40960 data: 2 2 2 2 3 3 3 3 2 2
11712024-03-06T20:58:29.539Z12/30 Write block 305 len 32768 data: 2 2 2 2 2 2 2 2
11722024-03-06T20:58:29.539Z13/30 Write block 341 len 20480 data: 3 4 4 4 3
11732024-03-06T20:58:29.539Z14/30 Read block 320 len 36864
11742024-03-06T20:58:29.539Z15/30 Write block 215 len 8192 data: 2 2
11752024-03-06T20:58:29.539Z16/30 Write block 421 len 4096 data: 3
11762024-03-06T20:58:29.539Z17/30 Write block 590 len 32768 data: 2 2 2 2 2 2 2 2
11772024-03-06T20:58:29.539Z18/30 Flush
11782024-03-06T20:58:29.539Z19/30 Write block 395 len 16384 data: 2 2 2 2
11792024-03-06T20:58:29.539Z20/30 Read block 244 len 4096
11802024-03-06T20:58:29.539Z21/30 Read block 577 len 12288
11812024-03-06T20:58:29.540Z22/30 Read block 356 len 36864
11822024-03-06T20:58:29.551Z23/30 Read block 319 len 36864
11832024-03-06T20:58:29.551Z24/30 Read block 550 len 28672
11842024-03-06T20:58:29.551Z25/30 Write block 11 len 16384 data: 2 2 2 2
11852024-03-06T20:58:29.551Z26/30 Read block 479 len 36864
11862024-03-06T20:58:29.551Z27/30 Read block 213 len 36864
11872024-03-06T20:58:29.551Z28/30 Write block 444 len 12288 data: 2 2 2
11882024-03-06T20:58:29.551Z29/30 Read block 583 len 16384
11892024-03-06T20:58:29.552Z30/30 Write block 421 len 40960 data: 4 3 2 2 2 3 3 3 3 3
11902024-03-06T20:58:29.552Z----------------------------------------------------------------
11912024-03-06T20:58:29.552Z Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:30
11922024-03-06T20:58:29.552ZStates: Active Active Active
11932024-03-06T20:58:29.552ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
11942024-03-06T20:58:29.552Z 1 Acked 1000 Write 8 Done Sent Done false
11952024-03-06T20:58:29.552Z 2 Acked 1001 Read 4 Done Sent Done false
11962024-03-06T20:58:29.552Z 3 Acked 1002 Flush 0 Done Sent Done false
11972024-03-06T20:58:29.552Z 4 Acked 1003 Write 4 Done Sent Done false
11982024-03-06T20:58:29.552Z 5 Acked 1004 Read 2 Done Sent Done false
11992024-03-06T20:58:29.552Z 6 Acked 1005 Write 2 Done Sent Done false
12002024-03-06T20:58:29.552Z 7 Acked 1006 Write 6 Done Sent Done false
12012024-03-06T20:58:29.552Z 8 Acked 1007 Read 6 Done Sent Done false
12022024-03-06T20:58:29.552Z 9 Acked 1008 Read 2 Done Sent Done false
12032024-03-06T20:58:29.552Z 10 Acked 1009 Read 5 Done Sent Done false
12042024-03-06T20:58:29.552Z 11 Acked 1010 Write 10 Done Sent Done false
12052024-03-06T20:58:29.552Z 12 Acked 1011 Write 8 Done Sent Done false
12062024-03-06T20:58:29.552Z 13 Acked 1012 Write 5 Done Sent Done false
12072024-03-06T20:58:29.552Z 14 Acked 1013 Read 9 Done Sent Done false
12082024-03-06T20:58:29.552Z 15 Acked 1014 Write 2 Done Sent Done false
12092024-03-06T20:58:29.552Z 16 Acked 1015 Write 1 Done Sent Done false
12102024-03-06T20:58:29.552Z 17 Acked 1016 Write 8 Done Sent Done false
12112024-03-06T20:58:29.552Z 18 Acked 1017 Flush 0 Done Sent Done false
12122024-03-06T20:58:29.553Z 19 Acked 1018 Write 4 Done Sent Done false
12132024-03-06T20:58:29.553Z 20 Acked 1019 Read 1 Done Sent Done false
12142024-03-06T20:58:29.553Z 21 Acked 1020 Read 3 Done Sent Done false
12152024-03-06T20:58:29.553Z 22 Acked 1021 Read 9 Done Sent Done false
12162024-03-06T20:58:29.553Z 23 Acked 1022 Read 9 Done Sent Done false
12172024-03-06T20:58:29.553Z 24 Acked 1023 Read 7 Done Sent Done false
12182024-03-06T20:58:29.553Z 25 Acked 1024 Write 4 Done Sent Done false
12192024-03-06T20:58:29.553Z 26 Acked 1025 Read 9 Done Sent Done false
12202024-03-06T20:58:29.553Z 27 Acked 1026 Read 9 Done Sent Done false
12212024-03-06T20:58:29.553Z 28 Acked 1027 Write 3 Done Sent Done false
12222024-03-06T20:58:29.553Z 29 Acked 1028 Read 4 Done Sent Done false
12232024-03-06T20:58:29.553Z 30 Acked 1029 Write 10 Sent Sent Sent false
12242024-03-06T20:58:29.553Z STATES DS:0 DS:1 DS:2 TOTAL
12252024-03-06T20:58:29.553Z New 0 0 0 0
12262024-03-06T20:58:29.553Z Sent 1 30 1 32
12272024-03-06T20:58:29.553Z Done 29 0 29 58
12282024-03-06T20:58:29.553Z Skipped 0 0 0 0
12292024-03-06T20:58:29.553Z Error 0 0 0 0
12302024-03-06T20:58:29.553ZLast Flush: 1017 0 1017
12312024-03-06T20:58:29.553ZDownstairs last five completed:
12322024-03-06T20:58:29.553ZUpstairs last five completed: 30 29 28 27 26
12332024-03-06T20:58:29.553Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T20:58:28.284988867Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1176}
12342024-03-06T20:58:29.554ZWrote out file "/tmp/test_repair_verify.data"
12352024-03-06T20:58:29.554Z
12362024-03-06T20:58:29.554ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
12372024-03-06T20:58:29.554ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12382024-03-06T20:58:29.554Z 0 000-019 1 1 1 1 1 1 T F T <---
12392024-03-06T20:58:29.554Z 1 020-039 8 8 8 9 9 9 F F F
12402024-03-06T20:58:29.554Z 2 040-059 1 1 1 1 1 1 F F F
12412024-03-06T20:58:29.554Z 3 060-079 10 9 10 12 10 12 F F F <---
12422024-03-06T20:58:29.554Z 4 080-099 9 9 9 10 10 10 F F F
12432024-03-06T20:58:29.554Z 5 100-119 1 1 1 1 1 1 F F F
12442024-03-06T20:58:29.554Z 6 120-139 1 1 1 1 1 1 F F F
12452024-03-06T20:58:29.554Z 7 140-159 10 1 10 12 1 12 F F F <---
12462024-03-06T20:58:29.554Z 8 160-179 10 9 10 12 10 12 F F F <---
12472024-03-06T20:58:29.554Z 9 180-199 9 9 9 10 10 10 F F F
12482024-03-06T20:58:29.554Z 10 200-219 10 8 10 12 9 12 F F F <---
12492024-03-06T20:58:29.554Z 11 220-239 6 6 6 7 7 7 F F F
12502024-03-06T20:58:29.554Z 12 240-259 10 6 10 12 4 12 F F F <---
12512024-03-06T20:58:29.554Z 13 260-279 10 3 10 12 3 12 F F F <---
12522024-03-06T20:58:29.554Z 14 280-299 8 8 8 9 9 9 F F F
12532024-03-06T20:58:29.554Z 15 300-319 10 1 10 12 1 12 F F F <---
12542024-03-06T20:58:29.554Z 16 320-339 1 1 1 1 1 1 F F F
12552024-03-06T20:58:29.554Z 17 340-359 10 9 10 12 10 12 F F F <---
12562024-03-06T20:58:29.554Z 18 360-379 9 9 9 10 10 10 F F F
12572024-03-06T20:58:29.555Z 19 380-399 3 3 3 3 3 3 T F T <---
12582024-03-06T20:58:29.555Z 20 400-419 9 9 9 10 10 10 F F F
12592024-03-06T20:58:29.555Z 21 420-439 10 7 10 12 8 12 T F F <---
12602024-03-06T20:58:29.555Z 22 440-459 8 8 8 9 9 9 T F T <---
12612024-03-06T20:58:29.555Z 23 460-479 9 9 9 10 10 10 F F F
12622024-03-06T20:58:29.555Z 24 480-499 10 2 10 11 2 11 F F F <---
12632024-03-06T20:58:29.555Z 25 500-519 1 1 1 1 1 1 F F F
12642024-03-06T20:58:29.555Z 26 520-539 9 9 9 10 10 10 F F F
12652024-03-06T20:58:29.555Z 27 540-559 8 8 8 9 9 9 F F F
12662024-03-06T20:58:29.555Z 28 560-579 2 2 2 2 2 2 F F F
12672024-03-06T20:58:29.555Z 29 580-599 10 1 10 12 1 12 F F F <---
12682024-03-06T20:58:29.555ZMax gen: 10, Max flush: 12
12692024-03-06T20:58:29.555ZError: Difference in extent metadata found!
12702024-03-06T20:58:29.555ZA Difference in extent metadata is expected here
12712024-03-06T20:58:29.555ZOn loop 5
12722024-03-06T20:58:29.555Z
12732024-03-06T20:58:29.555ZVerifying data now
12742024-03-06T20:58:34.253ZNo change after verify
12752024-03-06T20:58:34.253ZLoop: 5 Downstairs dump after verify (and repair):
12762024-03-06T20:58:34.267ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12772024-03-06T20:58:34.267Z 0 000-019 11 11 11 13 13 13 F F F
12782024-03-06T20:58:34.267Z 1 020-039 8 8 8 9 9 9 F F F
12792024-03-06T20:58:34.267Z 2 040-059 1 1 1 1 1 1 F F F
12802024-03-06T20:58:34.268Z 3 060-079 10 10 10 12 12 12 F F F
12812024-03-06T20:58:34.268Z 4 080-099 9 9 9 10 10 10 F F F
12822024-03-06T20:58:34.268Z 5 100-119 1 1 1 1 1 1 F F F
12832024-03-06T20:58:34.268Z 6 120-139 1 1 1 1 1 1 F F F
12842024-03-06T20:58:34.268Z 7 140-159 10 10 10 12 12 12 F F F
12852024-03-06T20:58:34.268Z 8 160-179 10 10 10 12 12 12 F F F
12862024-03-06T20:58:34.268Z 9 180-199 9 9 9 10 10 10 F F F
12872024-03-06T20:58:34.278Z 10 200-219 10 10 10 12 12 12 F F F
12882024-03-06T20:58:34.278Z 11 220-239 6 6 6 7 7 7 F F F
12892024-03-06T20:58:34.278Z 12 240-259 10 10 10 12 12 12 F F F
12902024-03-06T20:58:34.278Z 13 260-279 10 10 10 12 12 12 F F F
12912024-03-06T20:58:34.278Z 14 280-299 8 8 8 9 9 9 F F F
12922024-03-06T20:58:34.278Z 15 300-319 10 10 10 12 12 12 F F F
12932024-03-06T20:58:34.278Z 16 320-339 1 1 1 1 1 1 F F F
12942024-03-06T20:58:34.278Z 17 340-359 10 10 10 12 12 12 F F F
12952024-03-06T20:58:34.278Z 18 360-379 9 9 9 10 10 10 F F F
12962024-03-06T20:58:34.279Z 19 380-399 11 11 11 13 13 13 F F F
12972024-03-06T20:58:34.279Z 20 400-419 9 9 9 10 10 10 F F F
12982024-03-06T20:58:34.279Z 21 420-439 11 11 11 13 13 13 F F F
12992024-03-06T20:58:34.279Z 22 440-459 11 11 11 13 13 13 F F F
13002024-03-06T20:58:34.279Z 23 460-479 9 9 9 10 10 10 F F F
13012024-03-06T20:58:34.279Z 24 480-499 10 10 10 11 11 11 F F F
13022024-03-06T20:58:34.279Z 25 500-519 1 1 1 1 1 1 F F F
13032024-03-06T20:58:34.279Z 26 520-539 9 9 9 10 10 10 F F F
13042024-03-06T20:58:34.279Z 27 540-559 8 8 8 9 9 9 F F F
13052024-03-06T20:58:34.279Z 28 560-579 2 2 2 2 2 2 F F F
13062024-03-06T20:58:34.279Z 29 580-599 10 10 10 12 12 12 F F F
13072024-03-06T20:58:34.279ZMax gen: 11, Max flush: 13
13082024-03-06T20:58:34.279Z
13092024-03-06T20:58:34.280ZBegin loop 6
13102024-03-06T20:58:34.280ZDownstairs to restart: 2
13112024-03-06T20:58:34.290Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.059191933Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184}
13122024-03-06T20:58:34.290Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.059481453Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184}
13132024-03-06T20:58:34.301Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
13142024-03-06T20:58:34.302Z,"time":"2024-03-06T20:58:33.05952191Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184}
13152024-03-06T20:58:34.302Z{"msg":"Crucible 62c47ab5-e9be-4c2d-96b1-10d12f72f160 has session id: 86b3704e-e67b-447b-a657-511f8576178b","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.059567858Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13162024-03-06T20:58:34.302Z{"msg":"Upstairs opts: Upstairs UUID: 62c47ab5-e9be-4c2d-96b1-10d12f72f160, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.059615104Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13172024-03-06T20:58:34.302Z{"msg":"Crucible stats registered with UUID: 62c47ab5-e9be-4c2d-96b1-10d12f72f160","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.059657531Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13182024-03-06T20:58:34.302Z{"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.059686339Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184}
13192024-03-06T20:58:34.303Z{"msg":"62c47ab5-e9be-4c2d-96b1-10d12f72f160 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.059722037Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13202024-03-06T20:58:34.303Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.059749865Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"client":"0","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13212024-03-06T20:58:34.303Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.059779323Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"client":"0","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13222024-03-06T20:58:34.303Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.059804821Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"client":"1","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13232024-03-06T20:58:34.304Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.059832769Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"client":"1","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13242024-03-06T20:58:34.304Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.059860217Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"client":"2","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13252024-03-06T20:58:34.304Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.059899575Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"client":"2","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13262024-03-06T20:58:34.304Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.059926453Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"":"io task","client":"2","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13272024-03-06T20:58:34.304Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.05996186Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"":"io task","client":"1","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13282024-03-06T20:58:34.305Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.059992628Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"":"io task","client":"0","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13292024-03-06T20:58:34.305Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.060035755Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"":"io task","client":"2","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13302024-03-06T20:58:34.305Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.060066873Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"":"io task","client":"0","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13312024-03-06T20:58:34.305Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.06010689Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"":"io task","client":"1","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13322024-03-06T20:58:34.306Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.060717218Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"client":"0","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13332024-03-06T20:58:34.306Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.060761535Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"client":"0","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13342024-03-06T20:58:34.306Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.060866508Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"client":"1","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13352024-03-06T20:58:34.306Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.060914115Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"client":"1","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13362024-03-06T20:58:34.306Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.061141409Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"client":"2","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13372024-03-06T20:58:34.307Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.061177367Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"client":"2","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13382024-03-06T20:58:34.307Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.061332526Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"client":"0","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13392024-03-06T20:58:34.307Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.061383512Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"client":"1","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13402024-03-06T20:58:34.307Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.061810093Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"client":"2","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13412024-03-06T20:58:34.308Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.062012119Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"client":"0","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13422024-03-06T20:58:34.308Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.062046896Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"client":"0","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13432024-03-06T20:58:34.308Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.062063095Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13442024-03-06T20:58:34.308Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.062077485Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"client":"1","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13452024-03-06T20:58:34.309Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.062099413Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"client":"1","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13462024-03-06T20:58:34.309Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.062113222Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13472024-03-06T20:58:34.309Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.062269781Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"client":"2","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13482024-03-06T20:58:34.309Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.062299419Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"client":"2","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13492024-03-06T20:58:34.309Z{"msg":"[0]R flush_numbers[0..12]: [13, 9, 1, 12, 10, 1, 1, 12, 12, 10, 12, 7]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.062314128Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"":"downstairs","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13502024-03-06T20:58:34.310Z{"msg":"[0]R generation[0..12]: [11, 8, 1, 10, 9, 1, 1, 10, 10, 9, 10, 6]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.062335867Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"":"downstairs","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13512024-03-06T20:58:34.310Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.062351426Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"":"downstairs","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13522024-03-06T20:58:34.310Z{"msg":"[1]R flush_numbers[0..12]: [13, 9, 1, 12, 10, 1, 1, 12, 12, 10, 12, 7]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.062366165Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"":"downstairs","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13532024-03-06T20:58:34.311Z{"msg":"[1]R generation[0..12]: [11, 8, 1, 10, 9, 1, 1, 10, 10, 9, 10, 6]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.062387523Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"":"downstairs","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13542024-03-06T20:58:34.311Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.062402392Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"":"downstairs","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13552024-03-06T20:58:34.311Z{"msg":"[2]R flush_numbers[0..12]: [13, 9, 1, 12, 10, 1, 1, 12, 12, 10, 12, 7]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.062424101Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"":"downstairs","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13562024-03-06T20:58:34.311Z{"msg":"[2]R generation[0..12]: [11, 8, 1, 10, 9, 1, 1, 10, 10, 9, 10, 6]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
13572024-03-06T20:58:34.312Z,"time":"2024-03-06T20:58:33.062438349Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"":"downstairs","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13582024-03-06T20:58:34.312Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.062456518Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"":"downstairs","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13592024-03-06T20:58:34.312Z{"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.062469627Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"":"downstairs","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13602024-03-06T20:58:34.312Z{"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.062490336Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"":"downstairs","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13612024-03-06T20:58:34.313Z{"msg":"Next flush: 14","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.062503925Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"":"downstairs","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13622024-03-06T20:58:34.313Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.062516824Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"":"downstairs","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13632024-03-06T20:58:34.313Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.062530493Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13642024-03-06T20:58:34.313Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.062548872Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"client":"0","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13652024-03-06T20:58:34.313Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.06257086Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"client":"1","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13662024-03-06T20:58:34.314Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.06258483Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"client":"2","":"downstairs","session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13672024-03-06T20:58:34.314Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.062605768Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13682024-03-06T20:58:34.314Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.062618427Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13692024-03-06T20:58:34.314Z{"msg":"62c47ab5-e9be-4c2d-96b1-10d12f72f160 is now active with session: 86b3704e-e67b-447b-a657-511f8576178b","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.062638106Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13702024-03-06T20:58:34.314Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.062650705Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184,"session_id":"86b3704e-e67b-447b-a657-511f8576178b"}
13712024-03-06T20:58:34.315Z{"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:33.062670744Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184}
13722024-03-06T20:58:34.315ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
13732024-03-06T20:58:34.315ZLoading write count information from file "/tmp/test_repair_verify.data"
13742024-03-06T20:58:34.315ZRun Repair workload
13752024-03-06T20:58:34.315Z01/30 Write block 594 len 24576 data: 3 3 3 3 2 2
13762024-03-06T20:58:34.316Z02/30 Read block 530 len 28672
13772024-03-06T20:58:34.316Z03/30 Write block 315 len 32768 data: 2 2 2 2 2 2 2 2
13782024-03-06T20:58:34.323Z04/30 Write block 503 len 24576 data: 2 2 2 2 2 2
13792024-03-06T20:58:34.323Z05/30 Write block 577 len 4096 data: 2
13802024-03-06T20:58:34.323Z06/30 Read block 61 len 8192
13812024-03-06T20:58:34.323Z07/30 Write block 367 len 4096 data: 2
13822024-03-06T20:58:34.329Z08/30 Flush
13832024-03-06T20:58:34.329Z09/30 Write block 210 len 36864 data: 2 2 2 2 2 3 3 3 3
13842024-03-06T20:58:34.329Z10/30 Read block 181 len 20480
13852024-03-06T20:58:34.329Z11/30 Read block 499 len 28672
13862024-03-06T20:58:34.329Z12/30 Write block 369 len 12288 data: 3 4 3
13872024-03-06T20:58:34.329Z13/30 Write block 132 len 4096 data: 2
13882024-03-06T20:58:34.329Z14/30 Flush
13892024-03-06T20:58:34.330Z15/30 Flush
13902024-03-06T20:58:34.330Z16/30 Flush
13912024-03-06T20:58:34.330Z17/30 Write block 448 len 32768 data: 2 2 2 2 2 2 2 2
13922024-03-06T20:58:34.330Z18/30 Read block 487 len 20480
13932024-03-06T20:58:34.330Z19/30 Write block 141 len 8192 data: 2 2
13942024-03-06T20:58:34.330Z20/30 Write block 566 len 20480 data: 2 2 2 2 2
13952024-03-06T20:58:34.330Z21/30 Read block 523 len 12288
13962024-03-06T20:58:34.330Z22/30 Read block 363 len 4096
13972024-03-06T20:58:34.330Z23/30 Read block 595 len 16384
13982024-03-06T20:58:34.330Z24/30 Write block 234 len 20480 data: 3 3 3 3 4
13992024-03-06T20:58:34.330Z25/30 Read block 57 len 36864
14002024-03-06T20:58:34.330Z26/30 Read block 76 len 40960
14012024-03-06T20:58:34.330Z27/30 Write block 375 len 24576 data: 2 2 2 2 2 2
14022024-03-06T20:58:34.331Z28/30 Write block 60 len 40960 data: 2 2 2 2 2 2 3 3 3 3
14032024-03-06T20:58:34.331Z29/30 Write block 282 len 24576 data: 2 2 3 3 5 5
14042024-03-06T20:58:34.331Z30/30 Write block 69 len 40960 data: 4 3 3 4 3 3 3 2 2 3
14052024-03-06T20:58:34.331Z----------------------------------------------------------------
14062024-03-06T20:58:34.331Z Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30
14072024-03-06T20:58:34.331ZStates: Active Active Active
14082024-03-06T20:58:34.331ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
14092024-03-06T20:58:34.331Z 1 Acked 1000 Write 6 Done Done Done false
14102024-03-06T20:58:34.331Z 2 Acked 1001 Read 7 Done Done Done false
14112024-03-06T20:58:34.331Z 3 Acked 1002 Write 8 Done Done Done false
14122024-03-06T20:58:34.331Z 4 Acked 1003 Write 6 Done Done Done false
14132024-03-06T20:58:34.331Z 5 Acked 1004 Write 1 Done Done Done false
14142024-03-06T20:58:34.332Z 6 Acked 1005 Read 2 Done Done Done false
14152024-03-06T20:58:34.332Z 7 Acked 1006 Write 1 Done Done Sent false
14162024-03-06T20:58:34.332Z 8 Acked 1007 Flush 0 Done Done Sent false
14172024-03-06T20:58:34.332Z 9 Acked 1008 Write 9 Done Done Sent false
14182024-03-06T20:58:34.332Z 10 Acked 1009 Read 5 Done Done Sent false
14192024-03-06T20:58:34.332Z 11 Acked 1010 Read 7 Done Done Sent false
14202024-03-06T20:58:34.332Z 12 Acked 1011 Write 3 Done Done Sent false
14212024-03-06T20:58:34.332Z 13 Acked 1012 Write 1 Done Done Sent false
14222024-03-06T20:58:34.332Z 14 Acked 1013 Flush 0 Done Done Sent false
14232024-03-06T20:58:34.332Z 15 Acked 1014 Flush 0 Done Done Sent false
14242024-03-06T20:58:34.332Z 16 Acked 1015 Flush 0 Done Done Sent false
14252024-03-06T20:58:34.332Z 17 Acked 1016 Write 8 Done Done Sent false
14262024-03-06T20:58:34.332Z 18 Acked 1017 Read 5 Done Done Sent false
14272024-03-06T20:58:34.333Z 19 Acked 1018 Write 2 Done Done Sent false
14282024-03-06T20:58:34.333Z 20 Acked 1019 Write 5 Done Done Sent false
14292024-03-06T20:58:34.333Z 21 Acked 1020 Read 3 Done Done Sent false
14302024-03-06T20:58:34.333Z 22 Acked 1021 Read 1 Done Done Sent false
14312024-03-06T20:58:34.333Z 23 Acked 1022 Read 4 Done Done Sent false
14322024-03-06T20:58:34.333Z 24 Acked 1023 Write 5 Done Done Sent false
14332024-03-06T20:58:34.333Z 25 Acked 1024 Read 9 Done Done Sent false
14342024-03-06T20:58:34.333Z 26 Acked 1025 Read 10 Done Done Sent false
14352024-03-06T20:58:34.333Z 27 Acked 1026 Write 6 Done Done Sent false
14362024-03-06T20:58:34.333Z 28 Acked 1027 Write 10 Done Done Sent false
14372024-03-06T20:58:34.333Z 29 Acked 1028 Write 6 Done Done Sent false
14382024-03-06T20:58:34.333Z 30 Acked 1029 Write 10 Sent Sent Sent false
14392024-03-06T20:58:34.333Z STATES DS:0 DS:1 DS:2 TOTAL
14402024-03-06T20:58:34.334Z New 0 0 0 0
14412024-03-06T20:58:34.334Z Sent 1 1 24 26
14422024-03-06T20:58:34.334Z Done 29 29 6 64
14432024-03-06T20:58:34.334Z Skipped 0 0 0 0
14442024-03-06T20:58:34.334Z Error 0 0 0 0
14452024-03-06T20:58:34.334ZLast Flush: 1015 1015 0
14462024-03-06T20:58:34.334ZDownstairs last five completed:
14472024-03-06T20:58:34.334ZUpstairs last five completed: 30 29 28 27 26
14482024-03-06T20:58:34.334Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T20:58:33.095122297Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1184}
14492024-03-06T20:58:34.334ZWrote out file "/tmp/test_repair_verify.data"
14502024-03-06T20:58:34.334Z
14512024-03-06T20:58:34.338ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
14522024-03-06T20:58:34.352ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14532024-03-06T20:58:34.352Z 0 000-019 11 11 11 13 13 13 F F F
14542024-03-06T20:58:34.352Z 1 020-039 8 8 8 9 9 9 F F F
14552024-03-06T20:58:34.352Z 2 040-059 1 1 1 1 1 1 F F F
14562024-03-06T20:58:34.352Z 3 060-079 10 10 10 12 12 12 T T F <---
14572024-03-06T20:58:34.352Z 4 080-099 9 9 9 10 10 10 F F F
14582024-03-06T20:58:34.352Z 5 100-119 1 1 1 1 1 1 F F F
14592024-03-06T20:58:34.356Z 6 120-139 12 12 1 15 15 1 F F F <---
14602024-03-06T20:58:34.356Z 7 140-159 10 10 10 12 12 12 T T F <---
14612024-03-06T20:58:34.356Z 8 160-179 10 10 10 12 12 12 F F F
14622024-03-06T20:58:34.356Z 9 180-199 9 9 9 10 10 10 F F F
14632024-03-06T20:58:34.356Z 10 200-219 12 12 10 15 15 12 F F F <---
14642024-03-06T20:58:34.356Z 11 220-239 6 6 6 7 7 7 T T F <---
14652024-03-06T20:58:34.356Z 12 240-259 10 10 10 12 12 12 F F F
14662024-03-06T20:58:34.356Z 13 260-279 10 10 10 12 12 12 F F F
14672024-03-06T20:58:34.356Z 14 280-299 8 8 8 9 9 9 T T F <---
14682024-03-06T20:58:34.356Z 15 300-319 12 12 10 14 14 12 F F T <---
14692024-03-06T20:58:34.356Z 16 320-339 12 12 1 14 14 1 F F T <---
14702024-03-06T20:58:34.356Z 17 340-359 10 10 10 12 12 12 F F F
14712024-03-06T20:58:34.356Z 18 360-379 12 12 9 15 15 10 T T F <---
14722024-03-06T20:58:34.356Z 19 380-399 11 11 11 13 13 13 T T F <---
14732024-03-06T20:58:34.357Z 20 400-419 9 9 9 10 10 10 F F F
14742024-03-06T20:58:34.357Z 21 420-439 11 11 11 13 13 13 F F F
14752024-03-06T20:58:34.357Z 22 440-459 11 11 11 13 13 13 T T F <---
14762024-03-06T20:58:34.357Z 23 460-479 9 9 9 10 10 10 F F F
14772024-03-06T20:58:34.357Z 24 480-499 10 10 10 11 11 11 F F F
14782024-03-06T20:58:34.357Z 25 500-519 12 12 1 14 14 1 F F T <---
14792024-03-06T20:58:34.357Z 26 520-539 9 9 9 10 10 10 F F F
14802024-03-06T20:58:34.357Z 27 540-559 8 8 8 9 9 9 F F F
14812024-03-06T20:58:34.357Z 28 560-579 12 12 2 14 14 2 T T T <---
14822024-03-06T20:58:34.357Z 29 580-599 12 12 10 14 14 12 F F T <---
14832024-03-06T20:58:34.357ZMax gen: 12, Max flush: 15
14842024-03-06T20:58:34.357ZError: Difference in extent metadata found!
14852024-03-06T20:58:34.357ZA Difference in extent metadata is expected here
14862024-03-06T20:58:34.357ZOn loop 6
14872024-03-06T20:58:34.357Z
14882024-03-06T20:58:34.360ZVerifying data now
14892024-03-06T20:58:40.095ZNo change after verify
14902024-03-06T20:58:40.095ZLoop: 6 Downstairs dump after verify (and repair):
14912024-03-06T20:58:40.108ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14922024-03-06T20:58:40.108Z 0 000-019 11 11 11 13 13 13 F F F
14932024-03-06T20:58:40.108Z 1 020-039 8 8 8 9 9 9 F F F
14942024-03-06T20:58:40.108Z 2 040-059 1 1 1 1 1 1 F F F
14952024-03-06T20:58:40.108Z 3 060-079 13 13 13 16 16 16 F F F
14962024-03-06T20:58:40.108Z 4 080-099 9 9 9 10 10 10 F F F
14972024-03-06T20:58:40.108Z 5 100-119 1 1 1 1 1 1 F F F
14982024-03-06T20:58:40.108Z 6 120-139 12 12 12 15 15 15 F F F
14992024-03-06T20:58:40.108Z 7 140-159 13 13 13 16 16 16 F F F
15002024-03-06T20:58:40.108Z 8 160-179 10 10 10 12 12 12 F F F
15012024-03-06T20:58:40.108Z 9 180-199 9 9 9 10 10 10 F F F
15022024-03-06T20:58:40.108Z 10 200-219 12 12 12 15 15 15 F F F
15032024-03-06T20:58:40.112Z 11 220-239 13 13 13 16 16 16 F F F
15042024-03-06T20:58:40.112Z 12 240-259 10 10 10 12 12 12 F F F
15052024-03-06T20:58:40.112Z 13 260-279 10 10 10 12 12 12 F F F
15062024-03-06T20:58:40.112Z 14 280-299 13 13 13 16 16 16 F F F
15072024-03-06T20:58:40.112Z 15 300-319 12 12 12 14 14 14 F F F
15082024-03-06T20:58:40.112Z 16 320-339 12 12 12 14 14 14 F F F
15092024-03-06T20:58:40.112Z 17 340-359 10 10 10 12 12 12 F F F
15102024-03-06T20:58:40.112Z 18 360-379 13 13 13 16 16 16 F F F
15112024-03-06T20:58:40.112Z 19 380-399 13 13 13 16 16 16 F F F
15122024-03-06T20:58:40.112Z 20 400-419 9 9 9 10 10 10 F F F
15132024-03-06T20:58:40.112Z 21 420-439 11 11 11 13 13 13 F F F
15142024-03-06T20:58:40.112Z 22 440-459 13 13 13 16 16 16 F F F
15152024-03-06T20:58:40.113Z 23 460-479 9 9 9 10 10 10 F F F
15162024-03-06T20:58:40.113Z 24 480-499 10 10 10 11 11 11 F F F
15172024-03-06T20:58:40.113Z 25 500-519 12 12 12 14 14 14 F F F
15182024-03-06T20:58:40.113Z 26 520-539 9 9 9 10 10 10 F F F
15192024-03-06T20:58:40.113Z 27 540-559 8 8 8 9 9 9 F F F
15202024-03-06T20:58:40.113Z 28 560-579 13 13 13 16 16 16 F F F
15212024-03-06T20:58:40.113Z 29 580-599 12 12 12 14 14 14 F F F
15222024-03-06T20:58:40.113ZMax gen: 13, Max flush: 16
15232024-03-06T20:58:40.113Z
15242024-03-06T20:58:40.113ZBegin loop 7
15252024-03-06T20:58:40.113ZDownstairs to restart: 1
15262024-03-06T20:58:40.132Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.904669931Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192}
15272024-03-06T20:58:40.132Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.904908515Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192}
15282024-03-06T20:58:40.132Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.904927603Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192}
15292024-03-06T20:58:40.132Z{"msg":"Crucible 80138e1f-4b4d-40bf-bce9-774a15294a8e has session id: d6720130-6f7f-4b45-9ca3-748448f39d60","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.904958481Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15302024-03-06T20:58:40.132Z{"msg":"Crucible runtime is spawned
15312024-03-06T20:58:40.138ZUpstairs opts: Upstairs UUID: 80138e1f-4b4d-40bf-bce9-774a15294a8e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.905035256Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15322024-03-06T20:58:40.138Z{"msg":"Crucible stats registered with UUID: 80138e1f-4b4d-40bf-bce9-774a15294a8e","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.905072833Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15332024-03-06T20:58:40.138Z{"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.90512025Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192}
15342024-03-06T20:58:40.138Z{"msg":"80138e1f-4b4d-40bf-bce9-774a15294a8e active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.905178036Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15352024-03-06T20:58:40.138Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.90527157Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"client":"0","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15362024-03-06T20:58:40.138Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.905300937Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"client":"0","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15372024-03-06T20:58:40.138Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.905329226Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"client":"1","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15382024-03-06T20:58:40.138Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.905356284Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"client":"1","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15392024-03-06T20:58:40.138Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.905390821Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"client":"2","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15402024-03-06T20:58:40.138Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.905417529Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"client":"2","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15412024-03-06T20:58:40.138Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.905461526Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"":"io task","client":"0","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15422024-03-06T20:58:40.138Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.905511023Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"":"io task","client":"1","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15432024-03-06T20:58:40.138Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.9055503Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"":"io task","client":"2","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15442024-03-06T20:58:40.138Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.905645414Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"":"io task","client":"1","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15452024-03-06T20:58:40.138Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.9057023Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"":"io task","client":"2","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15462024-03-06T20:58:40.138Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.905781664Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"":"io task","client":"0","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15472024-03-06T20:58:40.138Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.906484256Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"client":"2","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15482024-03-06T20:58:40.139Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.906541932Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"client":"2","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15492024-03-06T20:58:40.139Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.906580649Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"client":"0","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15502024-03-06T20:58:40.139Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.906606218Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"client":"0","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15512024-03-06T20:58:40.139Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.906841791Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"client":"1","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15522024-03-06T20:58:40.139Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.906881008Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"client":"1","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15532024-03-06T20:58:40.139Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.907422411Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"client":"2","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15542024-03-06T20:58:40.139Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.907482267Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"client":"0","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15552024-03-06T20:58:40.139Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.907641736Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"client":"1","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15562024-03-06T20:58:40.139Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.908097755Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"client":"2","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15572024-03-06T20:58:40.139Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.908147761Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"client":"2","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15582024-03-06T20:58:40.139Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.908179389Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15592024-03-06T20:58:40.139Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.908211307Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"client":"0","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15602024-03-06T20:58:40.139Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.908244134Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"client":"0","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15612024-03-06T20:58:40.139Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.908278572Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15622024-03-06T20:58:40.139Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.90830894Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"client":"1","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15632024-03-06T20:58:40.139Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.908343678Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"client":"1","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15642024-03-06T20:58:40.139Z{"msg":"[0]R flush_numbers[0..12]: [13, 9, 1, 16, 10, 1, 15, 16, 12, 10, 15, 16]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.908385635Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"":"downstairs","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15652024-03-06T20:58:40.139Z{"msg":"[0]R generation[0..12]: [11, 8, 1, 13, 9, 1, 12, 13, 10, 9, 12, 13]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
15662024-03-06T20:58:40.139Z,"time":"2024-03-06T20:58:38.908411633Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"":"downstairs","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15672024-03-06T20:58:40.139Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.908447161Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"":"downstairs","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15682024-03-06T20:58:40.139Z{"msg":"[1]R flush_numbers[0..12]: [13, 9, 1, 16, 10, 1, 15, 16, 12, 10, 15, 16]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.908483868Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"":"downstairs","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15692024-03-06T20:58:40.139Z{"msg":"[1]R generation[0..12]: [11, 8, 1, 13, 9, 1, 12, 13, 10, 9, 12, 13]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.908521555Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"":"downstairs","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15702024-03-06T20:58:40.139Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.908567712Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"":"downstairs","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15712024-03-06T20:58:40.140Z{"msg":"[2]R flush_numbers[0..12]: [13, 9, 1, 16, 10, 1, 15, 16, 12, 10, 15, 16]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.90860774Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"":"downstairs","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15722024-03-06T20:58:40.143Z{"msg":"[2]R generation[0..12]: [11, 8, 1, 13, 9, 1, 12, 13, 10, 9, 12, 13]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.908688454Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"":"downstairs","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15732024-03-06T20:58:40.143Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
15742024-03-06T20:58:40.143Z,"time":"2024-03-06T20:58:38.90874323Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"":"downstairs","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15752024-03-06T20:58:40.144Z{"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.908772468Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"":"downstairs","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15762024-03-06T20:58:40.144Z{"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.908801046Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"":"downstairs","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15772024-03-06T20:58:40.144Z{"msg":"Next flush: 17","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.908815245Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"":"downstairs","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15782024-03-06T20:58:40.144Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.908836504Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"":"downstairs","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15792024-03-06T20:58:40.144Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.908850253Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15802024-03-06T20:58:40.144Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.908868971Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"client":"0","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15812024-03-06T20:58:40.144Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.908882441Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"client":"1","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15822024-03-06T20:58:40.144Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.90889572Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"client":"2","":"downstairs","session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15832024-03-06T20:58:40.144Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.908915558Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15842024-03-06T20:58:40.144Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.908928167Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15852024-03-06T20:58:40.144Z{"msg":"80138e1f-4b4d-40bf-bce9-774a15294a8e is now active with session: d6720130-6f7f-4b45-9ca3-748448f39d60","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.908947196Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15862024-03-06T20:58:40.144Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.908959335Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192,"session_id":"d6720130-6f7f-4b45-9ca3-748448f39d60"}
15872024-03-06T20:58:40.144Z{"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:38.908999953Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192}
15882024-03-06T20:58:40.144ZLoading write count information from file "/tmp/test_repair_verify.data"
15892024-03-06T20:58:40.144ZRun Repair workload
15902024-03-06T20:58:40.144Z01/30 Write block 489 len 4096 data: 4
15912024-03-06T20:58:40.144Z02/30 Write block 569 len 32768 data: 3 3 2 2 2 2 2 2
15922024-03-06T20:58:40.144Z03/30 Read block 276 len 8192
15932024-03-06T20:58:40.144Z04/30 Write block 448 len 4096 data: 3
15942024-03-06T20:58:40.144Z05/30 Read block 511 len 32768
15952024-03-06T20:58:40.144Z06/30 Read block 311 len 4096
15962024-03-06T20:58:40.144Z07/30 Write block 373 len 32768 data: 2 2 3 3 3 3 3 3
15972024-03-06T20:58:40.144Z08/30 Read block 213 len 16384
15982024-03-06T20:58:40.148Z09/30 Write block 282 len 40960 data: 3 3 4 4 6 6 5 5 5 5
15992024-03-06T20:58:40.148Z10/30 Write block 437 len 16384 data: 2 2 2 2
16002024-03-06T20:58:40.148Z11/30 Write block 27 len 40960 data: 2 2 2 2 2 2 3 3 3 2
16012024-03-06T20:58:40.148Z12/30 Read block 46 len 8192
16022024-03-06T20:58:40.151Z13/30 Write block 461 len 36864 data: 4 4 4 4 4 4 3 3 3
16032024-03-06T20:58:40.151Z14/30 Write block 225 len 24576 data: 2 2 2 2 2 2
16042024-03-06T20:58:40.151Z15/30 Read block 63 len 4096
16052024-03-06T20:58:40.154Z16/30 Read block 351 len 16384
16062024-03-06T20:58:40.154Z17/30 Write block 439 len 8192 data: 3 3
16072024-03-06T20:58:40.154Z18/30 Write block 226 len 8192 data: 3 3
16082024-03-06T20:58:40.154Z19/30 Write block 370 len 12288 data: 5 4 3
16092024-03-06T20:58:40.157Z20/30 Write block 413 len 36864 data: 3 3 3 3 3 3 3 3 5
16102024-03-06T20:58:40.157Z21/30 Read block 150 len 20480
16112024-03-06T20:58:40.157Z22/30 Read block 412 len 32768
16122024-03-06T20:58:40.161Z23/30 Read block 76 len 36864
16132024-03-06T20:58:40.161Z24/30 Flush
16142024-03-06T20:58:40.164Z25/30 Write block 530 len 24576 data: 4 4 4 4 2 3
16152024-03-06T20:58:40.168Z26/30 Flush
16162024-03-06T20:58:40.171Z27/30 Write block 62 len 24576 data: 3 3 3 3 4 4
16172024-03-06T20:58:40.172Z28/30 Read block 239 len 36864
16182024-03-06T20:58:40.172Z29/30 Write block 527 len 40960 data: 4 4 4 5 5 5 5 3 4 3
16192024-03-06T20:58:40.172Z30/30 Write block 235 len 36864 data: 4 4 4 5 4 3 4 3 3
16202024-03-06T20:58:40.176Z----------------------------------------------------------------
16212024-03-06T20:58:40.176Z Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:30
16222024-03-06T20:58:40.176ZStates: Active Active Active
16232024-03-06T20:58:40.176ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
16242024-03-06T20:58:40.176Z 1 Acked 1000 Write 1 Done Sent Done false
16252024-03-06T20:58:40.176Z 2 Acked 1001 Write 8 Done Sent Done false
16262024-03-06T20:58:40.176Z 3 Acked 1002 Read 2 Done Sent Done false
16272024-03-06T20:58:40.176Z 4 Acked 1003 Write 1 Done Sent Done false
16282024-03-06T20:58:40.176Z 5 Acked 1004 Read 8 Done Sent Done false
16292024-03-06T20:58:40.176Z 6 Acked 1005 Read 1 Done Sent Done false
16302024-03-06T20:58:40.176Z 7 Acked 1006 Write 8 Done Sent Done false
16312024-03-06T20:58:40.176Z 8 Acked 1007 Read 4 Done Sent Done false
16322024-03-06T20:58:40.176Z 9 Acked 1008 Write 10 Done Sent Done false
16332024-03-06T20:58:40.176Z 10 Acked 1009 Write 4 Done Sent Done false
16342024-03-06T20:58:40.176Z 11 Acked 1010 Write 10 Done Sent Done false
16352024-03-06T20:58:40.176Z 12 Acked 1011 Read 2 Done Sent Done false
16362024-03-06T20:58:40.176Z 13 Acked 1012 Write 9 Done Sent Done false
16372024-03-06T20:58:40.176Z 14 Acked 1013 Write 6 Done Sent Done false
16382024-03-06T20:58:40.176Z 15 Acked 1014 Read 1 Done Sent Done false
16392024-03-06T20:58:40.176Z 16 Acked 1015 Read 4 Done Sent Done false
16402024-03-06T20:58:40.176Z 17 Acked 1016 Write 2 Done Sent Done false
16412024-03-06T20:58:40.177Z 18 Acked 1017 Write 2 Done Sent Done false
16422024-03-06T20:58:40.177Z 19 Acked 1018 Write 3 Done Sent Done false
16432024-03-06T20:58:40.177Z 20 Acked 1019 Write 9 Done Sent Done false
16442024-03-06T20:58:40.177Z 21 Acked 1020 Read 5 Done Sent Done false
16452024-03-06T20:58:40.177Z 22 Acked 1021 Read 8 Done Sent Done false
16462024-03-06T20:58:40.177Z 23 Acked 1022 Read 9 Done Sent Done false
16472024-03-06T20:58:40.177Z 24 Acked 1023 Flush 0 Done Sent Done false
16482024-03-06T20:58:40.177Z 25 Acked 1024 Write 6 Done Sent Done false
16492024-03-06T20:58:40.177Z 26 Acked 1025 Flush 0 Done Sent Done false
16502024-03-06T20:58:40.177Z 27 Acked 1026 Write 6 Done Sent Done false
16512024-03-06T20:58:40.177Z 28 Acked 1027 Read 9 Done Sent Done false
16522024-03-06T20:58:40.177Z 29 Acked 1028 Write 10 Done Sent Sent false
16532024-03-06T20:58:40.177Z 30 Acked 1029 Write 9 Sent Sent Sent false
16542024-03-06T20:58:40.177Z STATES DS:0 DS:1 DS:2 TOTAL
16552024-03-06T20:58:40.177Z New 0 0 0 0
16562024-03-06T20:58:40.178Z Sent 1 30 2 33
16572024-03-06T20:58:40.178Z Done 29 0 28 57
16582024-03-06T20:58:40.178Z Skipped 0 0 0 0
16592024-03-06T20:58:40.178Z Error 0 0 0 0
16602024-03-06T20:58:40.178ZLast Flush: 1025 0 1025
16612024-03-06T20:58:40.178ZDownstairs last five completed:
16622024-03-06T20:58:40.178ZUpstairs last five completed: 30 29 28 27 26
16632024-03-06T20:58:40.178Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T20:58:38.945738121Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1192}
16642024-03-06T20:58:40.178ZWrote out file "/tmp/test_repair_verify.data"
16652024-03-06T20:58:40.181Z
16662024-03-06T20:58:40.184ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
16672024-03-06T20:58:40.199ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
16682024-03-06T20:58:40.199Z 0 000-019 11 11 11 13 13 13 F F F
16692024-03-06T20:58:40.199Z 1 020-039 14 8 14 17 9 17 F F F <---
16702024-03-06T20:58:40.199Z 2 040-059 1 1 1 1 1 1 F F F
16712024-03-06T20:58:40.199Z 3 060-079 13 13 13 16 16 16 T F T <---
16722024-03-06T20:58:40.199Z 4 080-099 9 9 9 10 10 10 F F F
16732024-03-06T20:58:40.199Z 5 100-119 1 1 1 1 1 1 F F F
16742024-03-06T20:58:40.199Z 6 120-139 12 12 12 15 15 15 F F F
16752024-03-06T20:58:40.199Z 7 140-159 13 13 13 16 16 16 F F F
16762024-03-06T20:58:40.199Z 8 160-179 10 10 10 12 12 12 F F F
16772024-03-06T20:58:40.199Z 9 180-199 9 9 9 10 10 10 F F F
16782024-03-06T20:58:40.199Z 10 200-219 12 12 12 15 15 15 F F F
16792024-03-06T20:58:40.199Z 11 220-239 14 13 14 17 16 17 T F F <---
16802024-03-06T20:58:40.199Z 12 240-259 10 10 10 12 12 12 T F F <---
16812024-03-06T20:58:40.199Z 13 260-279 10 10 10 12 12 12 F F F
16822024-03-06T20:58:40.199Z 14 280-299 14 13 14 17 16 17 F F F <---
16832024-03-06T20:58:40.199Z 15 300-319 12 12 12 14 14 14 F F F
16842024-03-06T20:58:40.199Z 16 320-339 12 12 12 14 14 14 F F F
16852024-03-06T20:58:40.200Z 17 340-359 10 10 10 12 12 12 F F F
16862024-03-06T20:58:40.200Z 18 360-379 14 13 14 17 16 17 F F F <---
16872024-03-06T20:58:40.200Z 19 380-399 14 13 14 17 16 17 F F F <---
16882024-03-06T20:58:40.200Z 20 400-419 14 9 14 17 10 17 F F F <---
16892024-03-06T20:58:40.200Z 21 420-439 14 11 14 17 13 17 F F F <---
16902024-03-06T20:58:40.200Z 22 440-459 14 13 14 17 16 17 F F F <---
16912024-03-06T20:58:40.200Z 23 460-479 14 9 14 17 10 17 F F F <---
16922024-03-06T20:58:40.200Z 24 480-499 14 10 14 17 11 17 F F F <---
16932024-03-06T20:58:40.203Z 25 500-519 12 12 12 14 14 14 F F F
16942024-03-06T20:58:40.203Z 26 520-539 14 9 14 18 10 18 T F T <---
16952024-03-06T20:58:40.204Z 27 540-559 8 8 8 9 9 9 F F F
16962024-03-06T20:58:40.204Z 28 560-579 14 13 14 17 16 17 F F F <---
16972024-03-06T20:58:40.204Z 29 580-599 12 12 12 14 14 14 F F F
16982024-03-06T20:58:40.204ZMax gen: 14, Max flush: 18
16992024-03-06T20:58:40.204ZError: Difference in extent metadata found!
17002024-03-06T20:58:40.204ZA Difference in extent metadata is expected here
17012024-03-06T20:58:40.204ZOn loop 7
17022024-03-06T20:58:40.204Z
17032024-03-06T20:58:40.207ZVerifying data now
17042024-03-06T20:58:44.937ZNo change after verify
17052024-03-06T20:58:44.937ZLoop: 7 Downstairs dump after verify (and repair):
17062024-03-06T20:58:44.944ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
17072024-03-06T20:58:44.945Z 0 000-019 11 11 11 13 13 13 F F F
17082024-03-06T20:58:44.945Z 1 020-039 14 14 14 17 17 17 F F F
17092024-03-06T20:58:44.945Z 2 040-059 1 1 1 1 1 1 F F F
17102024-03-06T20:58:44.945Z 3 060-079 15 15 15 19 19 19 F F F
17112024-03-06T20:58:44.945Z 4 080-099 9 9 9 10 10 10 F F F
17122024-03-06T20:58:44.945Z 5 100-119 1 1 1 1 1 1 F F F
17132024-03-06T20:58:44.945Z 6 120-139 12 12 12 15 15 15 F F F
17142024-03-06T20:58:44.945Z 7 140-159 13 13 13 16 16 16 F F F
17152024-03-06T20:58:44.945Z 8 160-179 10 10 10 12 12 12 F F F
17162024-03-06T20:58:44.945Z 9 180-199 9 9 9 10 10 10 F F F
17172024-03-06T20:58:44.945Z 10 200-219 12 12 12 15 15 15 F F F
17182024-03-06T20:58:44.945Z 11 220-239 15 15 15 19 19 19 F F F
17192024-03-06T20:58:44.945Z 12 240-259 15 15 15 19 19 19 F F F
17202024-03-06T20:58:44.945Z 13 260-279 10 10 10 12 12 12 F F F
17212024-03-06T20:58:44.945Z 14 280-299 14 14 14 17 17 17 F F F
17222024-03-06T20:58:44.945Z 15 300-319 12 12 12 14 14 14 F F F
17232024-03-06T20:58:44.945Z 16 320-339 12 12 12 14 14 14 F F F
17242024-03-06T20:58:44.945Z 17 340-359 10 10 10 12 12 12 F F F
17252024-03-06T20:58:44.945Z 18 360-379 14 14 14 17 17 17 F F F
17262024-03-06T20:58:44.945Z 19 380-399 14 14 14 17 17 17 F F F
17272024-03-06T20:58:44.945Z 20 400-419 14 14 14 17 17 17 F F F
17282024-03-06T20:58:44.945Z 21 420-439 14 14 14 17 17 17 F F F
17292024-03-06T20:58:44.945Z 22 440-459 14 14 14 17 17 17 F F F
17302024-03-06T20:58:44.946Z 23 460-479 14 14 14 17 17 17 F F F
17312024-03-06T20:58:44.946Z 24 480-499 14 14 14 17 17 17 F F F
17322024-03-06T20:58:44.946Z 25 500-519 12 12 12 14 14 14 F F F
17332024-03-06T20:58:44.946Z 26 520-539 15 15 15 19 19 19 F F F
17342024-03-06T20:58:44.946Z 27 540-559 8 8 8 9 9 9 F F F
17352024-03-06T20:58:44.946Z 28 560-579 14 14 14 17 17 17 F F F
17362024-03-06T20:58:44.946Z 29 580-599 12 12 12 14 14 14 F F F
17372024-03-06T20:58:44.946ZMax gen: 15, Max flush: 19
17382024-03-06T20:58:44.958Z
17392024-03-06T20:58:44.958ZBegin loop 8
17402024-03-06T20:58:44.958ZDownstairs to restart: 2
17412024-03-06T20:58:44.988Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.735300669Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200}
17422024-03-06T20:58:44.988Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.73556952Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200}
17432024-03-06T20:58:44.988ZCrucible runtime is spawned
17442024-03-06T20:58:44.988Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.735637116Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200}
17452024-03-06T20:58:44.988Z{"msg":"Crucible 9a70fa12-3203-4664-bf57-93cc7772d692 has session id: ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.735661234Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17462024-03-06T20:58:44.988Z{"msg":"Upstairs opts: Upstairs UUID: 9a70fa12-3203-4664-bf57-93cc7772d692, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.735695082Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17472024-03-06T20:58:44.988Z{"msg":"Crucible stats registered with UUID: 9a70fa12-3203-4664-bf57-93cc7772d692","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.73571026Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17482024-03-06T20:58:44.988Z{"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.73572466Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200}
17492024-03-06T20:58:44.988Z{"msg":"9a70fa12-3203-4664-bf57-93cc7772d692 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.735736379Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17502024-03-06T20:58:44.988Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.735759627Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"client":"0","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17512024-03-06T20:58:44.988Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.735785035Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"client":"0","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17522024-03-06T20:58:44.988Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.735809774Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"client":"1","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17532024-03-06T20:58:44.988Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.735833782Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"client":"1","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17542024-03-06T20:58:44.988Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.73585849Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"client":"2","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17552024-03-06T20:58:44.988Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.735881929Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"client":"2","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17562024-03-06T20:58:44.988Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.735918566Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17572024-03-06T20:58:44.988Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.735950884Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"":"io task","client":"0","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17582024-03-06T20:58:44.988Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.735988601Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"":"io task","client":"1","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17592024-03-06T20:58:44.988Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.736092484Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"":"io task","client":"1","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17602024-03-06T20:58:44.988Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.73614876Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17612024-03-06T20:58:44.989Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.736174469Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"":"io task","client":"0","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17622024-03-06T20:58:44.989Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.73688421Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"client":"0","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17632024-03-06T20:58:44.989Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.736913478Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"client":"0","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17642024-03-06T20:58:44.989Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.736935336Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"client":"1","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17652024-03-06T20:58:44.989Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.736971524Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"client":"1","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17662024-03-06T20:58:44.995Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.737402484Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"client":"1","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17672024-03-06T20:58:44.996Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.737534205Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"client":"0","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17682024-03-06T20:58:44.996Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.737675555Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"client":"2","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17692024-03-06T20:58:44.996Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.737704953Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"client":"2","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17702024-03-06T20:58:44.996Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.737869342Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"client":"1","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17712024-03-06T20:58:44.996Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.737919718Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"client":"1","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17722024-03-06T20:58:44.996Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.737946197Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17732024-03-06T20:58:44.996Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.738072358Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"client":"0","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17742024-03-06T20:58:44.996Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.738108905Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"client":"0","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17752024-03-06T20:58:44.996Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.738128094Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17762024-03-06T20:58:44.996Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.738312031Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"client":"2","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17772024-03-06T20:58:44.996Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.738707194Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"client":"2","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17782024-03-06T20:58:44.996Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.738747871Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"client":"2","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17792024-03-06T20:58:44.996Z{"msg":"[0]R flush_numbers[0..12]: [13, 17, 1, 19, 10, 1, 15, 16, 12, 10, 15, 19]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.738773829Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"":"downstairs","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17802024-03-06T20:58:44.996Z{"msg":"[0]R generation[0..12]: [11, 14, 1, 15, 9, 1, 12, 13, 10, 9, 12, 15]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.738802358Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"":"downstairs","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17812024-03-06T20:58:44.996Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.738828046Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"":"downstairs","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17822024-03-06T20:58:44.996Z{"msg":"[1]R flush_numbers[0..12]: [13, 17, 1, 19, 10, 1, 15, 16, 12, 10, 15, 19]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
17832024-03-06T20:58:44.996Z,"time":"2024-03-06T20:58:43.738860054Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"":"downstairs","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17842024-03-06T20:58:44.996Z{"msg":"[1]R generation[0..12]: [11, 14, 1, 15, 9, 1, 12, 13, 10, 9, 12, 15]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.738886152Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"":"downstairs","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17852024-03-06T20:58:44.996Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.73891418Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"":"downstairs","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17862024-03-06T20:58:44.996Z{"msg":"[2]R flush_numbers[0..12]: [13, 17, 1, 19, 10, 1, 15, 16, 12, 10, 15, 19]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.738938208Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"":"downstairs","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17872024-03-06T20:58:44.996Z{"msg":"[2]R generation[0..12]: [11, 14, 1, 15, 9, 1, 12, 13, 10, 9, 12, 15]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.738977875Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"":"downstairs","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17882024-03-06T20:58:44.996Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.739004024Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"":"downstairs","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17892024-03-06T20:58:44.996Z{"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.739025392Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"":"downstairs","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17902024-03-06T20:58:44.997Z{"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.739047801Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"":"downstairs","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17912024-03-06T20:58:44.997Z{"msg":"Next flush: 20","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.739078079Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"":"downstairs","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17922024-03-06T20:58:44.997Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.739100977Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"":"downstairs","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17932024-03-06T20:58:44.997Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.739128165Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17942024-03-06T20:58:44.997Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.739148674Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"client":"0","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17952024-03-06T20:58:44.997Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.739183791Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"client":"1","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17962024-03-06T20:58:44.997Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
17972024-03-06T20:58:44.997Z,"time":"2024-03-06T20:58:43.7392051Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"client":"2","":"downstairs","session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17982024-03-06T20:58:44.997Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.739242427Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
17992024-03-06T20:58:44.997Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.739273265Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
18002024-03-06T20:58:44.997Z{"msg":"9a70fa12-3203-4664-bf57-93cc7772d692 is now active with session: ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.739306193Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
18012024-03-06T20:58:44.997Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.739337091Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200,"session_id":"ccfeec1e-caa7-45a3-8f55-b8f399aa4cb7"}
18022024-03-06T20:58:44.997Z{"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:43.739366169Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200}
18032024-03-06T20:58:44.997ZLoading write count information from file "/tmp/test_repair_verify.data"
18042024-03-06T20:58:44.997ZRun Repair workload
18052024-03-06T20:58:44.997Z01/30 Write block 444 len 4096 data: 3
18062024-03-06T20:58:44.997Z02/30 Read block 158 len 32768
18072024-03-06T20:58:44.997Z03/30 Write block 305 len 36864 data: 3 3 3 3 3 3 3 3 2
18082024-03-06T20:58:44.997Z04/30 Read block 183 len 36864
18092024-03-06T20:58:44.997Z05/30 Read block 153 len 32768
18102024-03-06T20:58:44.997Z06/30 Read block 396 len 20480
18112024-03-06T20:58:44.997Z07/30 Write block 98 len 40960 data: 2 2 2 2 2 2 2 2 2 2
18122024-03-06T20:58:44.997Z08/30 Read block 465 len 20480
18132024-03-06T20:58:44.998Z09/30 Flush
18142024-03-06T20:58:44.998Z10/30 Write block 491 len 40960 data: 3 3 3 3 2 2 2 2 2 2
18152024-03-06T20:58:44.998Z11/30 Write block 101 len 28672 data: 3 3 3 3 3 3 3
18162024-03-06T20:58:44.998Z12/30 Write block 10 len 24576 data: 2 3 3 3 3 2
18172024-03-06T20:58:44.998Z13/30 Flush
18182024-03-06T20:58:44.998Z14/30 Write block 225 len 32768 data: 3 4 4 3 3 3 4 4
18192024-03-06T20:58:44.998Z15/30 Write block 361 len 16384 data: 4 3 3 3
18202024-03-06T20:58:44.998Z16/30 Write block 363 len 28672 data: 4 4 3 2 3 2 4
18212024-03-06T20:58:44.998Z17/30 Read block 320 len 4096
18222024-03-06T20:58:44.998Z18/30 Read block 308 len 16384
18232024-03-06T20:58:44.998Z19/30 Read block 21 len 8192
18242024-03-06T20:58:44.998Z20/30 Write block 587 len 24576 data: 2 2 2 3 3 3
18252024-03-06T20:58:44.998Z21/30 Flush
18262024-03-06T20:58:45.002Z22/30 Write block 143 len 36864 data: 2 2 2 2 2 2 2 2 2
18272024-03-06T20:58:45.002Z23/30 Read block 565 len 4096
18282024-03-06T20:58:45.002Z24/30 Write block 38 len 28672 data: 2 2 2 2 2 2 2
18292024-03-06T20:58:45.002Z25/30 Write block 247 len 24576 data: 2 2 2 2 2 3
18302024-03-06T20:58:45.002Z26/30 Write block 545 len 28672 data: 3 3 3 3 3 2 2
18312024-03-06T20:58:45.002Z27/30 Read block 398 len 40960
18322024-03-06T20:58:45.002Z28/30 Read block 564 len 28672
18332024-03-06T20:58:45.002Z29/30 Write block 38 len 40960 data: 3 3 3 3 3 3 3 2 2 2
18342024-03-06T20:58:45.006Z30/30 Read block 583 len 32768
18352024-03-06T20:58:45.006Z----------------------------------------------------------------
18362024-03-06T20:58:45.006Z Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30
18372024-03-06T20:58:45.006ZStates: Active Active Active
18382024-03-06T20:58:45.006ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
18392024-03-06T20:58:45.006Z 1 Acked 1000 Write 1 Done Done Done false
18402024-03-06T20:58:45.006Z 2 Acked 1001 Read 8 Done Done Done false
18412024-03-06T20:58:45.006Z 3 Acked 1002 Write 9 Done Done Sent false
18422024-03-06T20:58:45.006Z 4 Acked 1003 Read 9 Done Done Sent false
18432024-03-06T20:58:45.006Z 5 Acked 1004 Read 8 Done Done Sent false
18442024-03-06T20:58:45.006Z 6 Acked 1005 Read 5 Done Done Sent false
18452024-03-06T20:58:45.006Z 7 Acked 1006 Write 10 Done Done Sent false
18462024-03-06T20:58:45.006Z 8 Acked 1007 Read 5 Done Done Sent false
18472024-03-06T20:58:45.006Z 9 Acked 1008 Flush 0 Done Done Sent false
18482024-03-06T20:58:45.006Z 10 Acked 1009 Write 10 Done Done Sent false
18492024-03-06T20:58:45.006Z 11 Acked 1010 Write 7 Done Done Sent false
18502024-03-06T20:58:45.006Z 12 Acked 1011 Write 6 Done Done Sent false
18512024-03-06T20:58:45.007Z 13 Acked 1012 Flush 0 Done Done Sent false
18522024-03-06T20:58:45.007Z 14 Acked 1013 Write 8 Done Done Sent false
18532024-03-06T20:58:45.007Z 15 Acked 1014 Write 4 Done Done Sent false
18542024-03-06T20:58:45.007Z 16 Acked 1015 Write 7 Done Done Sent false
18552024-03-06T20:58:45.007Z 17 Acked 1016 Read 1 Done Done Sent false
18562024-03-06T20:58:45.007Z 18 Acked 1017 Read 4 Done Done Sent false
18572024-03-06T20:58:45.007Z 19 Acked 1018 Read 2 Done Done Sent false
18582024-03-06T20:58:45.007Z 20 Acked 1019 Write 6 Done Done Sent false
18592024-03-06T20:58:45.007Z 21 Acked 1020 Flush 0 Done Done Sent false
18602024-03-06T20:58:45.007Z 22 Acked 1021 Write 9 Done Done Sent false
18612024-03-06T20:58:45.007Z 23 Acked 1022 Read 1 Done Done Sent false
18622024-03-06T20:58:45.007Z 24 Acked 1023 Write 7 Done Done Sent false
18632024-03-06T20:58:45.007Z 25 Acked 1024 Write 6 Done Done Sent false
18642024-03-06T20:58:45.007Z 26 Acked 1025 Write 7 Done Done Sent false
18652024-03-06T20:58:45.007Z 27 Acked 1026 Read 10 Done Done Sent false
18662024-03-06T20:58:45.007Z 28 Acked 1027 Read 7 Done Done Sent false
18672024-03-06T20:58:45.007Z 29 Acked 1028 Write 10 Done Done Sent false
18682024-03-06T20:58:45.007Z 30 Acked 1029 Read 8 Done Sent Sent false
18692024-03-06T20:58:45.007Z STATES DS:0 DS:1 DS:2 TOTAL
18702024-03-06T20:58:45.007Z New 0 0 0 0
18712024-03-06T20:58:45.007Z Sent 0 1 28 29
18722024-03-06T20:58:45.007Z Done 30 29 2 61
18732024-03-06T20:58:45.007Z Skipped 0 0 0 0
18742024-03-06T20:58:45.007Z Error 0 0 0 0
18752024-03-06T20:58:45.008ZLast Flush: 1020 1020 0
18762024-03-06T20:58:45.008ZDownstairs last five completed:
18772024-03-06T20:58:45.008ZUpstairs last five completed: 30 29 28 27 26
18782024-03-06T20:58:45.008Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T20:58:43.777273436Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1200}
18792024-03-06T20:58:45.008ZWrote out file "/tmp/test_repair_verify.data"
18802024-03-06T20:58:45.011Z
18812024-03-06T20:58:45.014ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
18822024-03-06T20:58:45.030ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
18832024-03-06T20:58:45.030Z 0 000-019 16 16 11 21 21 13 F F F <---
18842024-03-06T20:58:45.030Z 1 020-039 14 14 14 17 17 17 T T F <---
18852024-03-06T20:58:45.030Z 2 040-059 1 1 1 1 1 1 T T F <---
18862024-03-06T20:58:45.030Z 3 060-079 15 15 15 19 19 19 F F F
18872024-03-06T20:58:45.030Z 4 080-099 16 16 9 20 20 10 F F F <---
18882024-03-06T20:58:45.030Z 5 100-119 16 16 1 21 21 1 F F F <---
18892024-03-06T20:58:45.030Z 6 120-139 12 12 12 15 15 15 F F F
18902024-03-06T20:58:45.030Z 7 140-159 13 13 13 16 16 16 T T F <---
18912024-03-06T20:58:45.031Z 8 160-179 10 10 10 12 12 12 F F F
18922024-03-06T20:58:45.031Z 9 180-199 9 9 9 10 10 10 F F F
18932024-03-06T20:58:45.031Z 10 200-219 12 12 12 15 15 15 F F F
18942024-03-06T20:58:45.031Z 11 220-239 16 16 15 22 22 19 F F F <---
18952024-03-06T20:58:45.037Z 12 240-259 15 15 15 19 19 19 T T F <---
18962024-03-06T20:58:45.037Z 13 260-279 10 10 10 12 12 12 F F F
18972024-03-06T20:58:45.037Z 14 280-299 14 14 14 17 17 17 F F F
18982024-03-06T20:58:45.037Z 15 300-319 16 16 12 20 20 14 F F F <---
18992024-03-06T20:58:45.037Z 16 320-339 12 12 12 14 14 14 F F F
19002024-03-06T20:58:45.037Z 17 340-359 10 10 10 12 12 12 F F F
19012024-03-06T20:58:45.037Z 18 360-379 16 16 14 22 22 17 F F F <---
19022024-03-06T20:58:45.037Z 19 380-399 14 14 14 17 17 17 F F F
19032024-03-06T20:58:45.037Z 20 400-419 14 14 14 17 17 17 F F F
19042024-03-06T20:58:45.037Z 21 420-439 14 14 14 17 17 17 F F F
19052024-03-06T20:58:45.037Z 22 440-459 16 16 14 20 20 17 F F T <---
19062024-03-06T20:58:45.037Z 23 460-479 14 14 14 17 17 17 F F F
19072024-03-06T20:58:45.038Z 24 480-499 16 16 14 21 21 17 F F F <---
19082024-03-06T20:58:45.038Z 25 500-519 16 16 12 21 21 14 F F F <---
19092024-03-06T20:58:45.038Z 26 520-539 15 15 15 19 19 19 F F F
19102024-03-06T20:58:45.038Z 27 540-559 8 8 8 9 9 9 T T F <---
19112024-03-06T20:58:45.038Z 28 560-579 14 14 14 17 17 17 F F F
19122024-03-06T20:58:45.038Z 29 580-599 16 16 12 22 22 14 F F F <---
19132024-03-06T20:58:45.038ZMax gen: 16, Max flush: 22
19142024-03-06T20:58:45.038ZError: Difference in extent metadata found!
19152024-03-06T20:58:45.038ZA Difference in extent metadata is expected here
19162024-03-06T20:58:45.038ZOn loop 8
19172024-03-06T20:58:45.038Z
19182024-03-06T20:58:45.041ZVerifying data now
19192024-03-06T20:58:49.794ZNo change after verify
19202024-03-06T20:58:49.794ZLoop: 8 Downstairs dump after verify (and repair):
19212024-03-06T20:58:49.808ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
19222024-03-06T20:58:49.808Z 0 000-019 16 16 16 21 21 21 F F F
19232024-03-06T20:58:49.809Z 1 020-039 17 17 17 23 23 23 F F F
19242024-03-06T20:58:49.809Z 2 040-059 17 17 17 23 23 23 F F F
19252024-03-06T20:58:49.809Z 3 060-079 15 15 15 19 19 19 F F F
19262024-03-06T20:58:49.809Z 4 080-099 16 16 16 20 20 20 F F F
19272024-03-06T20:58:49.809Z 5 100-119 16 16 16 21 21 21 F F F
19282024-03-06T20:58:49.809Z 6 120-139 12 12 12 15 15 15 F F F
19292024-03-06T20:58:49.809Z 7 140-159 17 17 17 23 23 23 F F F
19302024-03-06T20:58:49.809Z 8 160-179 10 10 10 12 12 12 F F F
19312024-03-06T20:58:49.809Z 9 180-199 9 9 9 10 10 10 F F F
19322024-03-06T20:58:49.809Z 10 200-219 12 12 12 15 15 15 F F F
19332024-03-06T20:58:49.809Z 11 220-239 16 16 16 22 22 22 F F F
19342024-03-06T20:58:49.809Z 12 240-259 17 17 17 23 23 23 F F F
19352024-03-06T20:58:49.809Z 13 260-279 10 10 10 12 12 12 F F F
19362024-03-06T20:58:49.809Z 14 280-299 14 14 14 17 17 17 F F F
19372024-03-06T20:58:49.813Z 15 300-319 16 16 16 20 20 20 F F F
19382024-03-06T20:58:49.813Z 16 320-339 12 12 12 14 14 14 F F F
19392024-03-06T20:58:49.813Z 17 340-359 10 10 10 12 12 12 F F F
19402024-03-06T20:58:49.813Z 18 360-379 16 16 16 22 22 22 F F F
19412024-03-06T20:58:49.813Z 19 380-399 14 14 14 17 17 17 F F F
19422024-03-06T20:58:49.813Z 20 400-419 14 14 14 17 17 17 F F F
19432024-03-06T20:58:49.813Z 21 420-439 14 14 14 17 17 17 F F F
19442024-03-06T20:58:49.813Z 22 440-459 16 16 16 20 20 20 F F F
19452024-03-06T20:58:49.813Z 23 460-479 14 14 14 17 17 17 F F F
19462024-03-06T20:58:49.813Z 24 480-499 16 16 16 21 21 21 F F F
19472024-03-06T20:58:49.813Z 25 500-519 16 16 16 21 21 21 F F F
19482024-03-06T20:58:49.813Z 26 520-539 15 15 15 19 19 19 F F F
19492024-03-06T20:58:49.813Z 27 540-559 17 17 17 23 23 23 F F F
19502024-03-06T20:58:49.813Z 28 560-579 14 14 14 17 17 17 F F F
19512024-03-06T20:58:49.813Z 29 580-599 16 16 16 22 22 22 F F F
19522024-03-06T20:58:49.814ZMax gen: 17, Max flush: 23
19532024-03-06T20:58:49.814Z
19542024-03-06T20:58:49.814ZBegin loop 9
19552024-03-06T20:58:49.814ZDownstairs to restart: 2
19562024-03-06T20:58:49.831Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.603532136Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208}
19572024-03-06T20:58:49.831Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\nCrucible runtime is spawned
19582024-03-06T20:58:49.836Z debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.603851824Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208}
19592024-03-06T20:58:49.837Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.603899351Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208}
19602024-03-06T20:58:49.837Z{"msg":"Crucible 7193af2e-a1d8-4f67-ad8a-6053c1eda352 has session id: 9c1b324d-8feb-48ad-b149-da15ea18b6b8","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.603928689Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19612024-03-06T20:58:49.837Z{"msg":"Upstairs opts: Upstairs UUID: 7193af2e-a1d8-4f67-ad8a-6053c1eda352, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.603980385Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19622024-03-06T20:58:49.837Z{"msg":"Crucible stats registered with UUID: 7193af2e-a1d8-4f67-ad8a-6053c1eda352","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.604022302Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19632024-03-06T20:58:49.837Z{"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.60405071Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208}
19642024-03-06T20:58:49.837Z{"msg":"7193af2e-a1d8-4f67-ad8a-6053c1eda352 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.604075128Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19652024-03-06T20:58:49.837Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.604110246Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"client":"0","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19662024-03-06T20:58:49.837Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.604162453Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"client":"0","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19672024-03-06T20:58:49.837Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.60419416Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"client":"1","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19682024-03-06T20:58:49.837Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.604234677Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"client":"1","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19692024-03-06T20:58:49.837Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.604303483Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"client":"2","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19702024-03-06T20:58:49.837Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.604359619Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"client":"2","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19712024-03-06T20:58:49.837Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.604383957Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19722024-03-06T20:58:49.837Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.604423964Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19732024-03-06T20:58:49.837Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.604451582Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19742024-03-06T20:58:49.837Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.60448994Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19752024-03-06T20:58:49.837Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.604665818Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19762024-03-06T20:58:49.837Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.604715504Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19772024-03-06T20:58:49.837Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.605286595Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"client":"0","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19782024-03-06T20:58:49.837Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.605327662Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"client":"0","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19792024-03-06T20:58:49.838Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.60536167Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"client":"1","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19802024-03-06T20:58:49.838Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.605403617Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"client":"1","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19812024-03-06T20:58:49.838Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.605697087Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"client":"2","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19822024-03-06T20:58:49.838Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.605733494Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"client":"2","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19832024-03-06T20:58:49.838Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.605928561Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"client":"1","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19842024-03-06T20:58:49.838Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.606065521Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"client":"0","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19852024-03-06T20:58:49.838Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.606392349Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"client":"1","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19862024-03-06T20:58:49.838Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.606428916Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"client":"1","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19872024-03-06T20:58:49.838Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.606447445Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19882024-03-06T20:58:49.838Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.606469123Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"client":"2","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19892024-03-06T20:58:49.838Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.606505231Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"client":"0","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19902024-03-06T20:58:49.838Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.606548578Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"client":"0","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19912024-03-06T20:58:49.838Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.606583316Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19922024-03-06T20:58:49.838Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.606854017Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"client":"2","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19932024-03-06T20:58:49.838Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.606895024Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"client":"2","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19942024-03-06T20:58:49.838Z{"msg":"[0]R flush_numbers[0..12]: [21, 23, 23, 19, 20, 21, 15, 23, 12, 10, 15, 22]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.606911923Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"":"downstairs","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19952024-03-06T20:58:49.838Z{"msg":"[0]R generation[0..12]: [16, 17, 17, 15, 16, 16, 12, 17, 10, 9, 12, 16]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.606926832Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"":"downstairs","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19962024-03-06T20:58:49.838Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.606949661Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"":"downstairs","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19972024-03-06T20:58:49.838Z{"msg":"[1]R flush_numbers[0..12]: [21, 23, 23, 19, 20, 21, 15, 23, 12, 10, 15, 22]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.606964229Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"":"downstairs","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19982024-03-06T20:58:49.838Z{"msg":"[1]R generation[0..12]: [16, 17, 17, 15, 16, 16, 12, 17, 10, 9, 12, 16]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.606989658Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"":"downstairs","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
19992024-03-06T20:58:49.838Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.607005086Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"":"downstairs","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
20002024-03-06T20:58:49.838ZWait for a query_work_queue command to finish before sending IO
20012024-03-06T20:58:49.838Z{"msg":"[2]R flush_numbers[0..12]: [21, 23, 23, 19, 20, 21, 15, 23, 12, 10, 15, 22]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.607036944Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"":"downstairs","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
20022024-03-06T20:58:49.838Z{"msg":"[2]R generation[0..12]: [16, 17, 17, 15, 16, 16, 12, 17, 10, 9, 12, 16]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.607052483Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"":"downstairs","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
20032024-03-06T20:58:49.839Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.607067532Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"":"downstairs","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
20042024-03-06T20:58:49.839Z{"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.607080641Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"":"downstairs","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
20052024-03-06T20:58:49.839Z{"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.60709494Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"":"downstairs","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
20062024-03-06T20:58:49.842Z{"msg":"Next flush: 24","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.60710909Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"":"downstairs","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
20072024-03-06T20:58:49.843Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.607142427Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"":"downstairs","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
20082024-03-06T20:58:49.843Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.607169505Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
20092024-03-06T20:58:49.843Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.607200053Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"client":"0","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
20102024-03-06T20:58:49.843Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.607223222Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"client":"1","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
20112024-03-06T20:58:49.843Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.607257379Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"client":"2","":"downstairs","session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
20122024-03-06T20:58:49.843Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.607283617Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
20132024-03-06T20:58:49.843Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
20142024-03-06T20:58:49.843Z,"time":"2024-03-06T20:58:48.607298506Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
20152024-03-06T20:58:49.843Z{"msg":"7193af2e-a1d8-4f67-ad8a-6053c1eda352 is now active with session: 9c1b324d-8feb-48ad-b149-da15ea18b6b8","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.607326015Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
20162024-03-06T20:58:49.843Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.607342833Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208,"session_id":"9c1b324d-8feb-48ad-b149-da15ea18b6b8"}
20172024-03-06T20:58:49.843Z{"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:48.607355393Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208}
20182024-03-06T20:58:49.843ZLoading write count information from file "/tmp/test_repair_verify.data"
20192024-03-06T20:58:49.843ZRun Repair workload
20202024-03-06T20:58:49.843Z01/30 Write block 329 len 20480 data: 2 2 2 2 2
20212024-03-06T20:58:49.843Z02/30 Write block 244 len 12288 data: 3 3 3
20222024-03-06T20:58:49.843Z03/30 Read block 498 len 4096
20232024-03-06T20:58:49.843Z04/30 Read block 365 len 12288
20242024-03-06T20:58:49.843Z05/30 Read block 51 len 28672
20252024-03-06T20:58:49.843Z06/30 Flush
20262024-03-06T20:58:49.847Z07/30 Write block 495 len 16384 data: 3 3 3 3
20272024-03-06T20:58:49.847Z08/30 Read block 576 len 24576
20282024-03-06T20:58:49.847Z09/30 Write block 128 len 24576 data: 2 2 2 2 3 2
20292024-03-06T20:58:49.847Z10/30 Write block 150 len 8192 data: 3 3
20302024-03-06T20:58:49.847Z11/30 Write block 512 len 4096 data: 2
20312024-03-06T20:58:49.847Z12/30 Read block 400 len 16384
20322024-03-06T20:58:49.850Z13/30 Read block 436 len 8192
20332024-03-06T20:58:49.850Z14/30 Read block 77 len 24576
20342024-03-06T20:58:49.850Z15/30 Write block 42 len 4096 data: 4
20352024-03-06T20:58:49.850Z16/30 Read block 64 len 4096
20362024-03-06T20:58:49.850Z17/30 Read block 331 len 28672
20372024-03-06T20:58:49.853Z18/30 Read block 449 len 16384
20382024-03-06T20:58:49.853Z19/30 Read block 401 len 28672
20392024-03-06T20:58:49.853Z20/30 Read block 280 len 12288
20402024-03-06T20:58:49.856Z21/30 Write block 26 len 40960 data: 2 3 3 3 3 3 3 4 4 4
20412024-03-06T20:58:49.856Z22/30 Read block 267 len 32768
20422024-03-06T20:58:49.856Z23/30 Write block 544 len 36864 data: 4 4 4 4 4 4 3 3 2
20432024-03-06T20:58:49.860Z24/30 Write block 487 len 40960 data: 4 4 5 3 4 4 4 4 4 4
20442024-03-06T20:58:49.860Z25/30 Flush
20452024-03-06T20:58:49.866Z26/30 Write block 208 len 32768 data: 2 2 3 3 3 3 3 4
20462024-03-06T20:58:49.870Z27/30 Read block 275 len 12288
20472024-03-06T20:58:49.870Z28/30 Write block 148 len 16384 data: 3 3 4 4
20482024-03-06T20:58:49.870Z29/30 Read block 37 len 24576
20492024-03-06T20:58:49.870Z30/30 Read block 516 len 24576
20502024-03-06T20:58:49.870Z----------------------------------------------------------------
20512024-03-06T20:58:49.870Z Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:30
20522024-03-06T20:58:49.870ZStates: Active Active Active
20532024-03-06T20:58:49.870ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
20542024-03-06T20:58:49.870Z 1 Acked 1000 Write 5 Done Done Done false
20552024-03-06T20:58:49.870Z 2 Acked 1001 Write 3 Done Done Done false
20562024-03-06T20:58:49.870Z 3 Acked 1002 Read 1 Done Done Done false
20572024-03-06T20:58:49.871Z 4 Acked 1003 Read 3 Done Done Done false
20582024-03-06T20:58:49.871Z 5 Acked 1004 Read 7 Done Done Sent false
20592024-03-06T20:58:49.871Z 6 Acked 1005 Flush 0 Done Done Sent false
20602024-03-06T20:58:49.871Z 7 Acked 1006 Write 4 Done Done Sent false
20612024-03-06T20:58:49.871Z 8 Acked 1007 Read 6 Done Done Sent false
20622024-03-06T20:58:49.871Z 9 Acked 1008 Write 6 Done Done Sent false
20632024-03-06T20:58:49.871Z 10 Acked 1009 Write 2 Done Done Sent false
20642024-03-06T20:58:49.871Z 11 Acked 1010 Write 1 Done Done Sent false
20652024-03-06T20:58:49.871Z 12 Acked 1011 Read 4 Done Done Sent false
20662024-03-06T20:58:49.871Z 13 Acked 1012 Read 2 Done Done Sent false
20672024-03-06T20:58:49.871Z 14 Acked 1013 Read 6 Done Done Sent false
20682024-03-06T20:58:49.871Z 15 Acked 1014 Write 1 Done Done Sent false
20692024-03-06T20:58:49.871Z 16 Acked 1015 Read 1 Done Done Sent false
20702024-03-06T20:58:49.871Z 17 Acked 1016 Read 7 Done Done Sent false
20712024-03-06T20:58:49.871Z 18 Acked 1017 Read 4 Done Done Sent false
20722024-03-06T20:58:49.871Z 19 Acked 1018 Read 7 Done Done Sent false
20732024-03-06T20:58:49.871Z 20 Acked 1019 Read 3 Done Done Sent false
20742024-03-06T20:58:49.871Z 21 Acked 1020 Write 10 Done Done Sent false
20752024-03-06T20:58:49.871Z 22 Acked 1021 Read 8 Done Done Sent false
20762024-03-06T20:58:49.871Z 23 Acked 1022 Write 9 Done Done Sent false
20772024-03-06T20:58:49.871Z 24 Acked 1023 Write 10 Done Done Sent false
20782024-03-06T20:58:49.871Z 25 Acked 1024 Flush 0 Done Done Sent false
20792024-03-06T20:58:49.871Z 26 Acked 1025 Write 8 Done Done Sent false
20802024-03-06T20:58:49.871Z 27 Acked 1026 Read 3 Done Done Sent false
20812024-03-06T20:58:49.871Z 28 Acked 1027 Write 4 Done Done Sent false
20822024-03-06T20:58:49.872Z 29 Acked 1028 Read 6 Done Done Sent false
20832024-03-06T20:58:49.872Z 30 Acked 1029 Read 6 Done Sent Sent false
20842024-03-06T20:58:49.872Z STATES DS:0 DS:1 DS:2 TOTAL
20852024-03-06T20:58:49.872Z New 0 0 0 0
20862024-03-06T20:58:49.872Z Sent 0 1 26 27
20872024-03-06T20:58:49.872Z Done 30 29 4 63
20882024-03-06T20:58:49.872Z Skipped 0 0 0 0
20892024-03-06T20:58:49.872Z Error 0 0 0 0
20902024-03-06T20:58:49.872ZLast Flush: 1024 1024 0
20912024-03-06T20:58:49.872ZDownstairs last five completed:
20922024-03-06T20:58:49.872ZUpstairs last five completed: 30 29 28 27 26
20932024-03-06T20:58:49.872Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T20:58:48.640549675Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1208}
20942024-03-06T20:58:49.872ZWrote out file "/tmp/test_repair_verify.data"
20952024-03-06T20:58:49.875Z
20962024-03-06T20:58:49.878ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
20972024-03-06T20:58:49.894ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
20982024-03-06T20:58:49.894Z 0 000-019 16 16 16 21 21 21 F F F
20992024-03-06T20:58:49.894Z 1 020-039 18 18 17 25 25 23 F F F <---
21002024-03-06T20:58:49.894Z 2 040-059 18 18 17 25 25 23 F F F <---
21012024-03-06T20:58:49.894Z 3 060-079 15 15 15 19 19 19 F F F
21022024-03-06T20:58:49.894Z 4 080-099 16 16 16 20 20 20 F F F
21032024-03-06T20:58:49.894Z 5 100-119 16 16 16 21 21 21 F F F
21042024-03-06T20:58:49.894Z 6 120-139 18 18 12 25 25 15 F F F <---
21052024-03-06T20:58:49.894Z 7 140-159 18 18 17 25 25 23 T T F <---
21062024-03-06T20:58:49.894Z 8 160-179 10 10 10 12 12 12 F F F
21072024-03-06T20:58:49.902Z 9 180-199 9 9 9 10 10 10 F F F
21082024-03-06T20:58:49.902Z 10 200-219 12 12 12 15 15 15 T T F <---
21092024-03-06T20:58:49.902Z 11 220-239 16 16 16 22 22 22 F F F
21102024-03-06T20:58:49.902Z 12 240-259 18 18 17 24 24 23 F F T <---
21112024-03-06T20:58:49.903Z 13 260-279 10 10 10 12 12 12 F F F
21122024-03-06T20:58:49.903Z 14 280-299 14 14 14 17 17 17 F F F
21132024-03-06T20:58:49.903Z 15 300-319 16 16 16 20 20 20 F F F
21142024-03-06T20:58:49.903Z 16 320-339 18 18 12 24 24 14 F F T <---
21152024-03-06T20:58:49.903Z 17 340-359 10 10 10 12 12 12 F F F
21162024-03-06T20:58:49.903Z 18 360-379 16 16 16 22 22 22 F F F
21172024-03-06T20:58:49.903Z 19 380-399 14 14 14 17 17 17 F F F
21182024-03-06T20:58:49.903Z 20 400-419 14 14 14 17 17 17 F F F
21192024-03-06T20:58:49.903Z 21 420-439 14 14 14 17 17 17 F F F
21202024-03-06T20:58:49.903Z 22 440-459 16 16 16 20 20 20 F F F
21212024-03-06T20:58:49.903Z 23 460-479 14 14 14 17 17 17 F F F
21222024-03-06T20:58:49.903Z 24 480-499 18 18 16 25 25 21 F F F <---
21232024-03-06T20:58:49.903Z 25 500-519 18 18 16 25 25 21 F F F <---
21242024-03-06T20:58:49.903Z 26 520-539 15 15 15 19 19 19 F F F
21252024-03-06T20:58:49.903Z 27 540-559 18 18 17 25 25 23 F F F <---
21262024-03-06T20:58:49.903Z 28 560-579 14 14 14 17 17 17 F F F
21272024-03-06T20:58:49.903Z 29 580-599 16 16 16 22 22 22 F F F
21282024-03-06T20:58:49.903ZMax gen: 18, Max flush: 25
21292024-03-06T20:58:49.903ZError: Difference in extent metadata found!
21302024-03-06T20:58:49.903ZA Difference in extent metadata is expected here
21312024-03-06T20:58:49.903ZOn loop 9
21322024-03-06T20:58:49.903Z
21332024-03-06T20:58:49.903ZVerifying data now
21342024-03-06T20:58:54.849ZNo change after verify
21352024-03-06T20:58:54.849ZLoop: 9 Downstairs dump after verify (and repair):
21362024-03-06T20:58:54.898ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
21372024-03-06T20:58:54.899Z 0 000-019 16 16 16 21 21 21 F F F
21382024-03-06T20:58:54.899Z 1 020-039 18 18 18 25 25 25 F F F
21392024-03-06T20:58:54.899Z 2 040-059 18 18 18 25 25 25 F F F
21402024-03-06T20:58:54.899Z 3 060-079 15 15 15 19 19 19 F F F
21412024-03-06T20:58:54.899Z 4 080-099 16 16 16 20 20 20 F F F
21422024-03-06T20:58:54.900Z 5 100-119 16 16 16 21 21 21 F F F
21432024-03-06T20:58:54.900Z 6 120-139 18 18 18 25 25 25 F F F
21442024-03-06T20:58:54.900Z 7 140-159 19 19 19 26 26 26 F F F
21452024-03-06T20:58:54.900Z 8 160-179 10 10 10 12 12 12 F F F
21462024-03-06T20:58:54.900Z 9 180-199 9 9 9 10 10 10 F F F
21472024-03-06T20:58:54.900Z 10 200-219 19 19 19 26 26 26 F F F
21482024-03-06T20:58:54.901Z 11 220-239 16 16 16 22 22 22 F F F
21492024-03-06T20:58:54.901Z 12 240-259 18 18 18 24 24 24 F F F
21502024-03-06T20:58:54.901Z 13 260-279 10 10 10 12 12 12 F F F
21512024-03-06T20:58:54.901Z 14 280-299 14 14 14 17 17 17 F F F
21522024-03-06T20:58:54.901Z 15 300-319 16 16 16 20 20 20 F F F
21532024-03-06T20:58:54.901Z 16 320-339 18 18 18 24 24 24 F F F
21542024-03-06T20:58:54.902Z 17 340-359 10 10 10 12 12 12 F F F
21552024-03-06T20:58:54.902Z 18 360-379 16 16 16 22 22 22 F F F
21562024-03-06T20:58:54.902Z 19 380-399 14 14 14 17 17 17 F F F
21572024-03-06T20:58:54.902Z 20 400-419 14 14 14 17 17 17 F F F
21582024-03-06T20:58:54.902Z 21 420-439 14 14 14 17 17 17 F F F
21592024-03-06T20:58:54.902Z 22 440-459 16 16 16 20 20 20 F F F
21602024-03-06T20:58:54.902Z 23 460-479 14 14 14 17 17 17 F F F
21612024-03-06T20:58:54.903Z 24 480-499 18 18 18 25 25 25 F F F
21622024-03-06T20:58:54.903Z 25 500-519 18 18 18 25 25 25 F F F
21632024-03-06T20:58:54.903Z 26 520-539 15 15 15 19 19 19 F F F
21642024-03-06T20:58:54.903Z 27 540-559 18 18 18 25 25 25 F F F
21652024-03-06T20:58:54.903Z 28 560-579 14 14 14 17 17 17 F F F
21662024-03-06T20:58:54.903Z 29 580-599 16 16 16 22 22 22 F F F
21672024-03-06T20:58:54.903ZMax gen: 19, Max flush: 26
21682024-03-06T20:58:54.903Z
21692024-03-06T20:58:54.904ZBegin loop 10
21702024-03-06T20:58:54.904ZDownstairs to restart: 0
21712024-03-06T20:58:54.904Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.245136971Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216}
21722024-03-06T20:58:54.904Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.245406513Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216}
21732024-03-06T20:58:54.904Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.245432891Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216}
21742024-03-06T20:58:54.904ZCrucible runtime is spawned
21752024-03-06T20:58:54.904Z{"msg":"Crucible 05fbf607-83f7-49c3-a073-f67bbe0fd2e4 has session id: 31c8f1f1-65ca-433b-aae7-ade32a30e636","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.245482917Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
21762024-03-06T20:58:54.904Z{"msg":"Upstairs opts: Upstairs UUID: 05fbf607-83f7-49c3-a073-f67bbe0fd2e4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.245524665Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
21772024-03-06T20:58:54.905Z{"msg":"Crucible stats registered with UUID: 05fbf607-83f7-49c3-a073-f67bbe0fd2e4","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.245565862Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
21782024-03-06T20:58:54.905Z{"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.245582201Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216}
21792024-03-06T20:58:54.905Z{"msg":"05fbf607-83f7-49c3-a073-f67bbe0fd2e4 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.245712832Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
21802024-03-06T20:58:54.905Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.24573995Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"client":"0","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
21812024-03-06T20:58:54.905Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.245755659Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"client":"0","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
21822024-03-06T20:58:54.905Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.245770008Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"client":"1","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
21832024-03-06T20:58:54.906Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.245800056Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"client":"1","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
21842024-03-06T20:58:54.906Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.245823204Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"client":"2","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
21852024-03-06T20:58:54.906Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.245854092Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"client":"2","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
21862024-03-06T20:58:54.906Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.24588249Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
21872024-03-06T20:58:54.906Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.245917458Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
21882024-03-06T20:58:54.906Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.245933737Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
21892024-03-06T20:58:54.906Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.246068007Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
21902024-03-06T20:58:54.907Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.246108945Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
21912024-03-06T20:58:54.907Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.246137672Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
21922024-03-06T20:58:54.907Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.246884341Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"client":"2","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
21932024-03-06T20:58:54.907Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.246926228Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"client":"2","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
21942024-03-06T20:58:54.907Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.246952796Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"client":"1","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
21952024-03-06T20:58:54.907Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.246987914Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"client":"1","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
21962024-03-06T20:58:54.907Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.247377367Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"client":"0","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
21972024-03-06T20:58:54.907Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.247420304Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"client":"0","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
21982024-03-06T20:58:54.907Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.247452392Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"client":"1","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
21992024-03-06T20:58:54.908Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.247670027Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"client":"2","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22002024-03-06T20:58:54.908Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.247946928Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"client":"1","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22012024-03-06T20:58:54.908Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.247990745Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"client":"1","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22022024-03-06T20:58:54.908Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.24805228Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22032024-03-06T20:58:54.908Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.248079109Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"client":"0","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22042024-03-06T20:58:54.908Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.248117686Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"client":"2","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22052024-03-06T20:58:54.908Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.248135015Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"client":"2","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22062024-03-06T20:58:54.909Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.248164963Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22072024-03-06T20:58:54.909Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.248427705Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"client":"0","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22082024-03-06T20:58:54.909Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.248463002Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"client":"0","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22092024-03-06T20:58:54.909Z{"msg":"[0]R flush_numbers[0..12]: [21, 25, 25, 19, 20, 21, 25, 26, 12, 10, 26, 22]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.248478961Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"":"downstairs","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22102024-03-06T20:58:54.909Z{"msg":"[0]R generation[0..12]: [16, 18, 18, 15, 16, 16, 18, 19, 10, 9, 19, 16]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.24849393Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"":"downstairs","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22112024-03-06T20:58:54.909Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.248516139Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"":"downstairs","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22122024-03-06T20:58:54.909Z{"msg":"[1]R flush_numbers[0..12]: [21, 25, 25, 19, 20, 21, 25, 26, 12, 10, 26, 22]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.248530338Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"":"downstairs","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22132024-03-06T20:58:54.909Z{"msg":"[1]R generation[0..12]: [16, 18, 18, 15, 16, 16, 18, 19, 10, 9, 19, 16]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.248556316Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"":"downstairs","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22142024-03-06T20:58:54.910Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.248570935Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"":"downstairs","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22152024-03-06T20:58:54.910ZWait for a query_work_queue command to finish before sending IO
22162024-03-06T20:58:54.910Z{"msg":"[2]R flush_numbers[0..12]: [21, 25, 25, 19, 20, 21, 25, 26, 12, 10, 26, 22]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.248593753Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"":"downstairs","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22172024-03-06T20:58:54.910Z{"msg":"[2]R generation[0..12]: [16, 18, 18, 15, 16, 16, 18, 19, 10, 9, 19, 16]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.248608322Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"":"downstairs","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22182024-03-06T20:58:54.910Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.248628851Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"":"downstairs","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22192024-03-06T20:58:54.910Z{"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.248653629Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"":"downstairs","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22202024-03-06T20:58:54.910Z{"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.248667128Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"":"downstairs","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22212024-03-06T20:58:54.911Z{"msg":"Next flush: 27","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.248680117Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"":"downstairs","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22222024-03-06T20:58:54.911Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.248693396Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"":"downstairs","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22232024-03-06T20:58:54.911Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.248712955Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22242024-03-06T20:58:54.911Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.248725014Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"client":"0","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22252024-03-06T20:58:54.911Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.248744543Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"client":"1","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22262024-03-06T20:58:54.911Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.248757932Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"client":"2","":"downstairs","session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22272024-03-06T20:58:54.911Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.248778521Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22282024-03-06T20:58:54.911Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.248790869Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22292024-03-06T20:58:54.911Z{"msg":"05fbf607-83f7-49c3-a073-f67bbe0fd2e4 is now active with session: 31c8f1f1-65ca-433b-aae7-ade32a30e636","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.248809948Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22302024-03-06T20:58:54.912Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.248822178Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216,"session_id":"31c8f1f1-65ca-433b-aae7-ade32a30e636"}
22312024-03-06T20:58:54.912Z{"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:53.248840676Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216}
22322024-03-06T20:58:54.912ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
22332024-03-06T20:58:54.912ZLoading write count information from file "/tmp/test_repair_verify.data"
22342024-03-06T20:58:54.912ZRun Repair workload
22352024-03-06T20:58:54.912Z01/30 Write block 457 len 28672 data: 3 3 3 4 5 5 5
22362024-03-06T20:58:54.959Z02/30 Read block 410 len 8192
22372024-03-06T20:58:54.959Z03/30 Read block 124 len 12288
22382024-03-06T20:58:54.959Z04/30 Read block 42 len 12288
22392024-03-06T20:58:54.959Z05/30 Write block 397 len 16384 data: 3 3 2 2
22402024-03-06T20:58:54.959Z06/30 Read block 503 len 40960
22412024-03-06T20:58:54.959Z07/30 Write block 47 len 36864 data: 3 2 2 2 2 2 2 2 2
22422024-03-06T20:58:54.959Z08/30 Flush
22432024-03-06T20:58:54.960Z09/30 Write block 237 len 4096 data: 5
22442024-03-06T20:58:54.960Z10/30 Write block 107 len 4096 data: 4
22452024-03-06T20:58:54.960Z11/30 Read block 497 len 32768
22462024-03-06T20:58:54.960Z12/30 Write block 233 len 36864 data: 3 4 5 5 6 6 5 4 5
22472024-03-06T20:58:54.960Z13/30 Read block 525 len 12288
22482024-03-06T20:58:54.960Z14/30 Read block 253 len 32768
22492024-03-06T20:58:54.960Z15/30 Write block 253 len 12288 data: 3 3 3
22502024-03-06T20:58:54.960Z16/30 Write block 61 len 20480 data: 3 4 4 4 4
22512024-03-06T20:58:54.960Z17/30 Write block 242 len 4096 data: 4
22522024-03-06T20:58:54.960Z18/30 Read block 454 len 16384
22532024-03-06T20:58:54.960Z19/30 Flush
22542024-03-06T20:58:54.960Z20/30 Write block 22 len 8192 data: 2 3
22552024-03-06T20:58:54.960Z21/30 Write block 59 len 32768 data: 2 3 4 5 5 5 5 5
22562024-03-06T20:58:54.960Z22/30 Write block 86 len 36864 data: 2 2 2 2 2 2 2 2 2
22572024-03-06T20:58:54.960Z23/30 Flush
22582024-03-06T20:58:54.960Z24/30 Write block 98 len 20480 data: 3 3 3 4 4
22592024-03-06T20:58:54.960Z25/30 Read block 448 len 24576
22602024-03-06T20:58:54.960Z26/30 Read block 284 len 32768
22612024-03-06T20:58:54.960Z27/30 Read block 34 len 4096
22622024-03-06T20:58:54.960Z28/30 Read block 476 len 8192
22632024-03-06T20:58:54.960Z29/30 Read block 543 len 8192
22642024-03-06T20:58:54.960Z30/30 Write block 329 len 4096 data: 3
22652024-03-06T20:58:54.960Z----------------------------------------------------------------
22662024-03-06T20:58:54.960Z Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30
22672024-03-06T20:58:54.961ZStates: Active Active Active
22682024-03-06T20:58:54.961ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
22692024-03-06T20:58:54.961Z 1 Acked 1000 Write 7 Sent Done Done false
22702024-03-06T20:58:54.961Z 2 Acked 1001 Read 2 Sent Done Done false
22712024-03-06T20:58:54.961Z 3 Acked 1002 Read 3 Sent Done Done false
22722024-03-06T20:58:54.961Z 4 Acked 1003 Read 3 Sent Done Done false
22732024-03-06T20:58:54.961Z 5 Acked 1004 Write 4 Sent Done Done false
22742024-03-06T20:58:54.961Z 6 Acked 1005 Read 10 Sent Done Done false
22752024-03-06T20:58:54.961Z 7 Acked 1006 Write 9 Sent Done Done false
22762024-03-06T20:58:54.961Z 8 Acked 1007 Flush 0 Sent Done Done false
22772024-03-06T20:58:54.961Z 9 Acked 1008 Write 1 Sent Done Done false
22782024-03-06T20:58:54.961Z 10 Acked 1009 Write 1 Sent Done Done false
22792024-03-06T20:58:54.961Z 11 Acked 1010 Read 8 Sent Done Done false
22802024-03-06T20:58:54.961Z 12 Acked 1011 Write 9 Sent Done Done false
22812024-03-06T20:58:54.961Z 13 Acked 1012 Read 3 Sent Done Done false
22822024-03-06T20:58:54.961Z 14 Acked 1013 Read 8 Sent Done Done false
22832024-03-06T20:58:54.961Z 15 Acked 1014 Write 3 Sent Done Done false
22842024-03-06T20:58:54.961Z 16 Acked 1015 Write 5 Sent Done Done false
22852024-03-06T20:58:54.961Z 17 Acked 1016 Write 1 Sent Done Done false
22862024-03-06T20:58:54.961Z 18 Acked 1017 Read 4 Sent Done Done false
22872024-03-06T20:58:54.961Z 19 Acked 1018 Flush 0 Sent Done Done false
22882024-03-06T20:58:54.961Z 20 Acked 1019 Write 2 Sent Done Done false
22892024-03-06T20:58:54.961Z 21 Acked 1020 Write 8 Sent Done Done false
22902024-03-06T20:58:54.961Z 22 Acked 1021 Write 9 Sent Done Done false
22912024-03-06T20:58:54.962Z 23 Acked 1022 Flush 0 Sent Done Done false
22922024-03-06T20:58:54.962Z 24 Acked 1023 Write 5 Sent Done Done false
22932024-03-06T20:58:54.962Z 25 Acked 1024 Read 6 Sent Done Done false
22942024-03-06T20:58:54.962Z 26 Acked 1025 Read 8 Sent Done Done false
22952024-03-06T20:58:54.962Z 27 Acked 1026 Read 1 Sent Done Done false
22962024-03-06T20:58:54.962Z 28 Acked 1027 Read 2 Sent Done Done false
22972024-03-06T20:58:54.962Z 29 Acked 1028 Read 2 Sent Done Done false
22982024-03-06T20:58:54.962Z 30 Acked 1029 Write 1 Sent Sent Sent false
22992024-03-06T20:58:54.962Z STATES DS:0 DS:1 DS:2 TOTAL
23002024-03-06T20:58:54.962Z New 0 0 0 0
23012024-03-06T20:58:54.962Z Sent 30 1 1 32
23022024-03-06T20:58:54.962Z Done 0 29 29 58
23032024-03-06T20:58:54.962Z Skipped 0 0 0 0
23042024-03-06T20:58:54.962Z Error 0 0 0 0
23052024-03-06T20:58:54.962ZLast Flush: 0 1022 1022
23062024-03-06T20:58:54.962ZDownstairs last five completed:
23072024-03-06T20:58:54.962ZUpstairs last five completed: 30 29 28 27 26
23082024-03-06T20:58:54.962Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T20:58:53.282901859Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1216}
23092024-03-06T20:58:54.962ZWrote out file "/tmp/test_repair_verify.data"
23102024-03-06T20:58:54.962Z
23112024-03-06T20:58:54.962ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
23122024-03-06T20:58:54.962ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23132024-03-06T20:58:54.962Z 0 000-019 16 16 16 21 21 21 F F F
23142024-03-06T20:58:54.962Z 1 020-039 18 20 20 25 29 29 F F F <---
23152024-03-06T20:58:54.963Z 2 040-059 18 20 20 25 29 29 F F F <---
23162024-03-06T20:58:54.963Z 3 060-079 15 20 20 19 29 29 F F F <---
23172024-03-06T20:58:54.963Z 4 080-099 16 20 20 20 29 29 F T T <---
23182024-03-06T20:58:54.963Z 5 100-119 16 20 20 21 28 28 F T T <---
23192024-03-06T20:58:54.963Z 6 120-139 18 18 18 25 25 25 F F F
23202024-03-06T20:58:54.963Z 7 140-159 19 19 19 26 26 26 F F F
23212024-03-06T20:58:54.963Z 8 160-179 10 10 10 12 12 12 F F F
23222024-03-06T20:58:54.963Z 9 180-199 9 9 9 10 10 10 F F F
23232024-03-06T20:58:54.963Z 10 200-219 19 19 19 26 26 26 F F F
23242024-03-06T20:58:54.963Z 11 220-239 16 20 20 22 28 28 F F F <---
23252024-03-06T20:58:54.963Z 12 240-259 18 20 20 24 28 28 F F F <---
23262024-03-06T20:58:54.963Z 13 260-279 10 10 10 12 12 12 F F F
23272024-03-06T20:58:54.963Z 14 280-299 14 14 14 17 17 17 F F F
23282024-03-06T20:58:54.963Z 15 300-319 16 16 16 20 20 20 F F F
23292024-03-06T20:58:54.963Z 16 320-339 18 18 18 24 24 24 F T T <---
23302024-03-06T20:58:54.963Z 17 340-359 10 10 10 12 12 12 F F F
23312024-03-06T20:58:54.963Z 18 360-379 16 16 16 22 22 22 F F F
23322024-03-06T20:58:54.963Z 19 380-399 14 20 20 17 27 27 F F F <---
23332024-03-06T20:58:54.963Z 20 400-419 14 20 20 17 27 27 F F F <---
23342024-03-06T20:58:54.963Z 21 420-439 14 14 14 17 17 17 F F F
23352024-03-06T20:58:54.963Z 22 440-459 16 20 20 20 27 27 F F F <---
23362024-03-06T20:58:54.973Z 23 460-479 14 20 20 17 27 27 F F F <---
23372024-03-06T20:58:54.973Z 24 480-499 18 18 18 25 25 25 F F F
23382024-03-06T20:58:54.973Z 25 500-519 18 18 18 25 25 25 F F F
23392024-03-06T20:58:54.973Z 26 520-539 15 15 15 19 19 19 F F F
23402024-03-06T20:58:54.974Z 27 540-559 18 18 18 25 25 25 F F F
23412024-03-06T20:58:54.974Z 28 560-579 14 14 14 17 17 17 F F F
23422024-03-06T20:58:54.974Z 29 580-599 16 16 16 22 22 22 F F F
23432024-03-06T20:58:54.974ZMax gen: 20, Max flush: 29
23442024-03-06T20:58:54.974ZError: Difference in extent metadata found!
23452024-03-06T20:58:54.974ZA Difference in extent metadata is expected here
23462024-03-06T20:58:54.974ZOn loop 10
23472024-03-06T20:58:54.974Z
23482024-03-06T20:58:54.974ZVerifying data now
23492024-03-06T20:58:59.171ZNo change after verify
23502024-03-06T20:58:59.172ZLoop: 10 Downstairs dump after verify (and repair):
23512024-03-06T20:58:59.186ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23522024-03-06T20:58:59.186Z 0 000-019 16 16 16 21 21 21 F F F
23532024-03-06T20:58:59.187Z 1 020-039 20 20 20 29 29 29 F F F
23542024-03-06T20:58:59.187Z 2 040-059 20 20 20 29 29 29 F F F
23552024-03-06T20:58:59.187Z 3 060-079 20 20 20 29 29 29 F F F
23562024-03-06T20:58:59.187Z 4 080-099 21 21 21 30 30 30 F F F
23572024-03-06T20:58:59.187Z 5 100-119 21 21 21 30 30 30 F F F
23582024-03-06T20:58:59.187Z 6 120-139 18 18 18 25 25 25 F F F
23592024-03-06T20:58:59.195Z 7 140-159 19 19 19 26 26 26 F F F
23602024-03-06T20:58:59.195Z 8 160-179 10 10 10 12 12 12 F F F
23612024-03-06T20:58:59.196Z 9 180-199 9 9 9 10 10 10 F F F
23622024-03-06T20:58:59.196Z 10 200-219 19 19 19 26 26 26 F F F
23632024-03-06T20:58:59.196Z 11 220-239 20 20 20 28 28 28 F F F
23642024-03-06T20:58:59.196Z 12 240-259 20 20 20 28 28 28 F F F
23652024-03-06T20:58:59.196Z 13 260-279 10 10 10 12 12 12 F F F
23662024-03-06T20:58:59.196Z 14 280-299 14 14 14 17 17 17 F F F
23672024-03-06T20:58:59.196Z 15 300-319 16 16 16 20 20 20 F F F
23682024-03-06T20:58:59.196Z 16 320-339 21 21 21 30 30 30 F F F
23692024-03-06T20:58:59.196Z 17 340-359 10 10 10 12 12 12 F F F
23702024-03-06T20:58:59.196Z 18 360-379 16 16 16 22 22 22 F F F
23712024-03-06T20:58:59.196Z 19 380-399 20 20 20 27 27 27 F F F
23722024-03-06T20:58:59.196Z 20 400-419 20 20 20 27 27 27 F F F
23732024-03-06T20:58:59.196Z 21 420-439 14 14 14 17 17 17 F F F
23742024-03-06T20:58:59.196Z 22 440-459 20 20 20 27 27 27 F F F
23752024-03-06T20:58:59.197Z 23 460-479 20 20 20 27 27 27 F F F
23762024-03-06T20:58:59.197Z 24 480-499 18 18 18 25 25 25 F F F
23772024-03-06T20:58:59.197Z 25 500-519 18 18 18 25 25 25 F F F
23782024-03-06T20:58:59.197Z 26 520-539 15 15 15 19 19 19 F F F
23792024-03-06T20:58:59.197Z 27 540-559 18 18 18 25 25 25 F F F
23802024-03-06T20:58:59.197Z 28 560-579 14 14 14 17 17 17 F F F
23812024-03-06T20:58:59.197Z 29 580-599 16 16 16 22 22 22 F F F
23822024-03-06T20:58:59.197ZMax gen: 21, Max flush: 30
23832024-03-06T20:58:59.197Z
23842024-03-06T20:58:59.197ZBegin loop 11
23852024-03-06T20:58:59.197ZDownstairs to restart: 1
23862024-03-06T20:58:59.208Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.974238237Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224}
23872024-03-06T20:58:59.208Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.974491859Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224}
23882024-03-06T20:58:59.655Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.974512658Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224}
23892024-03-06T20:58:59.655Z{"msg":"Crucible 5d1bbb42-6862-4564-b153-32d43707a695 has session id: 9ac23876-4e7e-481a-ab39-5c3aaf33ae56","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.974542906Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
23902024-03-06T20:58:59.655Z{"msg":"Upstairs opts: Upstairs UUID: 5d1bbb42-6862-4564-b153-32d43707a695, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.974579233Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","Crucible runtime is spawned
23912024-03-06T20:58:59.655Zpid":1224,"session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
23922024-03-06T20:58:59.655Z{"msg":"Crucible stats registered with UUID: 5d1bbb42-6862-4564-b153-32d43707a695","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.97462251Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
23932024-03-06T20:58:59.655Z{"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.974683116Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224}
23942024-03-06T20:58:59.655Z{"msg":"5d1bbb42-6862-4564-b153-32d43707a695 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.974803988Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
23952024-03-06T20:58:59.655Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.974824516Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"client":"0","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
23962024-03-06T20:58:59.655Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.974839815Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"client":"0","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
23972024-03-06T20:58:59.655Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.974862584Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"client":"1","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
23982024-03-06T20:58:59.655Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.974877463Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"client":"1","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
23992024-03-06T20:58:59.655Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.974900631Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"client":"2","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24002024-03-06T20:58:59.655Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.974925939Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"client":"2","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24012024-03-06T20:58:59.655Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.974947578Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24022024-03-06T20:58:59.655Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.974972476Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24032024-03-06T20:58:59.655Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.975001634Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24042024-03-06T20:58:59.655Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.975174742Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24052024-03-06T20:58:59.655Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.97521329Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24062024-03-06T20:58:59.656Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.975240478Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24072024-03-06T20:58:59.656Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.975966078Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"client":"2","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24082024-03-06T20:58:59.656Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.976020444Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"client":"2","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24092024-03-06T20:58:59.656Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.97606852Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"client":"0","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24102024-03-06T20:58:59.656Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.976105208Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"client":"0","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24112024-03-06T20:58:59.656Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.976496151Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"client":"0","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24122024-03-06T20:58:59.656Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.976646771Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"client":"2","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24132024-03-06T20:58:59.656Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.976999027Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"client":"0","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24142024-03-06T20:58:59.656Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.977037494Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"client":"0","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24152024-03-06T20:58:59.656Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.977055093Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24162024-03-06T20:58:59.656Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.977144357Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"client":"2","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24172024-03-06T20:58:59.656Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.977176294Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"client":"2","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24182024-03-06T20:58:59.656Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.977192323Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24192024-03-06T20:58:59.656Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.97723319Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"client":"1","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24202024-03-06T20:58:59.656Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.977256249Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"client":"1","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24212024-03-06T20:58:59.656Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.977703798Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"client":"1","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24222024-03-06T20:58:59.656Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.978089171Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"client":"1","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24232024-03-06T20:58:59.656Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.978136108Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"client":"1","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24242024-03-06T20:58:59.656Z{"msg":"[0]R flush_numbers[0..12]: [21, 29, 29, 29, 30, 30, 25, 26, 12, 10, 26, 28]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.978156217Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"":"downstairs","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24252024-03-06T20:58:59.656Z{"msg":"[0]R generation[0..12]: [16, 20, 20, 20, 21, 21, 18, 19, 10, 9, 19, 20]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.978178975Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"":"downstairs","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24262024-03-06T20:58:59.656Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.978194004Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"":"downstairs","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24272024-03-06T20:58:59.656Z{"msg":"[1]R flush_numbers[0..12]: [21, 29, 29, 29, 30, 30, 25, 26, 12, 10, 26, 28]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.978214533Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"":"downstairs","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24282024-03-06T20:58:59.657Z{"msg":"[1]R generation[0..12]: [16, 20, 20, 20, 21, 21, 18, 19, 10, 9, 19, 20]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.978228832Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"":"downstairs","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24292024-03-06T20:58:59.657Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.978242991Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"":"downstairs","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24302024-03-06T20:58:59.657Z{"msg":"[2]R flush_numbers[0..12]: [21, 29, 29, 29, 30, 30, 25, 26, 12, 10, 26, 28]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.97825672Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"":"downstairs","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24312024-03-06T20:58:59.657Z{"msg":"Wait for a query_work_queue command to finish before sending IO
24322024-03-06T20:58:59.657Z[2]R generation[0..12]: [16, 20, 20, 20, 21, 21, 18, 19, 10, 9, 19, 20]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.978280378Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"":"downstairs","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24332024-03-06T20:58:59.657Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.978295347Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"":"downstairs","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24342024-03-06T20:58:59.657Z{"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.978314856Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"":"downstairs","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24352024-03-06T20:58:59.657Z{"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.978328525Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"":"downstairs","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24362024-03-06T20:58:59.657Z{"msg":"Next flush: 31","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.978348043Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"":"downstairs","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24372024-03-06T20:58:59.657Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.978361293Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"":"downstairs","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24382024-03-06T20:58:59.657Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.978381421Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24392024-03-06T20:58:59.657Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.97839351Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"client":"0","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24402024-03-06T20:58:59.657Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.978407589Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"client":"1","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24412024-03-06T20:58:59.657Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.978420869Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"client":"2","":"downstairs","session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24422024-03-06T20:58:59.657Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.978441017Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24432024-03-06T20:58:59.657Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.978453216Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24442024-03-06T20:58:59.657Z{"msg":"5d1bbb42-6862-4564-b153-32d43707a695 is now active with session: 9ac23876-4e7e-481a-ab39-5c3aaf33ae56","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.978466675Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24452024-03-06T20:58:59.657Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.978484994Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224,"session_id":"9ac23876-4e7e-481a-ab39-5c3aaf33ae56"}
24462024-03-06T20:58:59.657Z{"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:58:57.978497323Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224}
24472024-03-06T20:58:59.657ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
24482024-03-06T20:58:59.657ZLoading write count information from file "/tmp/test_repair_verify.data"
24492024-03-06T20:58:59.657ZRun Repair workload
24502024-03-06T20:58:59.657Z01/30 Write block 136 len 20480 data: 2 2 2 2 2
24512024-03-06T20:58:59.658Z02/30 Read block 388 len 20480
24522024-03-06T20:58:59.658Z03/30 Read block 496 len 4096
24532024-03-06T20:58:59.658Z04/30 Write block 425 len 40960 data: 3 4 4 4 4 4 3 3 3 3
24542024-03-06T20:58:59.658Z05/30 Read block 557 len 8192
24552024-03-06T20:58:59.658Z06/30 Write block 360 len 8192 data: 4 5
24562024-03-06T20:58:59.658Z07/30 Write block 58 len 40960 data: 2 3 4 5 6 6 6 6 6 5
24572024-03-06T20:58:59.658Z08/30 Read block 203 len 16384
24582024-03-06T20:58:59.658Z09/30 Read block 267 len 32768
24592024-03-06T20:58:59.658Z10/30 Write block 106 len 28672 data: 4 5 2 2 2 2 2
24602024-03-06T20:58:59.658Z11/30 Read block 185 len 12288
24612024-03-06T20:58:59.658Z12/30 Read block 189 len 24576
24622024-03-06T20:58:59.658Z13/30 Read block 359 len 28672
24632024-03-06T20:58:59.658Z14/30 Read block 130 len 28672
24642024-03-06T20:58:59.668Z15/30 Read block 308 len 12288
24652024-03-06T20:58:59.668Z16/30 Write block 55 len 36864 data: 3 2 2 3 4 5 6 7 7
24662024-03-06T20:58:59.668Z17/30 Read block 88 len 8192
24672024-03-06T20:58:59.668Z18/30 Write block 281 len 4096 data: 2
24682024-03-06T20:58:59.668Z19/30 Read block 396 len 36864
24692024-03-06T20:58:59.668Z20/30 Write block 216 len 24576 data: 4 4 4 2 2 2
24702024-03-06T20:58:59.669Z21/30 Read block 373 len 20480
24712024-03-06T20:58:59.669Z22/30 Read block 566 len 8192
24722024-03-06T20:58:59.669Z23/30 Flush
24732024-03-06T20:58:59.669Z24/30 Write block 571 len 24576 data: 3 3 3 3 3 3
24742024-03-06T20:58:59.669Z25/30 Read block 469 len 4096
24752024-03-06T20:58:59.669Z26/30 Write block 542 len 24576 data: 5 5 5 5 5 5
24762024-03-06T20:58:59.669Z27/30 Write block 295 len 24576 data: 3 2 2 2 2 2
24772024-03-06T20:58:59.669Z28/30 Write block 57 len 28672 data: 3 4 5 6 7 8 8
24782024-03-06T20:58:59.669Z29/30 Read block 127 len 4096
24792024-03-06T20:58:59.669Z30/30 Write block 328 len 16384 data: 2 4 3 3
24802024-03-06T20:58:59.669Z----------------------------------------------------------------
24812024-03-06T20:58:59.669Z Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:30
24822024-03-06T20:58:59.669ZStates: Active Active Active
24832024-03-06T20:58:59.669ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
24842024-03-06T20:58:59.669Z 1 Acked 1000 Write 5 Done Done Done false
24852024-03-06T20:58:59.669Z 2 Acked 1001 Read 5 Done Done Done false
24862024-03-06T20:58:59.669Z 3 Acked 1002 Read 1 Done Sent Done false
24872024-03-06T20:58:59.669Z 4 Acked 1003 Write 10 Done Sent Done false
24882024-03-06T20:58:59.669Z 5 Acked 1004 Read 2 Done Sent Done false
24892024-03-06T20:58:59.669Z 6 Acked 1005 Write 2 Done Sent Done false
24902024-03-06T20:58:59.669Z 7 Acked 1006 Write 10 Done Sent Done false
24912024-03-06T20:58:59.669Z 8 Acked 1007 Read 4 Done Sent Done false
24922024-03-06T20:58:59.670Z 9 Acked 1008 Read 8 Done Sent Done false
24932024-03-06T20:58:59.670Z 10 Acked 1009 Write 7 Done Sent Done false
24942024-03-06T20:58:59.670Z 11 Acked 1010 Read 3 Done Sent Done false
24952024-03-06T20:58:59.670Z 12 Acked 1011 Read 6 Done Sent Done false
24962024-03-06T20:58:59.670Z 13 Acked 1012 Read 7 Done Sent Done false
24972024-03-06T20:58:59.670Z 14 Acked 1013 Read 7 Done Sent Done false
24982024-03-06T20:58:59.670Z 15 Acked 1014 Read 3 Done Sent Done false
24992024-03-06T20:58:59.670Z 16 Acked 1015 Write 9 Done Sent Done false
25002024-03-06T20:58:59.670Z 17 Acked 1016 Read 2 Done Sent Done false
25012024-03-06T20:58:59.670Z 18 Acked 1017 Write 1 Done Sent Done false
25022024-03-06T20:58:59.670Z 19 Acked 1018 Read 9 Done Sent Done false
25032024-03-06T20:58:59.670Z 20 Acked 1019 Write 6 Done Sent Done false
25042024-03-06T20:58:59.670Z 21 Acked 1020 Read 5 Done Sent Done false
25052024-03-06T20:58:59.670Z 22 Acked 1021 Read 2 Done Sent Done false
25062024-03-06T20:58:59.670Z 23 Acked 1022 Flush 0 Done Sent Done false
25072024-03-06T20:58:59.670Z 24 Acked 1023 Write 6 Done Sent Done false
25082024-03-06T20:58:59.670Z 25 Acked 1024 Read 1 Done Sent Done false
25092024-03-06T20:58:59.670Z 26 Acked 1025 Write 6 Done Sent Done false
25102024-03-06T20:58:59.670Z 27 Acked 1026 Write 6 Done Sent Done false
25112024-03-06T20:58:59.670Z 28 Acked 1027 Write 7 Done Sent Done false
25122024-03-06T20:58:59.670Z 29 Acked 1028 Read 1 Done Sent Done false
25132024-03-06T20:58:59.670Z 30 Acked 1029 Write 4 Sent Sent Sent false
25142024-03-06T20:58:59.670Z STATES DS:0 DS:1 DS:2 TOTAL
25152024-03-06T20:58:59.670Z New 0 0 0 0
25162024-03-06T20:58:59.671Z Sent 1 28 1 30
25172024-03-06T20:58:59.671Z Done 29 2 29 60
25182024-03-06T20:58:59.671Z Skipped 0 0 0 0
25192024-03-06T20:58:59.671Z Error 0 0 0 0
25202024-03-06T20:58:59.671ZLast Flush: 1022 0 1022
25212024-03-06T20:58:59.671ZDownstairs last five completed:
25222024-03-06T20:58:59.671ZUpstairs last five completed: 30 29 28 27 26
25232024-03-06T20:58:59.671Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T20:58:58.011392886Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1224}
25242024-03-06T20:58:59.671ZWrote out file "/tmp/test_repair_verify.data"
25252024-03-06T20:58:59.671Z
25262024-03-06T20:58:59.671ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
25272024-03-06T20:58:59.671ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25282024-03-06T20:58:59.671Z 0 000-019 16 16 16 21 21 21 F F F
25292024-03-06T20:58:59.671Z 1 020-039 20 20 20 29 29 29 F F F
25302024-03-06T20:58:59.671Z 2 040-059 22 20 22 31 29 31 T F T <---
25312024-03-06T20:58:59.671Z 3 060-079 22 20 22 31 29 31 T F T <---
25322024-03-06T20:58:59.671Z 4 080-099 21 21 21 30 30 30 F F F
25332024-03-06T20:58:59.671Z 5 100-119 22 21 22 31 30 31 F F F <---
25342024-03-06T20:58:59.671Z 6 120-139 22 18 22 31 25 31 F T F <---
25352024-03-06T20:58:59.671Z 7 140-159 22 19 22 31 26 31 F T F <---
25362024-03-06T20:58:59.671Z 8 160-179 10 10 10 12 12 12 F F F
25372024-03-06T20:58:59.671Z 9 180-199 9 9 9 10 10 10 F F F
25382024-03-06T20:58:59.671Z 10 200-219 22 19 22 31 26 31 F F F <---
25392024-03-06T20:58:59.672Z 11 220-239 22 20 22 31 28 31 F F F <---
25402024-03-06T20:58:59.672Z 12 240-259 20 20 20 28 28 28 F F F
25412024-03-06T20:58:59.672Z 13 260-279 10 10 10 12 12 12 F F F
25422024-03-06T20:58:59.672Z 14 280-299 22 14 22 31 17 31 T F T <---
25432024-03-06T20:58:59.672Z 15 300-319 16 16 16 20 20 20 T F T <---
25442024-03-06T20:58:59.672Z 16 320-339 21 21 21 30 30 30 T F T <---
25452024-03-06T20:58:59.672Z 17 340-359 10 10 10 12 12 12 F F F
25462024-03-06T20:58:59.672Z 18 360-379 22 16 22 31 22 31 F F F <---
25472024-03-06T20:58:59.672Z 19 380-399 20 20 20 27 27 27 F F F
25482024-03-06T20:58:59.672Z 20 400-419 20 20 20 27 27 27 F F F
25492024-03-06T20:58:59.672Z 21 420-439 22 14 22 31 17 31 F F F <---
25502024-03-06T20:58:59.672Z 22 440-459 20 20 20 27 27 27 F F F
25512024-03-06T20:58:59.672Z 23 460-479 20 20 20 27 27 27 F F F
25522024-03-06T20:58:59.672Z 24 480-499 18 18 18 25 25 25 F F F
25532024-03-06T20:58:59.672Z 25 500-519 18 18 18 25 25 25 F F F
25542024-03-06T20:58:59.672Z 26 520-539 15 15 15 19 19 19 F F F
25552024-03-06T20:58:59.672Z 27 540-559 18 18 18 25 25 25 T F T <---
25562024-03-06T20:58:59.672Z 28 560-579 14 14 14 17 17 17 T F T <---
25572024-03-06T20:58:59.672Z 29 580-599 16 16 16 22 22 22 F F F
25582024-03-06T20:58:59.672ZMax gen: 22, Max flush: 31
25592024-03-06T20:58:59.672ZError: Difference in extent metadata found!
25602024-03-06T20:58:59.672ZA Difference in extent metadata is expected here
25612024-03-06T20:58:59.672ZOn loop 11
25622024-03-06T20:58:59.672Z
25632024-03-06T20:58:59.673ZVerifying data now
25642024-03-06T20:59:04.987ZNo change after verify
25652024-03-06T20:59:05.413ZLoop: 11 Downstairs dump after verify (and repair):
25662024-03-06T20:59:05.414ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25672024-03-06T20:59:05.414Z 0 000-019 16 16 16 21 21 21 F F F
25682024-03-06T20:59:05.414Z 1 020-039 20 20 20 29 29 29 F F F
25692024-03-06T20:59:05.414Z 2 040-059 23 23 23 32 32 32 F F F
25702024-03-06T20:59:05.414Z 3 060-079 23 23 23 32 32 32 F F F
25712024-03-06T20:59:05.414Z 4 080-099 21 21 21 30 30 30 F F F
25722024-03-06T20:59:05.414Z 5 100-119 22 22 22 31 31 31 F F F
25732024-03-06T20:59:05.414Z 6 120-139 22 22 22 31 31 31 F F F
25742024-03-06T20:59:05.414Z 7 140-159 22 22 22 31 31 31 F F F
25752024-03-06T20:59:05.414Z 8 160-179 10 10 10 12 12 12 F F F
25762024-03-06T20:59:05.414Z 9 180-199 9 9 9 10 10 10 F F F
25772024-03-06T20:59:05.414Z 10 200-219 22 22 22 31 31 31 F F F
25782024-03-06T20:59:05.414Z 11 220-239 22 22 22 31 31 31 F F F
25792024-03-06T20:59:05.414Z 12 240-259 20 20 20 28 28 28 F F F
25802024-03-06T20:59:05.414Z 13 260-279 10 10 10 12 12 12 F F F
25812024-03-06T20:59:05.414Z 14 280-299 23 23 23 32 32 32 F F F
25822024-03-06T20:59:05.415Z 15 300-319 23 23 23 32 32 32 F F F
25832024-03-06T20:59:05.415Z 16 320-339 23 23 23 32 32 32 F F F
25842024-03-06T20:59:05.415Z 17 340-359 10 10 10 12 12 12 F F F
25852024-03-06T20:59:05.415Z 18 360-379 22 22 22 31 31 31 F F F
25862024-03-06T20:59:05.415Z 19 380-399 20 20 20 27 27 27 F F F
25872024-03-06T20:59:05.415Z 20 400-419 20 20 20 27 27 27 F F F
25882024-03-06T20:59:05.415Z 21 420-439 22 22 22 31 31 31 F F F
25892024-03-06T20:59:05.415Z 22 440-459 20 20 20 27 27 27 F F F
25902024-03-06T20:59:05.415Z 23 460-479 20 20 20 27 27 27 F F F
25912024-03-06T20:59:05.415Z 24 480-499 18 18 18 25 25 25 F F F
25922024-03-06T20:59:05.415Z 25 500-519 18 18 18 25 25 25 F F F
25932024-03-06T20:59:05.415Z 26 520-539 15 15 15 19 19 19 F F F
25942024-03-06T20:59:05.415Z 27 540-559 23 23 23 32 32 32 F F F
25952024-03-06T20:59:05.415Z 28 560-579 23 23 23 32 32 32 F F F
25962024-03-06T20:59:05.415Z 29 580-599 16 16 16 22 22 22 F F F
25972024-03-06T20:59:05.415ZMax gen: 23, Max flush: 32
25982024-03-06T20:59:05.415Z
25992024-03-06T20:59:05.415ZBegin loop 12
26002024-03-06T20:59:05.416ZDownstairs to restart: 0
26012024-03-06T20:59:05.430Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.78436891Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232}
26022024-03-06T20:59:05.430Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.784627592Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232}
26032024-03-06T20:59:05.431Z{Crucible runtime is spawned
26042024-03-06T20:59:05.431Z"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.784700107Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232}
26052024-03-06T20:59:05.431Z{"msg":"Crucible 92705345-b544-4d80-aab1-66480bc4dbe2 has session id: 7e07f3b0-5fa9-4d2e-a034-e45a546090d4","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.784733795Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26062024-03-06T20:59:05.431Z{"msg":"Upstairs opts: Upstairs UUID: 92705345-b544-4d80-aab1-66480bc4dbe2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.784780141Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26072024-03-06T20:59:05.431Z{"msg":"Crucible stats registered with UUID: 92705345-b544-4d80-aab1-66480bc4dbe2","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.7847985Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26082024-03-06T20:59:05.431Z{"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.784823228Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232}
26092024-03-06T20:59:05.432Z{"msg":"92705345-b544-4d80-aab1-66480bc4dbe2 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.784927231Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26102024-03-06T20:59:05.432Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.784962769Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"client":"0","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26112024-03-06T20:59:05.432Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.784981378Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"client":"0","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26122024-03-06T20:59:05.432Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.784997116Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"client":"1","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26132024-03-06T20:59:05.432Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.785028024Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"client":"1","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26142024-03-06T20:59:05.432Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.785047493Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"client":"2","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26152024-03-06T20:59:05.433Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.785076741Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"client":"2","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26162024-03-06T20:59:05.433Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.785102069Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26172024-03-06T20:59:05.433Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.785142876Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26182024-03-06T20:59:05.433Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.785169134Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26192024-03-06T20:59:05.433Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.785289976Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26202024-03-06T20:59:05.433Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.785318884Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26212024-03-06T20:59:05.433Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.785417328Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26222024-03-06T20:59:05.433Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.786096071Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"client":"1","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26232024-03-06T20:59:05.434Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.786137368Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"client":"1","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26242024-03-06T20:59:05.434Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.786174875Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"client":"2","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26252024-03-06T20:59:05.434Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.786198014Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"client":"2","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26262024-03-06T20:59:05.434Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.78625825Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"client":"0","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26272024-03-06T20:59:05.434Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.786312936Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"client":"0","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26282024-03-06T20:59:05.434Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.786710538Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"client":"1","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26292024-03-06T20:59:05.434Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.786767784Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"client":"2","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26302024-03-06T20:59:05.434Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.786797672Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"client":"0","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26312024-03-06T20:59:05.434Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.787283239Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"client":"2","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26322024-03-06T20:59:05.434Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.787335895Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"client":"2","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26332024-03-06T20:59:05.435Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.787365093Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26342024-03-06T20:59:05.435Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.787380362Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"client":"0","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26352024-03-06T20:59:05.435Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.787394631Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"client":"0","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26362024-03-06T20:59:05.435Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.78741685Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26372024-03-06T20:59:05.435Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.787436248Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"client":"1","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26382024-03-06T20:59:05.435Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.787457357Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"client":"1","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26392024-03-06T20:59:05.435Z{"msg":"[0]R flush_numbers[0..12]: [21, 29, 32, 32, 30, 31, 31, 31, 12, 10, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.787474456Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"":"downstairs","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26402024-03-06T20:59:05.435Z{"msg":"[0]R generation[0..12]: [16, 20, 23, 23, 21, 22, 22, 22, 10, 9, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.787489245Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"":"downstairs","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26412024-03-06T20:59:05.436Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.787505264Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"":"downstairs","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26422024-03-06T20:59:05.436Z{"msg":"[1]R flush_numbers[0..12]: [21, 29, 32, 32, 30, 31, 31, 31, 12, 10, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.787526352Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"":"downstairs","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26432024-03-06T20:59:05.436Z{"msg":"[1]R generation[0..12]: [16, 20, 23, 23, 21, 22, 22, 22, 10, 9, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.787540361Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"":"downstairs","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26442024-03-06T20:59:05.436Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.78756125Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"":"downstairs","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26452024-03-06T20:59:05.436Z{"msg":"[2]R flush_numbers[0..12]: [21, 29, 32, 32, 30, 31, 31, 31, 12, 10, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.787575249Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"":"downstairs","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26462024-03-06T20:59:05.436Z{"msg":"[2]R generation[0..12]: [16, 20, 23, 23, 21, 22, 22, 22, 10, 9, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.787595477Z","hostname":"Wait for a query_work_queue command to finish before sending IO
26472024-03-06T20:59:05.436Zw-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"":"downstairs","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26482024-03-06T20:59:05.436Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.787613186Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"":"downstairs","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26492024-03-06T20:59:05.436Z{"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.787633085Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"":"downstairs","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26502024-03-06T20:59:05.436Z{"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.787646484Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"":"downstairs","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26512024-03-06T20:59:05.437Z{"msg":"Next flush: 33","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.787665762Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"":"downstairs","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26522024-03-06T20:59:05.437Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.787679092Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"":"downstairs","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26532024-03-06T20:59:05.437Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.78769874Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26542024-03-06T20:59:05.437Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.787710859Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"client":"0","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26552024-03-06T20:59:05.437Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.787724918Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"client":"1","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26562024-03-06T20:59:05.437Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.787744367Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"client":"2","":"downstairs","session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26572024-03-06T20:59:05.437Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.787757876Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26582024-03-06T20:59:05.437Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.787776365Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26592024-03-06T20:59:05.437Z{"msg":"92705345-b544-4d80-aab1-66480bc4dbe2 is now active with session: 7e07f3b0-5fa9-4d2e-a034-e45a546090d4","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.787789374Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26602024-03-06T20:59:05.437Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.787807493Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232,"session_id":"7e07f3b0-5fa9-4d2e-a034-e45a546090d4"}
26612024-03-06T20:59:05.437Z{"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:03.787820072Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232}
26622024-03-06T20:59:05.438ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
26632024-03-06T20:59:05.438ZLoading write count information from file "/tmp/test_repair_verify.data"
26642024-03-06T20:59:05.438ZRun Repair workload
26652024-03-06T20:59:05.438Z01/30 Write block 30 len 24576 data: 4 4 4 5 5 5
26662024-03-06T20:59:05.438Z02/30 Write block 67 len 40960 data: 6 4 5 4 4 5 4 4 4 3
26672024-03-06T20:59:05.438Z03/30 Write block 348 len 4096 data: 3
26682024-03-06T20:59:05.438Z04/30 Read block 519 len 28672
26692024-03-06T20:59:05.438Z05/30 Read block 512 len 24576
26702024-03-06T20:59:05.438Z06/30 Write block 111 len 4096 data: 3
26712024-03-06T20:59:05.438Z07/30 Read block 331 len 8192
26722024-03-06T20:59:05.438Z08/30 Write block 47 len 12288 data: 4 3 3
26732024-03-06T20:59:05.439Z09/30 Write block 122 len 36864 data: 2 2 2 2 2 2 3 3 3
26742024-03-06T20:59:05.439Z10/30 Read block 58 len 4096
26752024-03-06T20:59:05.439Z11/30 Read block 116 len 8192
26762024-03-06T20:59:05.439Z12/30 Write block 384 len 40960 data: 3 3 3 3 2 2 3 3 3 3
26772024-03-06T20:59:05.439Z13/30 Read block 75 len 24576
26782024-03-06T20:59:05.439Z14/30 Write block 295 len 40960 data: 4 3 3 3 3 3 2 2 2 2
26792024-03-06T20:59:05.439Z15/30 Read block 324 len 24576
26802024-03-06T20:59:05.440Z16/30 Read block 69 len 36864
26812024-03-06T20:59:05.440Z17/30 Read block 85 len 20480
26822024-03-06T20:59:05.440Z18/30 Read block 234 len 12288
26832024-03-06T20:59:05.440Z19/30 Write block 516 len 16384 data: 2 2 2 2
26842024-03-06T20:59:05.440Z20/30 Write block 486 len 20480 data: 3 5 5 6 4
26852024-03-06T20:59:05.440Z21/30 Read block 457 len 4096
26862024-03-06T20:59:05.440Z22/30 Write block 41 len 24576 data: 4 5 4 4 3 3
26872024-03-06T20:59:05.440Z23/30 Write block 592 len 4096 data: 4
26882024-03-06T20:59:05.440Z24/30 Write block 576 len 12288 data: 4 3 2
26892024-03-06T20:59:05.440Z25/30 Read block 410 len 8192
26902024-03-06T20:59:05.440Z26/30 Read block 392 len 32768
26912024-03-06T20:59:05.441Z27/30 Read block 457 len 32768
26922024-03-06T20:59:05.441Z28/30 Write block 76 len 4096 data: 4
26932024-03-06T20:59:05.441Z29/30 Write block 455 len 40960 data: 3 2 4 4 4 5 6 6 6 5
26942024-03-06T20:59:05.441Z30/30 Read block 409 len 28672
26952024-03-06T20:59:05.441Z----------------------------------------------------------------
26962024-03-06T20:59:05.441Z Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:30
26972024-03-06T20:59:05.441ZStates: Active Active Active
26982024-03-06T20:59:05.441ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
26992024-03-06T20:59:05.441Z 1 Acked 1000 Write 6 Done Done Done false
27002024-03-06T20:59:05.441Z 2 Acked 1001 Write 10 Done Done Done false
27012024-03-06T20:59:05.468Z 3 Acked 1002 Write 1 Done Done Done false
27022024-03-06T20:59:05.468Z 4 Acked 1003 Read 7 Done Done Done false
27032024-03-06T20:59:05.468Z 5 Acked 1004 Read 6 Done Done Done false
27042024-03-06T20:59:05.468Z 6 Acked 1005 Write 1 Done Done Done false
27052024-03-06T20:59:05.469Z 7 Acked 1006 Read 2 Done Done Done false
27062024-03-06T20:59:05.469Z 8 Acked 1007 Write 3 Done Done Done false
27072024-03-06T20:59:05.469Z 9 Acked 1008 Write 9 Done Done Done false
27082024-03-06T20:59:05.469Z 10 Acked 1009 Read 1 Done Done Done false
27092024-03-06T20:59:05.469Z 11 Acked 1010 Read 2 Done Done Done false
27102024-03-06T20:59:05.469Z 12 Acked 1011 Write 10 Done Done Done false
27112024-03-06T20:59:05.469Z 13 Acked 1012 Read 6 Done Done Done false
27122024-03-06T20:59:05.469Z 14 Acked 1013 Write 10 Done Done Done false
27132024-03-06T20:59:05.469Z 15 Acked 1014 Read 6 Done Done Done false
27142024-03-06T20:59:05.469Z 16 Acked 1015 Read 9 Sent Done Done false
27152024-03-06T20:59:05.469Z 17 Acked 1016 Read 5 Sent Done Done false
27162024-03-06T20:59:05.469Z 18 Acked 1017 Read 3 Sent Done Done false
27172024-03-06T20:59:05.469Z 19 Acked 1018 Write 4 Sent Done Done false
27182024-03-06T20:59:05.469Z 20 Acked 1019 Write 5 Sent Done Done false
27192024-03-06T20:59:05.469Z 21 Acked 1020 Read 1 Sent Done Done false
27202024-03-06T20:59:05.470Z 22 Acked 1021 Write 6 Sent Done Done false
27212024-03-06T20:59:05.470Z 23 Acked 1022 Write 1 Sent Done Done false
27222024-03-06T20:59:05.470Z 24 Acked 1023 Write 3 Sent Done Done false
27232024-03-06T20:59:05.470Z 25 Acked 1024 Read 2 Sent Done Done false
27242024-03-06T20:59:05.470Z 26 Acked 1025 Read 8 Sent Done Done false
27252024-03-06T20:59:05.470Z 27 Acked 1026 Read 8 Sent Done Done false
27262024-03-06T20:59:05.470Z 28 Acked 1027 Write 1 Sent Done Done false
27272024-03-06T20:59:05.470Z 29 Acked 1028 Write 10 Sent Done Done false
27282024-03-06T20:59:05.470Z 30 Acked 1029 Read 7 Sent Done Done false
27292024-03-06T20:59:05.470Z STATES DS:0 DS:1 DS:2 TOTAL
27302024-03-06T20:59:05.470Z New 0 0 0 0
27312024-03-06T20:59:05.470Z Sent 15 0 0 15
27322024-03-06T20:59:05.470Z Done 15 30 30 75
27332024-03-06T20:59:05.470Z Skipped 0 0 0 0
27342024-03-06T20:59:05.471Z Error 0 0 0 0
27352024-03-06T20:59:05.471ZLast Flush: 0 0 0
27362024-03-06T20:59:05.471ZDownstairs last five completed:
27372024-03-06T20:59:05.471ZUpstairs last five completed: 30 29 28 27 26
27382024-03-06T20:59:05.471Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T20:59:03.817061417Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1232}
27392024-03-06T20:59:05.471ZWrote out file "/tmp/test_repair_verify.data"
27402024-03-06T20:59:05.471Z
27412024-03-06T20:59:05.471ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
27422024-03-06T20:59:05.471ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27432024-03-06T20:59:05.471Z 0 000-019 16 16 16 21 21 21 F F F
27442024-03-06T20:59:05.471Z 1 020-039 20 20 20 29 29 29 T T T
27452024-03-06T20:59:05.472Z 2 040-059 23 23 23 32 32 32 T T T
27462024-03-06T20:59:05.472Z 3 060-079 23 23 23 32 32 32 T T T
27472024-03-06T20:59:05.472Z 4 080-099 21 21 21 30 30 30 F F F
27482024-03-06T20:59:05.472Z 5 100-119 22 22 22 31 31 31 T T T
27492024-03-06T20:59:05.472Z 6 120-139 22 22 22 31 31 31 T T T
27502024-03-06T20:59:05.472Z 7 140-159 22 22 22 31 31 31 F F F
27512024-03-06T20:59:05.472Z 8 160-179 10 10 10 12 12 12 F F F
27522024-03-06T20:59:05.472Z 9 180-199 9 9 9 10 10 10 F F F
27532024-03-06T20:59:05.472Z 10 200-219 22 22 22 31 31 31 F F F
27542024-03-06T20:59:05.472Z 11 220-239 22 22 22 31 31 31 F F F
27552024-03-06T20:59:05.472Z 12 240-259 20 20 20 28 28 28 F F F
27562024-03-06T20:59:05.472Z 13 260-279 10 10 10 12 12 12 F F F
27572024-03-06T20:59:05.472Z 14 280-299 23 23 23 32 32 32 T T T
27582024-03-06T20:59:05.472Z 15 300-319 23 23 23 32 32 32 T T T
27592024-03-06T20:59:05.473Z 16 320-339 23 23 23 32 32 32 F F F
27602024-03-06T20:59:05.473Z 17 340-359 10 10 10 12 12 12 T T T
27612024-03-06T20:59:05.473Z 18 360-379 22 22 22 31 31 31 F F F
27622024-03-06T20:59:05.473Z 19 380-399 20 20 20 27 27 27 T T T
27632024-03-06T20:59:05.473Z 20 400-419 20 20 20 27 27 27 F F F
27642024-03-06T20:59:05.473Z 21 420-439 22 22 22 31 31 31 F F F
27652024-03-06T20:59:05.473Z 22 440-459 20 20 20 27 27 27 F T T <---
27662024-03-06T20:59:05.473Z 23 460-479 20 20 20 27 27 27 F T T <---
27672024-03-06T20:59:05.473Z 24 480-499 18 18 18 25 25 25 F T T <---
27682024-03-06T20:59:05.473Z 25 500-519 18 18 18 25 25 25 F T T <---
27692024-03-06T20:59:05.473Z 26 520-539 15 15 15 19 19 19 F F F
27702024-03-06T20:59:05.473Z 27 540-559 23 23 23 32 32 32 F F F
27712024-03-06T20:59:05.474Z 28 560-579 23 23 23 32 32 32 F T T <---
27722024-03-06T20:59:05.474Z 29 580-599 16 16 16 22 22 22 F T T <---
27732024-03-06T20:59:05.474ZMax gen: 23, Max flush: 32
27742024-03-06T20:59:05.474ZError: Difference in extent metadata found!
27752024-03-06T20:59:05.474ZA Difference in extent metadata is expected here
27762024-03-06T20:59:05.474ZOn loop 12
27772024-03-06T20:59:05.474Z
27782024-03-06T20:59:05.474ZVerifying data now
27792024-03-06T20:59:10.917ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
27802024-03-06T20:59:10.935Z44d43
27812024-03-06T20:59:10.935Z< 3,
27822024-03-06T20:59:10.935Z45a45,47
27832024-03-06T20:59:10.935Z> 5,
27842024-03-06T20:59:10.935Z> 4,
27852024-03-06T20:59:10.936Z> 4,
27862024-03-06T20:59:10.936Z48,49d49
27872024-03-06T20:59:10.937Z< 2,
27882024-03-06T20:59:10.937Z< 2,
27892024-03-06T20:59:10.937Z79c79
27902024-03-06T20:59:10.937Z< 3,
27912024-03-06T20:59:10.937Z---
27922024-03-06T20:59:10.937Z> 4,
27932024-03-06T20:59:10.937Zdiff found after verify
27942024-03-06T20:59:10.938ZLoop: 12 Downstairs dump after verify (and repair):
27952024-03-06T20:59:10.947ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27962024-03-06T20:59:10.947Z 0 000-019 16 16 16 21 21 21 F F F
27972024-03-06T20:59:10.947Z 1 020-039 24 24 24 33 33 33 F F F
27982024-03-06T20:59:10.947Z 2 040-059 24 24 24 33 33 33 F F F
27992024-03-06T20:59:10.947Z 3 060-079 24 24 24 33 33 33 F F F
28002024-03-06T20:59:10.947Z 4 080-099 21 21 21 30 30 30 F F F
28012024-03-06T20:59:10.947Z 5 100-119 24 24 24 33 33 33 F F F
28022024-03-06T20:59:10.947Z 6 120-139 24 24 24 33 33 33 F F F
28032024-03-06T20:59:10.947Z 7 140-159 22 22 22 31 31 31 F F F
28042024-03-06T20:59:10.947Z 8 160-179 10 10 10 12 12 12 F F F
28052024-03-06T20:59:10.947Z 9 180-199 9 9 9 10 10 10 F F F
28062024-03-06T20:59:10.947Z 10 200-219 22 22 22 31 31 31 F F F
28072024-03-06T20:59:10.947Z 11 220-239 22 22 22 31 31 31 F F F
28082024-03-06T20:59:10.947Z 12 240-259 20 20 20 28 28 28 F F F
28092024-03-06T20:59:10.947Z 13 260-279 10 10 10 12 12 12 F F F
28102024-03-06T20:59:10.948Z 14 280-299 24 24 24 33 33 33 F F F
28112024-03-06T20:59:10.948Z 15 300-319 24 24 24 33 33 33 F F F
28122024-03-06T20:59:10.948Z 16 320-339 23 23 23 32 32 32 F F F
28132024-03-06T20:59:10.948Z 17 340-359 24 24 24 33 33 33 F F F
28142024-03-06T20:59:10.948Z 18 360-379 22 22 22 31 31 31 F F F
28152024-03-06T20:59:10.948Z 19 380-399 24 24 24 33 33 33 F F F
28162024-03-06T20:59:10.948Z 20 400-419 20 20 20 27 27 27 F F F
28172024-03-06T20:59:10.948Z 21 420-439 22 22 22 31 31 31 F F F
28182024-03-06T20:59:10.948Z 22 440-459 24 24 24 33 33 33 F F F
28192024-03-06T20:59:10.948Z 23 460-479 24 24 24 33 33 33 F F F
28202024-03-06T20:59:10.948Z 24 480-499 24 24 24 33 33 33 F F F
28212024-03-06T20:59:10.948Z 25 500-519 24 24 24 33 33 33 F F F
28222024-03-06T20:59:10.948Z 26 520-539 15 15 15 19 19 19 F F F
28232024-03-06T20:59:10.948Z 27 540-559 23 23 23 32 32 32 F F F
28242024-03-06T20:59:10.948Z 28 560-579 24 24 24 33 33 33 F F F
28252024-03-06T20:59:10.948Z 29 580-599 24 24 24 33 33 33 F F F
28262024-03-06T20:59:10.948ZMax gen: 24, Max flush: 33
28272024-03-06T20:59:10.948Z
28282024-03-06T20:59:10.948ZBegin loop 13
28292024-03-06T20:59:10.948ZDownstairs to restart: 0
28302024-03-06T20:59:10.954Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.724511404Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243}
28312024-03-06T20:59:10.954Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.724765696Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243}
28322024-03-06T20:59:10.960Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.724802284Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243}
28332024-03-06T20:59:10.960Z{"msg":"Crucible 7e1f8fd7-d5a5-4e14-ad3f-2973a9db6e0b has session id: f522d231-c450-458e-acc3-ee031cba7c00","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.724834692Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243Crucible runtime is spawned
28342024-03-06T20:59:10.960Z,"session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28352024-03-06T20:59:10.960Z{"msg":"Upstairs opts: Upstairs UUID: 7e1f8fd7-d5a5-4e14-ad3f-2973a9db6e0b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.724884198Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28362024-03-06T20:59:10.960Z{"msg":"Crucible stats registered with UUID: 7e1f8fd7-d5a5-4e14-ad3f-2973a9db6e0b","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.724917166Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28372024-03-06T20:59:10.960Z{"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.724939624Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243}
28382024-03-06T20:59:10.960Z{"msg":"7e1f8fd7-d5a5-4e14-ad3f-2973a9db6e0b active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.725112623Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28392024-03-06T20:59:10.960Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.725160369Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"client":"0","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28402024-03-06T20:59:10.960Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.725233514Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"client":"0","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28412024-03-06T20:59:10.960Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.725321848Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"client":"1","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28422024-03-06T20:59:10.960Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.725370995Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"client":"1","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28432024-03-06T20:59:10.960Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.725404102Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"client":"2","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28442024-03-06T20:59:10.960Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.72543771Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"client":"2","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28452024-03-06T20:59:10.960Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.725463798Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"":"io task","client":"0","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28462024-03-06T20:59:10.960Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.725503596Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"":"io task","client":"2","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28472024-03-06T20:59:10.961Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.725530224Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"":"io task","client":"1","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28482024-03-06T20:59:10.961Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.725551972Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"":"io task","client":"2","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28492024-03-06T20:59:10.961Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.725576031Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"":"io task","client":"1","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28502024-03-06T20:59:10.961Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.725610388Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"":"io task","client":"0","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28512024-03-06T20:59:10.961Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.726261923Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"client":"1","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28522024-03-06T20:59:10.961Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.72630816Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"client":"1","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28532024-03-06T20:59:10.961Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.726333698Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"client":"2","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28542024-03-06T20:59:10.961Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.726371246Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"client":"2","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28552024-03-06T20:59:10.961Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.726873681Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"client":"1","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28562024-03-06T20:59:10.961Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.72703436Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"client":"2","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28572024-03-06T20:59:10.961Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.727067698Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"client":"0","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28582024-03-06T20:59:10.961Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.727082517Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"client":"0","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28592024-03-06T20:59:10.961Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.7274687Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"client":"1","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28602024-03-06T20:59:10.961Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.727512797Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"client":"1","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28612024-03-06T20:59:10.961Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.727538915Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28622024-03-06T20:59:10.961Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.727693694Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"client":"2","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28632024-03-06T20:59:10.961Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.727731312Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"client":"2","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28642024-03-06T20:59:10.961Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.72775644Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28652024-03-06T20:59:10.961Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.727829325Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"client":"0","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28662024-03-06T20:59:10.961Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.728166602Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"client":"0","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28672024-03-06T20:59:10.961Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.728213459Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"client":"0","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28682024-03-06T20:59:10.961Z{"msg":"[0]R flush_numbers[0..12]: [21, 33, 33, 33, 30, 33, 33, 31, 12, 10, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.728241707Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"":"downstairs","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28692024-03-06T20:59:10.961Z{"msg":"[0]R generation[0..12]: [16, 24, 24, 24, 21, 24, 24, 22, 10, 9, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.728272875Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"":"downstairs","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28702024-03-06T20:59:10.966Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.728298743Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"":"downstairs","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28712024-03-06T20:59:10.966Z{"msg":"[1]R flush_numbers[0..12]: [21, 33, 33, 33, 30, 33, 33, 31, 12, 10, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.728326781Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"":"downstairs","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28722024-03-06T20:59:10.966Z{"msg":"[1]R generation[0..12]: [16, 24, 24, 24, 21, 24, 24, 22, 10, 9, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.728350719Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"":"downstairs","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28732024-03-06T20:59:10.966ZWait for a query_work_queue command to finish before sending IO
28742024-03-06T20:59:10.966Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.728384587Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"":"downstairs","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28752024-03-06T20:59:10.967Z{"msg":"[2]R flush_numbers[0..12]: [21, 33, 33, 33, 30, 33, 33, 31, 12, 10, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.728411395Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"":"downstairs","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28762024-03-06T20:59:10.967Z{"msg":"[2]R generation[0..12]: [16, 24, 24, 24, 21, 24, 24, 22, 10, 9, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.728442543Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"":"downstairs","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28772024-03-06T20:59:10.967Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.728468441Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"":"downstairs","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28782024-03-06T20:59:10.967Z{"msg":"Max found gen is 25","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.72849419Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"":"downstairs","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28792024-03-06T20:59:10.967Z{"msg":"Generation requested: 26 >= found:25","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.728517098Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"":"downstairs","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28802024-03-06T20:59:10.967Z{"msg":"Next flush: 34","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.728543286Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"":"downstairs","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28812024-03-06T20:59:10.967Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.728565865Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"":"downstairs","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28822024-03-06T20:59:10.967Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.728592433Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28832024-03-06T20:59:10.967Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.728612961Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"client":"0","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28842024-03-06T20:59:10.967Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.728640809Z",Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
28852024-03-06T20:59:10.967Z"hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"client":"1","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28862024-03-06T20:59:10.967Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.728670907Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"client":"2","":"downstairs","session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28872024-03-06T20:59:10.967Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.728701405Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28882024-03-06T20:59:10.967Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.728723194Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28892024-03-06T20:59:10.967Z{"msg":"7e1f8fd7-d5a5-4e14-ad3f-2973a9db6e0b is now active with session: f522d231-c450-458e-acc3-ee031cba7c00","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.728746152Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28902024-03-06T20:59:10.967Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.72877225Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243,"session_id":"f522d231-c450-458e-acc3-ee031cba7c00"}
28912024-03-06T20:59:10.967Z{"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:09.728793669Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243}
28922024-03-06T20:59:10.967ZLoading write count information from file "/tmp/test_repair_verify.data"
28932024-03-06T20:59:10.967ZRun Repair workload
28942024-03-06T20:59:10.967Z01/30 Write block 189 len 8192 data: 2 2
28952024-03-06T20:59:10.967Z02/30 Read block 355 len 24576
28962024-03-06T20:59:10.967Z03/30 Write block 268 len 28672 data: 4 3 3 2 2 2 2
28972024-03-06T20:59:10.968Z04/30 Write block 430 len 4096 data: 5
28982024-03-06T20:59:10.968Z05/30 Read block 386 len 8192
28992024-03-06T20:59:10.968Z06/30 Read block 129 len 4096
29002024-03-06T20:59:10.968Z07/30 Read block 364 len 20480
29012024-03-06T20:59:10.971Z08/30 Write block 173 len 36864 data: 3 3 3 3 3 2 4 4 4
29022024-03-06T20:59:10.971Z09/30 Read block 451 len 8192
29032024-03-06T20:59:10.971Z10/30 Read block 118 len 32768
29042024-03-06T20:59:10.971Z11/30 Flush
29052024-03-06T20:59:10.971Z12/30 Flush
29062024-03-06T20:59:10.975Z13/30 Write block 239 len 28672 data: 6 5 6 5 4 4 4
29072024-03-06T20:59:10.975Z14/30 Read block 32 len 16384
29082024-03-06T20:59:10.975Z15/30 Read block 56 len 32768
29092024-03-06T20:59:10.975Z16/30 Read block 47 len 4096
29102024-03-06T20:59:10.978Z17/30 Write block 117 len 36864 data: 2 2 2 2 2 3 3 3 3
29112024-03-06T20:59:10.978Z18/30 Write block 106 len 24576 data: 5 6 3 3 3 4
29122024-03-06T20:59:10.978Z19/30 Read block 272 len 4096
29132024-03-06T20:59:10.978Z20/30 Write block 451 len 12288 data: 3 3 3
29142024-03-06T20:59:10.981Z21/30 Read block 540 len 20480
29152024-03-06T20:59:10.981Z22/30 Write block 573 len 20480 data: 4 4 4 5 4
29162024-03-06T20:59:10.981Z23/30 Flush
29172024-03-06T20:59:10.986Z24/30 Write block 48 len 24576 data: 4 4 3 3 3 3
29182024-03-06T20:59:10.986Z25/30 Write block 285 len 24576 data: 5 7 7 6 6 6
29192024-03-06T20:59:10.990Z26/30 Read block 537 len 12288
29202024-03-06T20:59:10.990Z27/30 Read block 577 len 16384
29212024-03-06T20:59:10.990Z28/30 Write block 55 len 12288 data: 4 3 4
29222024-03-06T20:59:10.990Z29/30 Read block 464 len 36864
29232024-03-06T20:59:10.990Z30/30 Write block 338 len 32768 data: 2 2 3 4 5 5 5 4
29242024-03-06T20:59:10.990Z----------------------------------------------------------------
29252024-03-06T20:59:10.990Z Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:30
29262024-03-06T20:59:10.990ZStates: Active Active Active
29272024-03-06T20:59:10.990ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
29282024-03-06T20:59:10.990Z 1 Acked 1000 Write 2 Done Done Done false
29292024-03-06T20:59:10.991Z 2 Acked 1001 Read 6 Done Done Done false
29302024-03-06T20:59:10.991Z 3 Acked 1002 Write 7 Sent Done Done false
29312024-03-06T20:59:10.991Z 4 Acked 1003 Write 1 Sent Done Done false
29322024-03-06T20:59:10.991Z 5 Acked 1004 Read 2 Sent Done Done false
29332024-03-06T20:59:10.991Z 6 Acked 1005 Read 1 Sent Done Done false
29342024-03-06T20:59:10.991Z 7 Acked 1006 Read 5 Sent Done Done false
29352024-03-06T20:59:10.991Z 8 Acked 1007 Write 9 Sent Done Done false
29362024-03-06T20:59:10.991Z 9 Acked 1008 Read 2 Sent Done Done false
29372024-03-06T20:59:10.991Z 10 Acked 1009 Read 8 Sent Done Done false
29382024-03-06T20:59:10.991Z 11 Acked 1010 Flush 0 Sent Done Done false
29392024-03-06T20:59:10.991Z 12 Acked 1011 Flush 0 Sent Done Done false
29402024-03-06T20:59:10.991Z 13 Acked 1012 Write 7 Sent Done Done false
29412024-03-06T20:59:10.991Z 14 Acked 1013 Read 4 Sent Done Done false
29422024-03-06T20:59:10.991Z 15 Acked 1014 Read 8 Sent Done Done false
29432024-03-06T20:59:10.991Z 16 Acked 1015 Read 1 Sent Done Done false
29442024-03-06T20:59:10.991Z 17 Acked 1016 Write 9 Sent Done Done false
29452024-03-06T20:59:10.991Z 18 Acked 1017 Write 6 Sent Done Done false
29462024-03-06T20:59:10.991Z 19 Acked 1018 Read 1 Sent Done Done false
29472024-03-06T20:59:10.991Z 20 Acked 1019 Write 3 Sent Done Done false
29482024-03-06T20:59:10.991Z 21 Acked 1020 Read 5 Sent Done Done false
29492024-03-06T20:59:10.991Z 22 Acked 1021 Write 5 Sent Done Done false
29502024-03-06T20:59:10.991Z 23 Acked 1022 Flush 0 Sent Done Done false
29512024-03-06T20:59:10.991Z 24 Acked 1023 Write 6 Sent Done Done false
29522024-03-06T20:59:10.992Z 25 Acked 1024 Write 6 Sent Done Done false
29532024-03-06T20:59:10.992Z 26 Acked 1025 Read 3 Sent Done Done false
29542024-03-06T20:59:10.992Z 27 Acked 1026 Read 4 Sent Done Done false
29552024-03-06T20:59:10.992Z 28 Acked 1027 Write 3 Sent Done Done false
29562024-03-06T20:59:10.992Z 29 Acked 1028 Read 9 Sent Done Done false
29572024-03-06T20:59:10.992Z 30 Acked 1029 Write 8 Sent Sent Sent false
29582024-03-06T20:59:10.992Z STATES DS:0 DS:1 DS:2 TOTAL
29592024-03-06T20:59:10.992Z New 0 0 0 0
29602024-03-06T20:59:10.992Z Sent 28 1 1 30
29612024-03-06T20:59:10.992Z Done 2 29 29 60
29622024-03-06T20:59:10.992Z Skipped 0 0 0 0
29632024-03-06T20:59:10.992Z Error 0 0 0 0
29642024-03-06T20:59:10.992ZLast Flush: 0 1022 1022
29652024-03-06T20:59:10.992ZDownstairs last five completed:
29662024-03-06T20:59:10.992ZUpstairs last five completed: 30 29 28 27 26
29672024-03-06T20:59:10.995Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T20:59:09.760627605Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1243}
29682024-03-06T20:59:10.996ZWrote out file "/tmp/test_repair_verify.data"
29692024-03-06T20:59:10.996Z
29702024-03-06T20:59:10.999ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
29712024-03-06T20:59:11.018ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
29722024-03-06T20:59:11.018Z 0 000-019 16 16 16 21 21 21 F F F
29732024-03-06T20:59:11.018Z 1 020-039 24 24 24 33 33 33 F F F
29742024-03-06T20:59:11.018Z 2 040-059 24 24 24 33 33 33 F T T <---
29752024-03-06T20:59:11.018Z 3 060-079 24 24 24 33 33 33 F F F
29762024-03-06T20:59:11.018Z 4 080-099 21 21 21 30 30 30 F F F
29772024-03-06T20:59:11.018Z 5 100-119 24 26 26 33 36 36 F F F <---
29782024-03-06T20:59:11.018Z 6 120-139 24 26 26 33 36 36 F F F <---
29792024-03-06T20:59:11.018Z 7 140-159 22 22 22 31 31 31 F F F
29802024-03-06T20:59:11.018Z 8 160-179 10 26 26 12 34 34 F F F <---
29812024-03-06T20:59:11.018Z 9 180-199 9 26 26 10 34 34 T F F <---
29822024-03-06T20:59:11.018Z 10 200-219 22 22 22 31 31 31 F F F
29832024-03-06T20:59:11.018Z 11 220-239 22 26 26 31 36 36 F F F <---
29842024-03-06T20:59:11.018Z 12 240-259 20 26 26 28 36 36 F F F <---
29852024-03-06T20:59:11.018Z 13 260-279 10 26 26 12 34 34 F F F <---
29862024-03-06T20:59:11.018Z 14 280-299 24 24 24 33 33 33 F T T <---
29872024-03-06T20:59:11.018Z 15 300-319 24 24 24 33 33 33 F F F
29882024-03-06T20:59:11.018Z 16 320-339 23 23 23 32 32 32 F T T <---
29892024-03-06T20:59:11.018Z 17 340-359 24 24 24 33 33 33 F T T <---
29902024-03-06T20:59:11.018Z 18 360-379 22 22 22 31 31 31 F F F
29912024-03-06T20:59:11.018Z 19 380-399 24 24 24 33 33 33 F F F
29922024-03-06T20:59:11.018Z 20 400-419 20 20 20 27 27 27 F F F
29932024-03-06T20:59:11.019Z 21 420-439 22 26 26 31 34 34 F F F <---
29942024-03-06T20:59:11.019Z 22 440-459 24 26 26 33 36 36 F F F <---
29952024-03-06T20:59:11.019Z 23 460-479 24 24 24 33 33 33 F F F
29962024-03-06T20:59:11.019Z 24 480-499 24 24 24 33 33 33 F F F
29972024-03-06T20:59:11.019Z 25 500-519 24 24 24 33 33 33 F F F
29982024-03-06T20:59:11.019Z 26 520-539 15 15 15 19 19 19 F F F
29992024-03-06T20:59:11.019Z 27 540-559 23 23 23 32 32 32 F F F
30002024-03-06T20:59:11.019Z 28 560-579 24 26 26 33 36 36 F F F <---
30012024-03-06T20:59:11.019Z 29 580-599 24 24 24 33 33 33 F F F
30022024-03-06T20:59:11.019ZMax gen: 26, Max flush: 36
30032024-03-06T20:59:11.019ZError: Difference in extent metadata found!
30042024-03-06T20:59:11.023ZA Difference in extent metadata is expected here
30052024-03-06T20:59:11.024ZOn loop 13
30062024-03-06T20:59:11.024Z
30072024-03-06T20:59:11.024ZVerifying data now
30082024-03-06T20:59:16.741ZNo change after verify
30092024-03-06T20:59:16.741ZLoop: 13 Downstairs dump after verify (and repair):
30102024-03-06T20:59:16.752ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
30112024-03-06T20:59:16.752Z 0 000-019 16 16 16 21 21 21 F F F
30122024-03-06T20:59:16.752Z 1 020-039 24 24 24 33 33 33 F F F
30132024-03-06T20:59:16.752Z 2 040-059 27 27 27 37 37 37 F F F
30142024-03-06T20:59:16.752Z 3 060-079 24 24 24 33 33 33 F F F
30152024-03-06T20:59:16.752Z 4 080-099 21 21 21 30 30 30 F F F
30162024-03-06T20:59:16.752Z 5 100-119 26 26 26 36 36 36 F F F
30172024-03-06T20:59:16.752Z 6 120-139 26 26 26 36 36 36 F F F
30182024-03-06T20:59:16.752Z 7 140-159 22 22 22 31 31 31 F F F
30192024-03-06T20:59:16.762Z 8 160-179 26 26 26 34 34 34 F F F
30202024-03-06T20:59:16.762Z 9 180-199 26 26 26 34 34 34 F F F
30212024-03-06T20:59:16.763Z 10 200-219 22 22 22 31 31 31 F F F
30222024-03-06T20:59:16.763Z 11 220-239 26 26 26 36 36 36 F F F
30232024-03-06T20:59:16.763Z 12 240-259 26 26 26 36 36 36 F F F
30242024-03-06T20:59:16.763Z 13 260-279 26 26 26 34 34 34 F F F
30252024-03-06T20:59:16.763Z 14 280-299 27 27 27 37 37 37 F F F
30262024-03-06T20:59:16.763Z 15 300-319 24 24 24 33 33 33 F F F
30272024-03-06T20:59:16.763Z 16 320-339 27 27 27 37 37 37 F F F
30282024-03-06T20:59:16.763Z 17 340-359 27 27 27 37 37 37 F F F
30292024-03-06T20:59:16.763Z 18 360-379 22 22 22 31 31 31 F F F
30302024-03-06T20:59:16.763Z 19 380-399 24 24 24 33 33 33 F F F
30312024-03-06T20:59:16.763Z 20 400-419 20 20 20 27 27 27 F F F
30322024-03-06T20:59:16.763Z 21 420-439 26 26 26 34 34 34 F F F
30332024-03-06T20:59:16.763Z 22 440-459 26 26 26 36 36 36 F F F
30342024-03-06T20:59:16.763Z 23 460-479 24 24 24 33 33 33 F F F
30352024-03-06T20:59:16.763Z 24 480-499 24 24 24 33 33 33 F F F
30362024-03-06T20:59:16.763Z 25 500-519 24 24 24 33 33 33 F F F
30372024-03-06T20:59:16.763Z 26 520-539 15 15 15 19 19 19 F F F
30382024-03-06T20:59:16.763Z 27 540-559 23 23 23 32 32 32 F F F
30392024-03-06T20:59:16.763Z 28 560-579 26 26 26 36 36 36 F F F
30402024-03-06T20:59:16.763Z 29 580-599 24 24 24 33 33 33 F F F
30412024-03-06T20:59:16.763ZMax gen: 27, Max flush: 37
30422024-03-06T20:59:16.764Z
30432024-03-06T20:59:16.764ZBegin loop 14
30442024-03-06T20:59:16.764ZDownstairs to restart: 0
30452024-03-06T20:59:16.776Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.541811043Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251}
30462024-03-06T20:59:16.776Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.542128802Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251}
30472024-03-06T20:59:16.787Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.542174458Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251}
30482024-03-06T20:59:16.787Z{"msg":"Crucible 753293db-a6ce-4c38-acda-562c8c3576db has session id: a0b6ad02-b79f-4c32-bebc-7d72e016a3f9","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.542202487Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30492024-03-06T20:59:16.787ZCrucible runtime is spawned
30502024-03-06T20:59:16.787Z{"msg":"Upstairs opts: Upstairs UUID: 753293db-a6ce-4c38-acda-562c8c3576db, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.542267522Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30512024-03-06T20:59:16.787Z{"msg":"Crucible stats registered with UUID: 753293db-a6ce-4c38-acda-562c8c3576db","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.542306429Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30522024-03-06T20:59:16.787Z{"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.542334387Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251}
30532024-03-06T20:59:16.787Z{"msg":"753293db-a6ce-4c38-acda-562c8c3576db active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.542452109Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30542024-03-06T20:59:16.787Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.542496046Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"client":"0","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30552024-03-06T20:59:16.788Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.542536413Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"client":"0","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30562024-03-06T20:59:16.788Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.542565851Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"client":"1","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30572024-03-06T20:59:16.788Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.542601969Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"client":"1","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30582024-03-06T20:59:16.788Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.542643096Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"client":"2","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30592024-03-06T20:59:16.788Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.542686723Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"client":"2","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30602024-03-06T20:59:16.788Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.54273276Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"":"io task","client":"0","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30612024-03-06T20:59:16.788Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.542774667Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"":"io task","client":"2","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30622024-03-06T20:59:16.788Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.542808075Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"":"io task","client":"1","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30632024-03-06T20:59:16.788Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.542849982Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"":"io task","client":"2","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30642024-03-06T20:59:16.788Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.54287742Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"":"io task","client":"1","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30652024-03-06T20:59:16.788Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.543013611Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"":"io task","client":"0","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30662024-03-06T20:59:16.788Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.543661676Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"client":"1","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30672024-03-06T20:59:16.788Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.543701123Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"client":"1","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30682024-03-06T20:59:16.788Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.543719642Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"client":"2","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30692024-03-06T20:59:16.788Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.543735411Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"client":"2","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30702024-03-06T20:59:16.788Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.544379397Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"client":"1","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30712024-03-06T20:59:16.788Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.544420444Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"client":"2","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30722024-03-06T20:59:16.788Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.54461329Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"client":"0","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30732024-03-06T20:59:16.788Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.544659837Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"client":"0","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30742024-03-06T20:59:16.788Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.545024092Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"client":"2","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30752024-03-06T20:59:16.789Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.545069309Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"client":"2","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30762024-03-06T20:59:16.789Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.545085248Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30772024-03-06T20:59:16.789Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.545157353Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"client":"1","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30782024-03-06T20:59:16.789Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.545183461Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"client":"1","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30792024-03-06T20:59:16.789Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.54519926Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30802024-03-06T20:59:16.789Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.545383047Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"client":"0","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30812024-03-06T20:59:16.789Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.545831226Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"client":"0","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30822024-03-06T20:59:16.789Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.545877183Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"client":"0","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30832024-03-06T20:59:16.789Z{"msg":"[0]R flush_numbers[0..12]: [21, 33, 37, 33, 30, 36, 36, 31, 34, 34, 31, 36]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.545905211Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"":"downstairs","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30842024-03-06T20:59:16.789Z{"msg":"[0]R generation[0..12]: [16, 24, 27, 24, 21, 26, 26, 22, 26, 26, 22, 26]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.54592165Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"":"downstairs","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30852024-03-06T20:59:16.789Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.545945998Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"":"downstairs","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30862024-03-06T20:59:16.789Z{"msg":"[1]R flush_numbers[0..12]: [21, 33, 37, 33, 30, 36, 36, 31, 34, 34, 31, 36]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.545960437Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"":"downstairs","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30872024-03-06T20:59:16.789Z{"msg":"[1]R generation[0..12]: [16, 24, 27, 24, 21, 26, 26, 22, 26, 26, 22, 26]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.545974627Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"":"downstairs","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30882024-03-06T20:59:16.789Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.545988966Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"":"downstairs","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30892024-03-06T20:59:16.789Z{"msg":"[2]R flush_numbers[0..12]: [21, 33, 37, 33, 30, 36, 36, 31, 34, 34, 31, 36]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.546009954Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"":"downstairs","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30902024-03-06T20:59:16.789Z{"msg":"[2]R generation[0..12]: [16, 24, 27, 24, 21, 26, 26, 22, 26, 26, 22, 26]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.546023843Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"":"downstairs","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30912024-03-06T20:59:16.789Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.546038332Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"":"downstairs","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30922024-03-06T20:59:16.789Z{"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.54607489Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"":"downstairs","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30932024-03-06T20:59:16.789Z{"msg":"Generation requested: 28 >= found:28","v":0,"name":"Wait for a query_work_queue command to finish before sending IO
30942024-03-06T20:59:16.789Zcrucible","level":30,"time":"2024-03-06T20:59:15.546093088Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"":"downstairs","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30952024-03-06T20:59:16.789Z{"msg":"Next flush: 38","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.546106427Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"":"downstairs","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30962024-03-06T20:59:16.789Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.546120566Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"":"downstairs","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30972024-03-06T20:59:16.789Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.546133896Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30982024-03-06T20:59:16.789Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.546153864Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"client":"0","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
30992024-03-06T20:59:16.790Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.546168593Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"client":"1","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
31002024-03-06T20:59:16.790Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.546181872Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"client":"2","":"downstairs","session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
31012024-03-06T20:59:16.790Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.546195251Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
31022024-03-06T20:59:16.790Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.54620782Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
31032024-03-06T20:59:16.790Z{"msg":"753293db-a6ce-4c38-acda-562c8c3576db is now active with session: a0b6ad02-b79f-4c32-bebc-7d72e016a3f9","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.546228549Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
31042024-03-06T20:59:16.790Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.546241188Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251,"session_id":"a0b6ad02-b79f-4c32-bebc-7d72e016a3f9"}
31052024-03-06T20:59:16.790Z{"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:15.546253497Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251}
31062024-03-06T20:59:16.790ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
31072024-03-06T20:59:16.790ZLoading write count information from file "/tmp/test_repair_verify.data"
31082024-03-06T20:59:16.790ZRun Repair workload
31092024-03-06T20:59:16.790Z01/30 Write block 147 len 12288 data: 3 4 4
31102024-03-06T20:59:16.790Z02/30 Flush
31112024-03-06T20:59:16.801Z03/30 Flush
31122024-03-06T20:59:16.801Z04/30 Write block 6 len 8192 data: 2 2
31132024-03-06T20:59:16.802Z05/30 Write block 327 len 28672 data: 2 3 5 4 4 3 3
31142024-03-06T20:59:16.802Z06/30 Read block 364 len 4096
31152024-03-06T20:59:16.802Z07/30 Write block 181 len 4096 data: 5
31162024-03-06T20:59:16.802Z08/30 Write block 69 len 32768 data: 6 5 5 6 5 5 5 4
31172024-03-06T20:59:16.802Z09/30 Read block 480 len 12288
31182024-03-06T20:59:16.802Z10/30 Read block 506 len 36864
31192024-03-06T20:59:16.802Z11/30 Read block 47 len 28672
31202024-03-06T20:59:16.802Z12/30 Read block 259 len 28672
31212024-03-06T20:59:16.802Z13/30 Read block 587 len 8192
31222024-03-06T20:59:16.802Z14/30 Flush
31232024-03-06T20:59:16.802Z15/30 Write block 189 len 8192 data: 3 3
31242024-03-06T20:59:16.802Z16/30 Read block 467 len 12288
31252024-03-06T20:59:16.824Z17/30 Read block 389 len 12288
31262024-03-06T20:59:16.824Z18/30 Read block 578 len 12288
31272024-03-06T20:59:16.824Z19/30 Flush
31282024-03-06T20:59:16.824Z20/30 Write block 285 len 24576 data: 6 8 8 7 7 7
31292024-03-06T20:59:16.824Z21/30 Read block 347 len 32768
31302024-03-06T20:59:16.824Z22/30 Read block 45 len 8192
31312024-03-06T20:59:16.824Z23/30 Write block 446 len 4096 data: 3
31322024-03-06T20:59:16.824Z24/30 Write block 263 len 28672 data: 3 3 4 4 4 5 4
31332024-03-06T20:59:16.824Z25/30 Write block 384 len 12288 data: 4 4 4
31342024-03-06T20:59:16.824Z26/30 Write block 88 len 8192 data: 3 3
31352024-03-06T20:59:16.824Z27/30 Write block 77 len 32768 data: 3 4 3 3 3 3 3 2
31362024-03-06T20:59:16.824Z28/30 Read block 325 len 28672
31372024-03-06T20:59:16.825Z29/30 Write block 495 len 24576 data: 5 5 4 4 3 3
31382024-03-06T20:59:16.825Z30/30 Write block 516 len 20480 data: 3 3 3 3 2
31392024-03-06T20:59:16.825Z----------------------------------------------------------------
31402024-03-06T20:59:16.825Z Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:30
31412024-03-06T20:59:16.825ZStates: Active Active Active
31422024-03-06T20:59:16.825ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
31432024-03-06T20:59:16.825Z 1 Acked 1000 Write 3 Sent Done Done false
31442024-03-06T20:59:16.825Z 2 Acked 1001 Flush 0 Sent Done Done false
31452024-03-06T20:59:16.825Z 3 Acked 1002 Flush 0 Sent Done Done false
31462024-03-06T20:59:16.825Z 4 Acked 1003 Write 2 Sent Done Done false
31472024-03-06T20:59:16.825Z 5 Acked 1004 Write 7 Sent Done Done false
31482024-03-06T20:59:16.825Z 6 Acked 1005 Read 1 Sent Done Done false
31492024-03-06T20:59:16.825Z 7 Acked 1006 Write 1 Sent Done Done false
31502024-03-06T20:59:16.825Z 8 Acked 1007 Write 8 Sent Done Done false
31512024-03-06T20:59:16.825Z 9 Acked 1008 Read 3 Sent Done Done false
31522024-03-06T20:59:16.825Z 10 Acked 1009 Read 9 Sent Done Done false
31532024-03-06T20:59:16.826Z 11 Acked 1010 Read 7 Sent Done Done false
31542024-03-06T20:59:16.826Z 12 Acked 1011 Read 7 Sent Done Done false
31552024-03-06T20:59:16.826Z 13 Acked 1012 Read 2 Sent Done Done false
31562024-03-06T20:59:16.826Z 14 Acked 1013 Flush 0 Sent Done Done false
31572024-03-06T20:59:16.826Z 15 Acked 1014 Write 2 Sent Done Done false
31582024-03-06T20:59:16.826Z 16 Acked 1015 Read 3 Sent Done Done false
31592024-03-06T20:59:16.826Z 17 Acked 1016 Read 3 Sent Done Done false
31602024-03-06T20:59:16.826Z 18 Acked 1017 Read 3 Sent Done Done false
31612024-03-06T20:59:16.826Z 19 Acked 1018 Flush 0 Sent Done Done false
31622024-03-06T20:59:16.826Z 20 Acked 1019 Write 6 Sent Done Done false
31632024-03-06T20:59:16.826Z 21 Acked 1020 Read 8 Sent Done Done false
31642024-03-06T20:59:16.826Z 22 Acked 1021 Read 2 Sent Done Done false
31652024-03-06T20:59:16.826Z 23 Acked 1022 Write 1 Sent Done Done false
31662024-03-06T20:59:16.827Z 24 Acked 1023 Write 7 Sent Done Done false
31672024-03-06T20:59:16.827Z 25 Acked 1024 Write 3 Sent Done Done false
31682024-03-06T20:59:16.827Z 26 Acked 1025 Write 2 Sent Done Done false
31692024-03-06T20:59:16.827Z 27 Acked 1026 Write 8 Sent Done Done false
31702024-03-06T20:59:16.827Z 28 Acked 1027 Read 7 Sent Done Done false
31712024-03-06T20:59:16.827Z 29 Acked 1028 Write 6 Sent Done Done false
31722024-03-06T20:59:16.828Z 30 Acked 1029 Write 5 Sent Sent Sent false
31732024-03-06T20:59:16.828Z STATES DS:0 DS:1 DS:2 TOTAL
31742024-03-06T20:59:16.828Z New 0 0 0 0
31752024-03-06T20:59:16.828Z Sent 30 1 1 32
31762024-03-06T20:59:16.828Z Done 0 29 29 58
31772024-03-06T20:59:16.828Z Skipped 0 0 0 0
31782024-03-06T20:59:16.829Z Error 0 0 0 0
31792024-03-06T20:59:16.829ZLast Flush: 0 1018 1018
31802024-03-06T20:59:16.829ZDownstairs last five completed:
31812024-03-06T20:59:16.829ZUpstairs last five completed: 30 29 28 27 26
31822024-03-06T20:59:16.861Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T20:59:15.57870805Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1251}
31832024-03-06T20:59:16.861ZWrote out file "/tmp/test_repair_verify.data"
31842024-03-06T20:59:16.861Z
31852024-03-06T20:59:16.861ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
31862024-03-06T20:59:16.878ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
31872024-03-06T20:59:16.878Z 0 000-019 16 28 28 21 40 40 F F F <---
31882024-03-06T20:59:16.878Z 1 020-039 24 24 24 33 33 33 F F F
31892024-03-06T20:59:16.878Z 2 040-059 27 27 27 37 37 37 F F F
31902024-03-06T20:59:16.878Z 3 060-079 24 28 28 33 40 40 F T T <---
31912024-03-06T20:59:16.878Z 4 080-099 21 21 21 30 30 30 F T T <---
31922024-03-06T20:59:16.878Z 5 100-119 26 26 26 36 36 36 F F F
31932024-03-06T20:59:16.878Z 6 120-139 26 26 26 36 36 36 F F F
31942024-03-06T20:59:16.878Z 7 140-159 22 28 28 31 38 38 F F F <---
31952024-03-06T20:59:16.878Z 8 160-179 26 26 26 34 34 34 F F F
31962024-03-06T20:59:16.878Z 9 180-199 26 28 28 34 41 41 F F F <---
31972024-03-06T20:59:16.878Z 10 200-219 22 22 22 31 31 31 F F F
31982024-03-06T20:59:16.878Z 11 220-239 26 26 26 36 36 36 F F F
31992024-03-06T20:59:16.878Z 12 240-259 26 26 26 36 36 36 F F F
32002024-03-06T20:59:16.878Z 13 260-279 26 26 26 34 34 34 F T T <---
32012024-03-06T20:59:16.878Z 14 280-299 27 27 27 37 37 37 F T T <---
32022024-03-06T20:59:16.878Z 15 300-319 24 24 24 33 33 33 F F F
32032024-03-06T20:59:16.878Z 16 320-339 27 28 28 37 40 40 F F F <---
32042024-03-06T20:59:16.878Z 17 340-359 27 27 27 37 37 37 F F F
32052024-03-06T20:59:16.878Z 18 360-379 22 22 22 31 31 31 F F F
32062024-03-06T20:59:16.879Z 19 380-399 24 24 24 33 33 33 F T T <---
32072024-03-06T20:59:16.879Z 20 400-419 20 20 20 27 27 27 F F F
32082024-03-06T20:59:16.879Z 21 420-439 26 26 26 34 34 34 F F F
32092024-03-06T20:59:16.879Z 22 440-459 26 26 26 36 36 36 F T T <---
32102024-03-06T20:59:16.879Z 23 460-479 24 24 24 33 33 33 F F F
32112024-03-06T20:59:16.879Z 24 480-499 24 24 24 33 33 33 F T T <---
32122024-03-06T20:59:16.879Z 25 500-519 24 24 24 33 33 33 F T T <---
32132024-03-06T20:59:16.879Z 26 520-539 15 15 15 19 19 19 F T T <---
32142024-03-06T20:59:16.879Z 27 540-559 23 23 23 32 32 32 F F F
32152024-03-06T20:59:16.879Z 28 560-579 26 26 26 36 36 36 F F F
32162024-03-06T20:59:16.879Z 29 580-599 24 24 24 33 33 33 F F F
32172024-03-06T20:59:16.879ZMax gen: 28, Max flush: 41
32182024-03-06T20:59:16.879ZError: Difference in extent metadata found!
32192024-03-06T20:59:16.879ZA Difference in extent metadata is expected here
32202024-03-06T20:59:16.879ZOn loop 14
32212024-03-06T20:59:16.879Z
32222024-03-06T20:59:16.879ZVerifying data now
32232024-03-06T20:59:21.552ZNo change after verify
32242024-03-06T20:59:21.552ZLoop: 14 Downstairs dump after verify (and repair):
32252024-03-06T20:59:21.566ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
32262024-03-06T20:59:21.566Z 0 000-019 28 28 28 40 40 40 F F F
32272024-03-06T20:59:21.566Z 1 020-039 24 24 24 33 33 33 F F F
32282024-03-06T20:59:21.566Z 2 040-059 27 27 27 37 37 37 F F F
32292024-03-06T20:59:21.566Z 3 060-079 29 29 29 42 42 42 F F F
32302024-03-06T20:59:21.566Z 4 080-099 29 29 29 42 42 42 F F F
32312024-03-06T20:59:21.566Z 5 100-119 26 26 26 36 36 36 F F F
32322024-03-06T20:59:21.566Z 6 120-139 26 26 26 36 36 36 F F F
32332024-03-06T20:59:21.566Z 7 140-159 28 28 28 38 38 38 F F F
32342024-03-06T20:59:21.566Z 8 160-179 26 26 26 34 34 34 F F F
32352024-03-06T20:59:21.566Z 9 180-199 28 28 28 41 41 41 F F F
32362024-03-06T20:59:21.567Z 10 200-219 22 22 22 31 31 31 F F F
32372024-03-06T20:59:21.567Z 11 220-239 26 26 26 36 36 36 F F F
32382024-03-06T20:59:21.567Z 12 240-259 26 26 26 36 36 36 F F F
32392024-03-06T20:59:21.572Z 13 260-279 29 29 29 42 42 42 F F F
32402024-03-06T20:59:21.572Z 14 280-299 29 29 29 42 42 42 F F F
32412024-03-06T20:59:21.572Z 15 300-319 24 24 24 33 33 33 F F F
32422024-03-06T20:59:21.572Z 16 320-339 28 28 28 40 40 40 F F F
32432024-03-06T20:59:21.572Z 17 340-359 27 27 27 37 37 37 F F F
32442024-03-06T20:59:21.572Z 18 360-379 22 22 22 31 31 31 F F F
32452024-03-06T20:59:21.572Z 19 380-399 29 29 29 42 42 42 F F F
32462024-03-06T20:59:21.572Z 20 400-419 20 20 20 27 27 27 F F F
32472024-03-06T20:59:21.572Z 21 420-439 26 26 26 34 34 34 F F F
32482024-03-06T20:59:21.572Z 22 440-459 29 29 29 42 42 42 F F F
32492024-03-06T20:59:21.572Z 23 460-479 24 24 24 33 33 33 F F F
32502024-03-06T20:59:21.572Z 24 480-499 29 29 29 42 42 42 F F F
32512024-03-06T20:59:21.572Z 25 500-519 29 29 29 42 42 42 F F F
32522024-03-06T20:59:21.572Z 26 520-539 29 29 29 42 42 42 F F F
32532024-03-06T20:59:21.572Z 27 540-559 23 23 23 32 32 32 F F F
32542024-03-06T20:59:21.572Z 28 560-579 26 26 26 36 36 36 F F F
32552024-03-06T20:59:21.572Z 29 580-599 24 24 24 33 33 33 F F F
32562024-03-06T20:59:21.572ZMax gen: 29, Max flush: 42
32572024-03-06T20:59:21.572Z
32582024-03-06T20:59:21.572ZBegin loop 15
32592024-03-06T20:59:21.572ZDownstairs to restart: 1
32602024-03-06T20:59:21.589Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.358406149Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259}
32612024-03-06T20:59:21.595Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
32622024-03-06T20:59:21.595Z,"time":"2024-03-06T20:59:20.358693809Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259}
32632024-03-06T20:59:21.595Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.358833959Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259}
32642024-03-06T20:59:21.595Z{"msg":"Crucible dac183d9-cc63-4afd-a533-e90484af6417 has session id: fa46f74d-22e2-4794-8a50-d212d6c33730","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.358870657Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32652024-03-06T20:59:21.595Z{"msg":"Upstairs opts: Upstairs UUID: dac183d9-cc63-4afd-a533-e90484af6417, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.358906444Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32662024-03-06T20:59:21.595Z{"msg":"Crucible stats registered with UUID: dac183d9-cc63-4afd-a533-e90484af6417","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.358940182Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32672024-03-06T20:59:21.595Z{"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.35896355Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259}
32682024-03-06T20:59:21.595Z{"msg":"dac183d9-cc63-4afd-a533-e90484af6417 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.358990548Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32692024-03-06T20:59:21.595Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.359011157Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"client":"0","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32702024-03-06T20:59:21.595Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.359042845Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"client":"0","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32712024-03-06T20:59:21.595Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.359067563Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"client":"1","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32722024-03-06T20:59:21.596Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.359101451Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"client":"1","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32732024-03-06T20:59:21.596Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.359125259Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"client":"2","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32742024-03-06T20:59:21.596Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.359160567Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"client":"2","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32752024-03-06T20:59:21.596Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.359245361Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"":"io task","client":"1","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32762024-03-06T20:59:21.596Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.359287418Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"":"io task","client":"2","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32772024-03-06T20:59:21.596Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.359491104Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"":"io task","client":"0","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32782024-03-06T20:59:21.596Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.359520242Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"":"io task","client":"1","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32792024-03-06T20:59:21.596Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.359559879Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"":"io task","client":"0","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32802024-03-06T20:59:21.596Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.359584208Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"":"io task","client":"2","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32812024-03-06T20:59:21.596Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.359966111Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"client":"2","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32822024-03-06T20:59:21.596Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.360076654Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"client":"2","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32832024-03-06T20:59:21.596Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.36013072Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"client":"0","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32842024-03-06T20:59:21.596Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.360238212Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"client":"0","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32852024-03-06T20:59:21.596Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.360287029Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"client":"1","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32862024-03-06T20:59:21.596Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.360322067Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"client":"1","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32872024-03-06T20:59:21.596Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.36056033Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"client":"2","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32882024-03-06T20:59:21.596Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.36070513Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"client":"1","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32892024-03-06T20:59:21.596Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.360748267Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"client":"0","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32902024-03-06T20:59:21.596Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.361060106Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"client":"2","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32912024-03-06T20:59:21.596Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.361090114Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"client":"2","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32922024-03-06T20:59:21.596Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.361106552Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32932024-03-06T20:59:21.596Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.361151209Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"client":"0","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32942024-03-06T20:59:21.597Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.361185727Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"client":"0","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32952024-03-06T20:59:21.597Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.361204096Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32962024-03-06T20:59:21.597Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.361248063Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"client":"1","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32972024-03-06T20:59:21.597Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.361278721Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"client":"1","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32982024-03-06T20:59:21.597Z{"msg":"[0]R flush_numbers[0..12]: [40, 33, 37, 42, 42, 36, 36, 38, 34, 41, 31, 36]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.361298979Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"":"downstairs","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
32992024-03-06T20:59:21.597Z{"msg":"[0]R generation[0..12]: [28, 24, 27, 29, 29, 26, 26, 28, 26, 28, 22, 26]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.361327927Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"":"downstairs","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
33002024-03-06T20:59:21.597Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.361348696Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"":"downstairs","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
33012024-03-06T20:59:21.597Z{"msg":"[1]R flush_numbers[0..12]: [40, 33, 37, 42, 42, 36, 36, 38, 34, 41, 31, 36]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.361377284Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"":"downstairs","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
33022024-03-06T20:59:21.597Z{"msg":"[1]R generation[0..12]: [28, 24, 27, 29, 29, 26, 26, 28, 26, 28, 22, 26]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.361399062Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"":"downstairs","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
33032024-03-06T20:59:21.597Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
33042024-03-06T20:59:21.597Z,"time":"2024-03-06T20:59:20.36142727Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"":"downstairs","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
33052024-03-06T20:59:21.597Z{"msg":"[2]R flush_numbers[0..12]: [40, 33, 37, 42, 42, 36, 36, 38, 34, 41, 31, 36]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.361459548Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"":"downstairs","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
33062024-03-06T20:59:21.597Z{"msg":"[2]R generation[0..12]: [28, 24, 27, 29, 29, 26, 26, 28, 26, 28, 22, 26]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.361493126Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"":"downstairs","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
33072024-03-06T20:59:21.597Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.361511235Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"":"downstairs","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
33082024-03-06T20:59:21.597Z{"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.361542493Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"":"downstairs","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
33092024-03-06T20:59:21.597Z{"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.36157669Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"":"downstairs","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
33102024-03-06T20:59:21.597Z{"msg":"Next flush: 43","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.361603718Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"":"downstairs","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
33112024-03-06T20:59:21.598Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.361640626Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"":"downstairs","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
33122024-03-06T20:59:21.598Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.361692802Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
33132024-03-06T20:59:21.598Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.36172966Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"client":"0","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
33142024-03-06T20:59:21.598Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.361768057Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"client":"1","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
33152024-03-06T20:59:21.598Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.361794295Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"client":"2","":"downstairs","session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
33162024-03-06T20:59:21.598Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.361830113Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
33172024-03-06T20:59:21.598Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.36186474Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
33182024-03-06T20:59:21.598ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
33192024-03-06T20:59:21.606Z{"msg":"dac183d9-cc63-4afd-a533-e90484af6417 is now active with session: fa46f74d-22e2-4794-8a50-d212d6c33730","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.361890329Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
33202024-03-06T20:59:21.606Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.361911387Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259,"session_id":"fa46f74d-22e2-4794-8a50-d212d6c33730"}
33212024-03-06T20:59:21.606Z{"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:20.361924706Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259}
33222024-03-06T20:59:21.606ZLoading write count information from file "/tmp/test_repair_verify.data"
33232024-03-06T20:59:21.606ZRun Repair workload
33242024-03-06T20:59:21.607Z01/30 Write block 186 len 28672 data: 2 2 2 4 4 2 3
33252024-03-06T20:59:21.607Z02/30 Flush
33262024-03-06T20:59:21.607Z03/30 Write block 350 len 8192 data: 3 3
33272024-03-06T20:59:21.607Z04/30 Write block 285 len 24576 data: 7 9 9 8 8 8
33282024-03-06T20:59:21.607Z05/30 Write block 334 len 28672 data: 2 2 2 2 3 3 4
33292024-03-06T20:59:21.607Z06/30 Write block 533 len 24576 data: 6 4 5 4 3 3
33302024-03-06T20:59:21.607Z07/30 Read block 364 len 4096
33312024-03-06T20:59:21.607Z08/30 Write block 156 len 20480 data: 2 2 3 3 3
33322024-03-06T20:59:21.611Z09/30 Write block 11 len 12288 data: 4 4 4
33332024-03-06T20:59:21.611Z10/30 Write block 493 len 32768 data: 5 5 6 6 5 5 4 4
33342024-03-06T20:59:21.611Z11/30 Read block 259 len 16384
33352024-03-06T20:59:21.611Z12/30 Read block 37 len 36864
33362024-03-06T20:59:21.611Z13/30 Flush
33372024-03-06T20:59:21.612Z14/30 Write block 580 len 24576 data: 2 2 2 2 2 2
33382024-03-06T20:59:21.615Z15/30 Write block 20 len 28672 data: 2 2 3 4 3 3 3
33392024-03-06T20:59:21.615Z16/30 Read block 398 len 12288
33402024-03-06T20:59:21.615Z17/30 Write block 136 len 4096 data: 3
33412024-03-06T20:59:21.615Z18/30 Read block 315 len 12288
33422024-03-06T20:59:21.615Z19/30 Read block 242 len 28672
33432024-03-06T20:59:21.615Z20/30 Read block 487 len 24576
33442024-03-06T20:59:21.625Z21/30 Read block 191 len 12288
33452024-03-06T20:59:21.625Z22/30 Read block 54 len 32768
33462024-03-06T20:59:21.625Z23/30 Flush
33472024-03-06T20:59:21.630Z24/30 Flush
33482024-03-06T20:59:21.631Z25/30 Write block 305 len 40960 data: 4 4 4 4 4 4 4 4 3 2
33492024-03-06T20:59:21.631Z26/30 Write block 94 len 16384 data: 3 2 2 2
33502024-03-06T20:59:21.631Z27/30 Write block 59 len 40960 data: 6 7 8 9 9 7 7 7 7 5
33512024-03-06T20:59:21.631Z28/30 Write block 338 len 16384 data: 4 4 5 5
33522024-03-06T20:59:21.631Z29/30 Read block 463 len 28672
33532024-03-06T20:59:21.631Z30/30 Read block 216 len 4096
33542024-03-06T20:59:21.631Z----------------------------------------------------------------
33552024-03-06T20:59:21.631Z Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:28
33562024-03-06T20:59:21.631ZStates: Active Active Active
33572024-03-06T20:59:21.631ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
33582024-03-06T20:59:21.631Z 3 Acked 1002 Write 2 Done Sent Done false
33592024-03-06T20:59:21.631Z 4 Acked 1003 Write 6 Done Sent Done false
33602024-03-06T20:59:21.631Z 5 Acked 1004 Write 7 Done Sent Done false
33612024-03-06T20:59:21.631Z 6 Acked 1005 Write 6 Done Sent Done false
33622024-03-06T20:59:21.631Z 7 Acked 1006 Read 1 Done Sent Done false
33632024-03-06T20:59:21.631Z 8 Acked 1007 Write 5 Done Sent Done false
33642024-03-06T20:59:21.631Z 9 Acked 1008 Write 3 Done Sent Done false
33652024-03-06T20:59:21.631Z 10 Acked 1009 Write 8 Done Sent Done false
33662024-03-06T20:59:21.631Z 11 Acked 1010 Read 4 Done Sent Done false
33672024-03-06T20:59:21.631Z 12 Acked 1011 Read 9 Done Sent Done false
33682024-03-06T20:59:21.631Z 13 Acked 1012 Flush 0 Done Sent Done false
33692024-03-06T20:59:21.631Z 14 Acked 1013 Write 6 Done Sent Done false
33702024-03-06T20:59:21.632Z 15 Acked 1014 Write 7 Done Sent Done false
33712024-03-06T20:59:21.632Z 16 Acked 1015 Read 3 Done Sent Done false
33722024-03-06T20:59:21.632Z 17 Acked 1016 Write 1 Done Sent Done false
33732024-03-06T20:59:21.632Z 18 Acked 1017 Read 3 Done Sent Done false
33742024-03-06T20:59:21.632Z 19 Acked 1018 Read 7 Done Sent Done false
33752024-03-06T20:59:21.632Z 20 Acked 1019 Read 6 Done Sent Done false
33762024-03-06T20:59:21.632Z 21 Acked 1020 Read 3 Done Sent Done false
33772024-03-06T20:59:21.632Z 22 Acked 1021 Read 8 Done Sent Done false
33782024-03-06T20:59:21.632Z 23 Acked 1022 Flush 0 Done Sent Done false
33792024-03-06T20:59:21.632Z 24 Acked 1023 Flush 0 Done Sent Done false
33802024-03-06T20:59:21.632Z 25 Acked 1024 Write 10 Done Sent Done false
33812024-03-06T20:59:21.632Z 26 Acked 1025 Write 4 Done Sent Done false
33822024-03-06T20:59:21.632Z 27 Acked 1026 Write 10 Done Sent Done false
33832024-03-06T20:59:21.632Z 28 Acked 1027 Write 4 Done Sent Done false
33842024-03-06T20:59:21.632Z 29 Acked 1028 Read 7 Done Sent Done false
33852024-03-06T20:59:21.632Z 30 Acked 1029 Read 1 Done Sent Done false
33862024-03-06T20:59:21.632Z STATES DS:0 DS:1 DS:2 TOTAL
33872024-03-06T20:59:21.632Z New 0 0 0 0
33882024-03-06T20:59:21.633Z Sent 0 28 0 28
33892024-03-06T20:59:21.633Z Done 28 0 28 56
33902024-03-06T20:59:21.633Z Skipped 0 0 0 0
33912024-03-06T20:59:21.633Z Error 0 0 0 0
33922024-03-06T20:59:21.633ZLast Flush: 1023 1001 1023
33932024-03-06T20:59:21.633ZDownstairs last five completed: 1001 1000
33942024-03-06T20:59:21.633ZUpstairs last five completed: 30 29 28 27 26
33952024-03-06T20:59:21.633Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T20:59:20.396095611Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1259}
33962024-03-06T20:59:21.633ZWrote out file "/tmp/test_repair_verify.data"
33972024-03-06T20:59:21.657Z
33982024-03-06T20:59:21.657ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
33992024-03-06T20:59:21.669ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
34002024-03-06T20:59:21.669Z 0 000-019 30 28 30 44 40 44 F F F <---
34012024-03-06T20:59:21.669Z 1 020-039 30 24 30 45 33 45 F F F <---
34022024-03-06T20:59:21.669Z 2 040-059 27 27 27 37 37 37 T F T <---
34032024-03-06T20:59:21.669Z 3 060-079 29 29 29 42 42 42 T F T <---
34042024-03-06T20:59:21.669Z 4 080-099 29 29 29 42 42 42 T F T <---
34052024-03-06T20:59:21.669Z 5 100-119 26 26 26 36 36 36 F F F
34062024-03-06T20:59:21.669Z 6 120-139 30 26 30 45 36 45 F F F <---
34072024-03-06T20:59:21.670Z 7 140-159 30 28 30 44 38 44 F F F <---
34082024-03-06T20:59:21.670Z 8 160-179 30 26 30 44 34 44 F F F <---
34092024-03-06T20:59:21.670Z 9 180-199 30 30 30 43 43 43 F F F
34102024-03-06T20:59:21.670Z 10 200-219 22 22 22 31 31 31 F F F
34112024-03-06T20:59:21.670Z 11 220-239 26 26 26 36 36 36 F F F
34122024-03-06T20:59:21.670Z 12 240-259 26 26 26 36 36 36 F F F
34132024-03-06T20:59:21.670Z 13 260-279 29 29 29 42 42 42 F F F
34142024-03-06T20:59:21.670Z 14 280-299 30 29 30 44 42 44 F F F <---
34152024-03-06T20:59:21.670Z 15 300-319 24 24 24 33 33 33 T F T <---
34162024-03-06T20:59:21.670Z 16 320-339 30 28 30 44 40 44 T F T <---
34172024-03-06T20:59:21.670Z 17 340-359 30 27 30 44 37 44 T F T <---
34182024-03-06T20:59:21.670Z 18 360-379 22 22 22 31 31 31 F F F
34192024-03-06T20:59:21.670Z 19 380-399 29 29 29 42 42 42 F F F
34202024-03-06T20:59:21.670Z 20 400-419 20 20 20 27 27 27 F F F
34212024-03-06T20:59:21.670Z 21 420-439 26 26 26 34 34 34 F F F
34222024-03-06T20:59:21.670Z 22 440-459 29 29 29 42 42 42 F F F
34232024-03-06T20:59:21.670Z 23 460-479 24 24 24 33 33 33 F F F
34242024-03-06T20:59:21.670Z 24 480-499 30 29 30 44 42 44 F F F <---
34252024-03-06T20:59:21.670Z 25 500-519 30 29 30 44 42 44 F F F <---
34262024-03-06T20:59:21.670Z 26 520-539 30 29 30 44 42 44 F F F <---
34272024-03-06T20:59:21.671Z 27 540-559 23 23 23 32 32 32 F F F
34282024-03-06T20:59:21.671Z 28 560-579 26 26 26 36 36 36 F F F
34292024-03-06T20:59:21.671Z 29 580-599 30 24 30 45 33 45 F F F <---
34302024-03-06T20:59:21.671ZMax gen: 30, Max flush: 45
34312024-03-06T20:59:21.671ZError: Difference in extent metadata found!
34322024-03-06T20:59:21.671ZA Difference in extent metadata is expected here
34332024-03-06T20:59:21.671ZOn loop 15
34342024-03-06T20:59:21.671Z
34352024-03-06T20:59:21.683ZVerifying data now
34362024-03-06T20:59:26.453ZNo change after verify
34372024-03-06T20:59:26.453ZLoop: 15 Downstairs dump after verify (and repair):
34382024-03-06T20:59:26.465ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
34392024-03-06T20:59:26.465Z 0 000-019 30 30 30 44 44 44 F F F
34402024-03-06T20:59:26.465Z 1 020-039 30 30 30 45 45 45 F F F
34412024-03-06T20:59:26.465Z 2 040-059 31 31 31 46 46 46 F F F
34422024-03-06T20:59:26.465Z 3 060-079 31 31 31 46 46 46 F F F
34432024-03-06T20:59:26.465Z 4 080-099 31 31 31 46 46 46 F F F
34442024-03-06T20:59:26.465Z 5 100-119 26 26 26 36 36 36 F F F
34452024-03-06T20:59:26.465Z 6 120-139 30 30 30 45 45 45 F F F
34462024-03-06T20:59:26.465Z 7 140-159 30 30 30 44 44 44 F F F
34472024-03-06T20:59:26.465Z 8 160-179 30 30 30 44 44 44 F F F
34482024-03-06T20:59:26.465Z 9 180-199 30 30 30 43 43 43 F F F
34492024-03-06T20:59:26.465Z 10 200-219 22 22 22 31 31 31 F F F
34502024-03-06T20:59:26.465Z 11 220-239 26 26 26 36 36 36 F F F
34512024-03-06T20:59:26.465Z 12 240-259 26 26 26 36 36 36 F F F
34522024-03-06T20:59:26.465Z 13 260-279 29 29 29 42 42 42 F F F
34532024-03-06T20:59:26.465Z 14 280-299 30 30 30 44 44 44 F F F
34542024-03-06T20:59:26.465Z 15 300-319 31 31 31 46 46 46 F F F
34552024-03-06T20:59:26.465Z 16 320-339 31 31 31 46 46 46 F F F
34562024-03-06T20:59:26.465Z 17 340-359 31 31 31 46 46 46 F F F
34572024-03-06T20:59:26.466Z 18 360-379 22 22 22 31 31 31 F F F
34582024-03-06T20:59:26.466Z 19 380-399 29 29 29 42 42 42 F F F
34592024-03-06T20:59:26.466Z 20 400-419 20 20 20 27 27 27 F F F
34602024-03-06T20:59:26.466Z 21 420-439 26 26 26 34 34 34 F F F
34612024-03-06T20:59:26.470Z 22 440-459 29 29 29 42 42 42 F F F
34622024-03-06T20:59:26.470Z 23 460-479 24 24 24 33 33 33 F F F
34632024-03-06T20:59:26.470Z 24 480-499 30 30 30 44 44 44 F F F
34642024-03-06T20:59:26.470Z 25 500-519 30 30 30 44 44 44 F F F
34652024-03-06T20:59:26.470Z 26 520-539 30 30 30 44 44 44 F F F
34662024-03-06T20:59:26.470Z 27 540-559 23 23 23 32 32 32 F F F
34672024-03-06T20:59:26.470Z 28 560-579 26 26 26 36 36 36 F F F
34682024-03-06T20:59:26.470Z 29 580-599 30 30 30 45 45 45 F F F
34692024-03-06T20:59:26.470ZMax gen: 31, Max flush: 46
34702024-03-06T20:59:26.470Z
34712024-03-06T20:59:26.470ZBegin loop 16
34722024-03-06T20:59:26.470ZDownstairs to restart: 0
34732024-03-06T20:59:26.490Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.259305384Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267}
34742024-03-06T20:59:26.490Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.259595214Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267}
34752024-03-06T20:59:26.490Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.259621862Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267}
34762024-03-06T20:59:26.501Z{"msg":"Crucible 4a6deb66-addc-4ea1-8614-e2115632c4a6 has session id: 4e3414b7-5639-4049-852a-9d952638755a","v":0Crucible runtime is spawned
34772024-03-06T20:59:26.501Z,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.259687887Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
34782024-03-06T20:59:26.501Z{"msg":"Upstairs opts: Upstairs UUID: 4a6deb66-addc-4ea1-8614-e2115632c4a6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.259737744Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
34792024-03-06T20:59:26.501Z{"msg":"Crucible stats registered with UUID: 4a6deb66-addc-4ea1-8614-e2115632c4a6","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.259772011Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
34802024-03-06T20:59:26.501Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.2597959Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267}
34812024-03-06T20:59:26.501Z{"msg":"4a6deb66-addc-4ea1-8614-e2115632c4a6 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.259827158Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
34822024-03-06T20:59:26.501Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.259849716Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"client":"0","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
34832024-03-06T20:59:26.501Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.259879504Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"client":"0","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
34842024-03-06T20:59:26.501Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.259921701Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"client":"1","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
34852024-03-06T20:59:26.502Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.259948339Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"client":"1","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
34862024-03-06T20:59:26.502Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.259980197Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"client":"2","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
34872024-03-06T20:59:26.502Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.260005066Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"client":"2","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
34882024-03-06T20:59:26.502Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.260029884Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"":"io task","client":"0","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
34892024-03-06T20:59:26.502Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.260061911Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"":"io task","client":"1","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
34902024-03-06T20:59:26.502Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.260120658Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"":"io task","client":"2","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
34912024-03-06T20:59:26.502Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.260150236Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"":"io task","client":"0","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
34922024-03-06T20:59:26.502Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.260190203Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"":"io task","client":"1","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
34932024-03-06T20:59:26.502Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.260311714Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"":"io task","client":"2","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
34942024-03-06T20:59:26.502Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.260970479Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"client":"1","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
34952024-03-06T20:59:26.502Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.261015576Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"client":"1","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
34962024-03-06T20:59:26.502Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.261033255Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"client":"2","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
34972024-03-06T20:59:26.502Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.261055493Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"client":"2","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
34982024-03-06T20:59:26.502Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.261565418Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"client":"1","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
34992024-03-06T20:59:26.503Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.261700629Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"client":"2","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35002024-03-06T20:59:26.503Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.261751535Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"client":"0","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35012024-03-06T20:59:26.503Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.261774224Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"client":"0","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35022024-03-06T20:59:26.503Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.262045955Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"client":"1","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35032024-03-06T20:59:26.503Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.262082332Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"client":"1","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35042024-03-06T20:59:26.503Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.262101071Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35052024-03-06T20:59:26.503Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.262204164Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"client":"2","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35062024-03-06T20:59:26.503Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.262238892Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"client":"2","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35072024-03-06T20:59:26.503Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.262272979Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35082024-03-06T20:59:26.503Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.262448817Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"client":"0","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35092024-03-06T20:59:26.503Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.262878787Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"client":"0","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35102024-03-06T20:59:26.503Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.262915325Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"client":"0","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35112024-03-06T20:59:26.503Z{"msg":"[0]R flush_numbers[0..12]: [44, 45, 46, 46, 46, 36, 45, 44, 44, 43, 31, 36]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.262945973Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"":"downstairs","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35122024-03-06T20:59:26.503Z{"msg":"[0]R generation[0..12]: [30, 30, 31, 31, 31, 26, 30, 30, 30, 30, 22, 26]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.262976191Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"":"downstairs","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35132024-03-06T20:59:26.503Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.263015108Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"":"downstairs","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35142024-03-06T20:59:26.503Z{"msg":"[1]R flush_numbers[0..12]: [44, 45, 46, 46, 46, 36, 45, 44, 44, 43, 31, 36]","v":0,"Wait for a query_work_queue command to finish before sending IO
35152024-03-06T20:59:26.504Zname":"crucible","level":30,"time":"2024-03-06T20:59:25.263053575Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"":"downstairs","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35162024-03-06T20:59:26.504Z{"msg":"[1]R generation[0..12]: [30, 30, 31, 31, 31, 26, 30, 30, 30, 30, 22, 26]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.263079683Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"":"downstairs","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35172024-03-06T20:59:26.504Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.263110221Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"":"downstairs","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35182024-03-06T20:59:26.504Z{"msg":"[2]R flush_numbers[0..12]: [44, 45, 46, 46, 46, 36, 45, 44, 44, 43, 31, 36]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.26313454Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"":"downstairs","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35192024-03-06T20:59:26.504Z{"msg":"[2]R generation[0..12]: [30, 30, 31, 31, 31, 26, 30, 30, 30, 30, 22, 26]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.263159808Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"":"downstairs","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35202024-03-06T20:59:26.504Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.263193116Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"":"downstairs","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35212024-03-06T20:59:26.504Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.263217594Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"":"downstairs","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35222024-03-06T20:59:26.504Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.263240383Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"":"downstairs","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35232024-03-06T20:59:26.504Z{"msg":"Next flush: 47","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.26327027Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"":"downstairs","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35242024-03-06T20:59:26.504Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.263293219Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"":"downstairs","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35252024-03-06T20:59:26.504Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.263320567Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35262024-03-06T20:59:26.504Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.263341356Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"client":"0","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35272024-03-06T20:59:26.504Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.263368914Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"client":"1","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35282024-03-06T20:59:26.504Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
35292024-03-06T20:59:26.504ZTransition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.263408071Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"client":"2","":"downstairs","session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35302024-03-06T20:59:26.509Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.263432709Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35312024-03-06T20:59:26.509Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.263458857Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35322024-03-06T20:59:26.509Z{"msg":"4a6deb66-addc-4ea1-8614-e2115632c4a6 is now active with session: 4e3414b7-5639-4049-852a-9d952638755a","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.263481186Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35332024-03-06T20:59:26.509Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.263506564Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267,"session_id":"4e3414b7-5639-4049-852a-9d952638755a"}
35342024-03-06T20:59:26.509Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:25.263528273Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267}
35352024-03-06T20:59:26.509ZLoading write count information from file "/tmp/test_repair_verify.data"
35362024-03-06T20:59:26.509ZRun Repair workload
35372024-03-06T20:59:26.509Z01/30 Write block 189 len 40960 data: 5 5 3 4 2 2 2 2 2 2
35382024-03-06T20:59:26.509Z02/30 Write block 173 len 40960 data: 4 4 4 4 4 3 5 5 6 4
35392024-03-06T20:59:26.509Z03/30 Write block 333 len 12288 data: 4 3 3
35402024-03-06T20:59:26.509Z04/30 Read block 83 len 32768
35412024-03-06T20:59:26.509Z05/30 Read block 453 len 12288
35422024-03-06T20:59:26.509Z06/30 Write block 180 len 8192 data: 6 7
35432024-03-06T20:59:26.509Z07/30 Write block 584 len 16384 data: 3 3 2 3
35442024-03-06T20:59:26.509Z08/30 Read block 534 len 20480
35452024-03-06T20:59:26.509Z09/30 Read block 244 len 24576
35462024-03-06T20:59:26.509Z10/30 Write block 357 len 28672 data: 4 4 4 5 6 4 5
35472024-03-06T20:59:26.509Z11/30 Read block 75 len 16384
35482024-03-06T20:59:26.509Z12/30 Flush
35492024-03-06T20:59:26.513Z13/30 Write block 480 len 28672 data: 2 3 3 3 3 3 4
35502024-03-06T20:59:26.513Z14/30 Write block 11 len 4096 data: 5
35512024-03-06T20:59:26.513Z15/30 Write block 584 len 28672 data: 4 4 3 4 3 3 4
35522024-03-06T20:59:26.513Z16/30 Flush
35532024-03-06T20:59:26.516Z17/30 Write block 348 len 20480 data: 4 3 4 4 4
35542024-03-06T20:59:26.516Z18/30 Read block 301 len 20480
35552024-03-06T20:59:26.519Z19/30 Flush
35562024-03-06T20:59:26.519Z20/30 Flush
35572024-03-06T20:59:26.522Z21/30 Flush
35582024-03-06T20:59:26.522Z22/30 Write block 52 len 28672 data: 4 4 3 5 4 5 5
35592024-03-06T20:59:26.522Z23/30 Read block 22 len 16384
35602024-03-06T20:59:26.523Z24/30 Read block 526 len 28672
35612024-03-06T20:59:26.526Z25/30 Read block 252 len 40960
35622024-03-06T20:59:26.526Z26/30 Read block 105 len 28672
35632024-03-06T20:59:26.526Z27/30 Read block 477 len 40960
35642024-03-06T20:59:26.529Z28/30 Write block 97 len 32768 data: 3 4 4 4 5 5 4 4
35652024-03-06T20:59:26.529Z29/30 Read block 103 len 24576
35662024-03-06T20:59:26.533Z30/30 Write block 288 len 16384 data: 9 9 9 6
35672024-03-06T20:59:26.533Z----------------------------------------------------------------
35682024-03-06T20:59:26.533Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30
35692024-03-06T20:59:26.533ZStates: Active Active Active
35702024-03-06T20:59:26.533ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
35712024-03-06T20:59:26.533Z 1 Acked 1000 Write 10 Done Done Done false
35722024-03-06T20:59:26.533Z 2 Acked 1001 Write 10 Done Done Done false
35732024-03-06T20:59:26.533Z 3 Acked 1002 Write 3 Done Done Done false
35742024-03-06T20:59:26.533Z 4 Acked 1003 Read 8 Done Done Done false
35752024-03-06T20:59:26.533Z 5 Acked 1004 Read 3 Sent Done Done false
35762024-03-06T20:59:26.533Z 6 Acked 1005 Write 2 Sent Done Done false
35772024-03-06T20:59:26.533Z 7 Acked 1006 Write 4 Sent Done Done false
35782024-03-06T20:59:26.533Z 8 Acked 1007 Read 5 Sent Done Done false
35792024-03-06T20:59:26.533Z 9 Acked 1008 Read 6 Sent Done Done false
35802024-03-06T20:59:26.533Z 10 Acked 1009 Write 7 Sent Done Done false
35812024-03-06T20:59:26.533Z 11 Acked 1010 Read 4 Sent Done Done false
35822024-03-06T20:59:26.533Z 12 Acked 1011 Flush 0 Sent Done Done false
35832024-03-06T20:59:26.533Z 13 Acked 1012 Write 7 Sent Done Done false
35842024-03-06T20:59:26.533Z 14 Acked 1013 Write 1 Sent Done Done false
35852024-03-06T20:59:26.533Z 15 Acked 1014 Write 7 Sent Done Done false
35862024-03-06T20:59:26.533Z 16 Acked 1015 Flush 0 Sent Done Done false
35872024-03-06T20:59:26.533Z 17 Acked 1016 Write 5 Sent Done Done false
35882024-03-06T20:59:26.534Z 18 Acked 1017 Read 5 Sent Done Done false
35892024-03-06T20:59:26.534Z 19 Acked 1018 Flush 0 Sent Done Done false
35902024-03-06T20:59:26.534Z 20 Acked 1019 Flush 0 Sent Done Done false
35912024-03-06T20:59:26.534Z 21 Acked 1020 Flush 0 Sent Done Done false
35922024-03-06T20:59:26.534Z 22 Acked 1021 Write 7 Sent Done Done false
35932024-03-06T20:59:26.534Z 23 Acked 1022 Read 4 Sent Done Done false
35942024-03-06T20:59:26.534Z 24 Acked 1023 Read 7 Sent Done Done false
35952024-03-06T20:59:26.534Z 25 Acked 1024 Read 10 Sent Done Done false
35962024-03-06T20:59:26.534Z 26 Acked 1025 Read 7 Sent Done Done false
35972024-03-06T20:59:26.534Z 27 Acked 1026 Read 10 Sent Done Done false
35982024-03-06T20:59:26.534Z 28 Acked 1027 Write 8 Sent Done Done false
35992024-03-06T20:59:26.534Z 29 Acked 1028 Read 6 Sent Done Done false
36002024-03-06T20:59:26.534Z 30 Acked 1029 Write 4 Sent Sent Sent false
36012024-03-06T20:59:26.534Z STATES DS:0 DS:1 DS:2 TOTAL
36022024-03-06T20:59:26.534Z New 0 0 0 0
36032024-03-06T20:59:26.534Z Sent 26 1 1 28
36042024-03-06T20:59:26.534Z Done 4 29 29 62
36052024-03-06T20:59:26.534Z Skipped 0 0 0 0
36062024-03-06T20:59:26.534Z Error 0 0 0 0
36072024-03-06T20:59:26.534ZLast Flush: 0 1020 1020
36082024-03-06T20:59:26.534ZDownstairs last five completed:
36092024-03-06T20:59:26.534ZUpstairs last five completed: 30 29 28 27 26
36102024-03-06T20:59:26.534Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T20:59:25.299526812Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1267}
36112024-03-06T20:59:26.534ZWrote out file "/tmp/test_repair_verify.data"
36122024-03-06T20:59:26.538Z
36132024-03-06T20:59:26.541ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
36142024-03-06T20:59:26.557ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
36152024-03-06T20:59:26.557Z 0 000-019 30 32 32 44 48 48 F F F <---
36162024-03-06T20:59:26.557Z 1 020-039 30 30 30 45 45 45 F F F
36172024-03-06T20:59:26.557Z 2 040-059 31 31 31 46 46 46 F T T <---
36182024-03-06T20:59:26.557Z 3 060-079 31 31 31 46 46 46 F F F
36192024-03-06T20:59:26.557Z 4 080-099 31 31 31 46 46 46 F T T <---
36202024-03-06T20:59:26.557Z 5 100-119 26 26 26 36 36 36 F T T <---
36212024-03-06T20:59:26.557Z 6 120-139 30 30 30 45 45 45 F F F
36222024-03-06T20:59:26.557Z 7 140-159 30 30 30 44 44 44 F F F
36232024-03-06T20:59:26.557Z 8 160-179 30 32 32 44 47 47 T F F <---
36242024-03-06T20:59:26.557Z 9 180-199 30 32 32 43 47 47 T F F <---
36252024-03-06T20:59:26.557Z 10 200-219 22 22 22 31 31 31 F F F
36262024-03-06T20:59:26.557Z 11 220-239 26 26 26 36 36 36 F F F
36272024-03-06T20:59:26.557Z 12 240-259 26 26 26 36 36 36 F F F
36282024-03-06T20:59:26.557Z 13 260-279 29 29 29 42 42 42 F F F
36292024-03-06T20:59:26.558Z 14 280-299 30 30 30 44 44 44 F T T <---
36302024-03-06T20:59:26.558Z 15 300-319 31 31 31 46 46 46 F F F
36312024-03-06T20:59:26.558Z 16 320-339 31 32 32 46 47 47 T F F <---
36322024-03-06T20:59:26.583Z 17 340-359 31 32 32 46 49 49 F F F <---
36332024-03-06T20:59:26.583Z 18 360-379 22 32 32 31 47 47 F F F <---
36342024-03-06T20:59:26.583Z 19 380-399 29 29 29 42 42 42 F F F
36352024-03-06T20:59:26.583Z 20 400-419 20 20 20 27 27 27 F F F
36362024-03-06T20:59:26.583Z 21 420-439 26 26 26 34 34 34 F F F
36372024-03-06T20:59:26.583Z 22 440-459 29 29 29 42 42 42 F F F
36382024-03-06T20:59:26.583Z 23 460-479 24 24 24 33 33 33 F F F
36392024-03-06T20:59:26.583Z 24 480-499 30 32 32 44 48 48 F F F <---
36402024-03-06T20:59:26.583Z 25 500-519 30 30 30 44 44 44 F F F
36412024-03-06T20:59:26.583Z 26 520-539 30 30 30 44 44 44 F F F
36422024-03-06T20:59:26.583Z 27 540-559 23 23 23 32 32 32 F F F
36432024-03-06T20:59:26.583Z 28 560-579 26 26 26 36 36 36 F F F
36442024-03-06T20:59:26.584Z 29 580-599 30 32 32 45 48 48 F F F <---
36452024-03-06T20:59:26.584ZMax gen: 32, Max flush: 49
36462024-03-06T20:59:26.584ZError: Difference in extent metadata found!
36472024-03-06T20:59:26.584ZA Difference in extent metadata is expected here
36482024-03-06T20:59:26.584ZOn loop 16
36492024-03-06T20:59:26.584Z
36502024-03-06T20:59:26.584ZVerifying data now
36512024-03-06T20:59:33.594ZNo change after verify
36522024-03-06T20:59:33.688ZLoop: 16 Downstairs dump after verify (and repair):
36532024-03-06T20:59:33.688ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
36542024-03-06T20:59:33.688Z 0 000-019 32 32 32 48 48 48 F F F
36552024-03-06T20:59:33.688Z 1 020-039 30 30 30 45 45 45 F F F
36562024-03-06T20:59:33.688Z 2 040-059 33 33 33 50 50 50 F F F
36572024-03-06T20:59:33.688Z 3 060-079 31 31 31 46 46 46 F F F
36582024-03-06T20:59:33.688Z 4 080-099 33 33 33 50 50 50 F F F
36592024-03-06T20:59:33.688Z 5 100-119 33 33 33 50 50 50 F F F
36602024-03-06T20:59:33.688Z 6 120-139 30 30 30 45 45 45 F F F
36612024-03-06T20:59:33.689Z 7 140-159 30 30 30 44 44 44 F F F
36622024-03-06T20:59:33.689Z 8 160-179 32 32 32 47 47 47 F F F
36632024-03-06T20:59:33.689Z 9 180-199 32 32 32 47 47 47 F F F
36642024-03-06T20:59:33.689Z 10 200-219 22 22 22 31 31 31 F F F
36652024-03-06T20:59:33.689Z 11 220-239 26 26 26 36 36 36 F F F
36662024-03-06T20:59:33.689Z 12 240-259 26 26 26 36 36 36 F F F
36672024-03-06T20:59:33.689Z 13 260-279 29 29 29 42 42 42 F F F
36682024-03-06T20:59:33.689Z 14 280-299 33 33 33 50 50 50 F F F
36692024-03-06T20:59:33.689Z 15 300-319 31 31 31 46 46 46 F F F
36702024-03-06T20:59:33.689Z 16 320-339 32 32 32 47 47 47 F F F
36712024-03-06T20:59:33.689Z 17 340-359 32 32 32 49 49 49 F F F
36722024-03-06T20:59:33.689Z 18 360-379 32 32 32 47 47 47 F F F
36732024-03-06T20:59:33.689Z 19 380-399 29 29 29 42 42 42 F F F
36742024-03-06T20:59:33.689Z 20 400-419 20 20 20 27 27 27 F F F
36752024-03-06T20:59:33.689Z 21 420-439 26 26 26 34 34 34 F F F
36762024-03-06T20:59:33.689Z 22 440-459 29 29 29 42 42 42 F F F
36772024-03-06T20:59:33.689Z 23 460-479 24 24 24 33 33 33 F F F
36782024-03-06T20:59:33.689Z 24 480-499 32 32 32 48 48 48 F F F
36792024-03-06T20:59:33.689Z 25 500-519 30 30 30 44 44 44 F F F
36802024-03-06T20:59:33.689Z 26 520-539 30 30 30 44 44 44 F F F
36812024-03-06T20:59:33.689Z 27 540-559 23 23 23 32 32 32 F F F
36822024-03-06T20:59:33.689Z 28 560-579 26 26 26 36 36 36 F F F
36832024-03-06T20:59:33.689Z 29 580-599 32 32 32 48 48 48 F F F
36842024-03-06T20:59:33.690ZMax gen: 33, Max flush: 50
36852024-03-06T20:59:33.690Z
36862024-03-06T20:59:33.690ZBegin loop 17
36872024-03-06T20:59:33.690ZDownstairs to restart: 2
36882024-03-06T20:59:33.690Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.993614863Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275}
36892024-03-06T20:59:33.690Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.993915932Z","hostname":"Crucible runtime is spawned
36902024-03-06T20:59:33.690Zw-01HRARQTS0TVE14DKZ437MF5TB","pid":1275}
36912024-03-06T20:59:33.690Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.993985537Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275}
36922024-03-06T20:59:33.690Z{"msg":"Crucible d5e2cef0-f4c8-4fe5-9f05-e7179d8b62a1 has session id: 3d1d3a31-9b85-42ef-b310-8ef780512e5d","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.994030924Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
36932024-03-06T20:59:33.690Z{"msg":"Upstairs opts: Upstairs UUID: d5e2cef0-f4c8-4fe5-9f05-e7179d8b62a1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.994066241Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
36942024-03-06T20:59:33.690Z{"msg":"Crucible stats registered with UUID: d5e2cef0-f4c8-4fe5-9f05-e7179d8b62a1","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.994081Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
36952024-03-06T20:59:33.690Z{"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.994101349Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275}
36962024-03-06T20:59:33.690Z{"msg":"d5e2cef0-f4c8-4fe5-9f05-e7179d8b62a1 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.994127157Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
36972024-03-06T20:59:33.690Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.994147666Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"client":"0","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
36982024-03-06T20:59:33.690Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.994169284Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"client":"0","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
36992024-03-06T20:59:33.690Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.994183973Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"client":"1","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37002024-03-06T20:59:33.690Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.994204262Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"client":"1","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37012024-03-06T20:59:33.690Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.994217911Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"client":"2","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37022024-03-06T20:59:33.690Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.994245319Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"client":"2","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37032024-03-06T20:59:33.690Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.994276537Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"":"io task","client":"2","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37042024-03-06T20:59:33.690Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.994306155Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"":"io task","client":"0","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37052024-03-06T20:59:33.690Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.994338093Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"":"io task","client":"1","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37062024-03-06T20:59:33.690Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.994443605Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"":"io task","client":"2","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37072024-03-06T20:59:33.691Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.994487432Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"":"io task","client":"1","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37082024-03-06T20:59:33.691Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.994547338Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"":"io task","client":"0","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37092024-03-06T20:59:33.691Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.995213622Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"client":"0","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37102024-03-06T20:59:33.691Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.99524841Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"client":"0","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37112024-03-06T20:59:33.691Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.995279358Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"client":"1","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37122024-03-06T20:59:33.691Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.995314765Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"client":"1","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37132024-03-06T20:59:33.691Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.995690649Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"client":"2","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37142024-03-06T20:59:33.691Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.995728977Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"client":"2","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37152024-03-06T20:59:33.691Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.995758965Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"client":"1","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37162024-03-06T20:59:33.691Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.996019357Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"client":"0","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37172024-03-06T20:59:33.691Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.996297267Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"client":"1","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37182024-03-06T20:59:33.691Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.996332565Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"client":"1","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37192024-03-06T20:59:33.691Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.996364743Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37202024-03-06T20:59:33.691Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.996395931Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"client":"2","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37212024-03-06T20:59:33.691Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.996520322Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"client":"0","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37222024-03-06T20:59:33.691Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.996569329Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"client":"0","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37232024-03-06T20:59:33.691Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.996597697Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37242024-03-06T20:59:33.691Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.996924164Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"client":"2","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37252024-03-06T20:59:33.691Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.996972901Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"client":"2","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37262024-03-06T20:59:33.691Z{"msg":"[0]R flush_numbers[0..12]: [48, 45, 50, 46, 50, 50, 45, 44, 47, 47, 31, 36]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.997026197Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"":"downstairs","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37272024-03-06T20:59:33.692Z{"msg":"[0]R generation[0..12]: [32, 30, 33, 31, 33, 33, 30, 30, 32, 32, 22, 26]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.997061375Z","Wait for a query_work_queue command to finish before sending IO
37282024-03-06T20:59:33.692Zhostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"":"downstairs","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37292024-03-06T20:59:33.692Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.997100232Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"":"downstairs","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37302024-03-06T20:59:33.692Z{"msg":"[1]R flush_numbers[0..12]: [48, 45, 50, 46, 50, 50, 45, 44, 47, 47, 31, 36]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.9971297Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"":"downstairs","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37312024-03-06T20:59:33.692Z{"msg":"[1]R generation[0..12]: [32, 30, 33, 31, 33, 33, 30, 30, 32, 32, 22, 26]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.997165768Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"":"downstairs","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37322024-03-06T20:59:33.692Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.997199125Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"":"downstairs","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37332024-03-06T20:59:33.692Z{"msg":"[2]R flush_numbers[0..12]: [48, 45, 50, 46, 50, 50, 45, 44, 47, 47, 31, 36]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.997230563Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"":"downstairs","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37342024-03-06T20:59:33.692Z{"msg":"[2]R generation[0..12]: [32, 30, 33, 31, 33, 33, 30, 30, 32, 32, 22, 26]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.997261391Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"":"downstairs","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37352024-03-06T20:59:33.692Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.997293459Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"":"downstairs","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37362024-03-06T20:59:33.692Z{"msg":"Max found gen is 34","v":0,"name":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
37372024-03-06T20:59:33.692Zcrucible","level":30,"time":"2024-03-06T20:59:30.997328796Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"":"downstairs","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37382024-03-06T20:59:33.692Z{"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.997358285Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"":"downstairs","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37392024-03-06T20:59:33.692Z{"msg":"Next flush: 51","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.997387912Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"":"downstairs","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37402024-03-06T20:59:33.692Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.997417221Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"":"downstairs","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37412024-03-06T20:59:33.692Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.997447338Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37422024-03-06T20:59:33.692Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.997476136Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"client":"0","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37432024-03-06T20:59:33.692Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.997499955Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"client":"1","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37442024-03-06T20:59:33.692Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.997529513Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"client":"2","":"downstairs","session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37452024-03-06T20:59:33.693Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.997559911Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37462024-03-06T20:59:33.693Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.997588698Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37472024-03-06T20:59:33.693Z{"msg":"d5e2cef0-f4c8-4fe5-9f05-e7179d8b62a1 is now active with session: 3d1d3a31-9b85-42ef-b310-8ef780512e5d","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.997618536Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37482024-03-06T20:59:33.693Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.997646395Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275,"session_id":"3d1d3a31-9b85-42ef-b310-8ef780512e5d"}
37492024-03-06T20:59:33.693Z{"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:30.997667873Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275}
37502024-03-06T20:59:33.693ZLoading write count information from file "/tmp/test_repair_verify.data"
37512024-03-06T20:59:33.693ZRun Repair workload
37522024-03-06T20:59:33.714ZError: Difference in extent metadata found!
37532024-03-06T20:59:33.714Z01/30 Write block 172 len 16384 data: 3 5 5 5
37542024-03-06T20:59:33.714Z02/30 Write block 343 len 36864 data: 6 6 5 3 3 5 4 5 5
37552024-03-06T20:59:33.714Z03/30 Flush
37562024-03-06T20:59:33.714Z04/30 Flush
37572024-03-06T20:59:33.714Z05/30 Write block 179 len 40960 data: 6 7 8 5 3 3 3 3 3 3
37582024-03-06T20:59:33.715Z06/30 Read block 520 len 32768
37592024-03-06T20:59:33.715Z07/30 Read block 285 len 12288
37602024-03-06T20:59:33.715Z08/30 Write block 412 len 20480 data: 2 4 4 4 4
37612024-03-06T20:59:33.715Z09/30 Read block 82 len 4096
37622024-03-06T20:59:33.715Z10/30 Write block 181 len 40960 data: 9 6 4 4 4 4 4 4 6 6
37632024-03-06T20:59:33.715Z11/30 Read block 587 len 24576
37642024-03-06T20:59:33.715Z12/30 Read block 91 len 32768
37652024-03-06T20:59:33.715Z13/30 Read block 297 len 4096
37662024-03-06T20:59:33.715Z14/30 Write block 7 len 12288 data: 3 2 2
37672024-03-06T20:59:33.715Z15/30 Write block 169 len 28672 data: 2 2 2 4 6 6 6
37682024-03-06T20:59:33.716Z16/30 Read block 345 len 20480
37692024-03-06T20:59:33.716Z17/30 Write block 488 len 24576 data: 6 7 5 5 5 6
37702024-03-06T20:59:33.716Z18/30 Read block 276 len 32768
37712024-03-06T20:59:33.716Z19/30 Read block 495 len 40960
37722024-03-06T20:59:33.716Z20/30 Read block 215 len 16384
37732024-03-06T20:59:33.716Z21/30 Read block 35 len 24576
37742024-03-06T20:59:33.716Z22/30 Write block 539 len 8192 data: 3 5
37752024-03-06T20:59:33.716Z23/30 Read block 96 len 4096
37762024-03-06T20:59:33.716Z24/30 Flush
37772024-03-06T20:59:33.716Z25/30 Write block 157 len 36864 data: 3 4 4 4 3 3 3 2 2
37782024-03-06T20:59:33.716Z26/30 Read block 404 len 24576
37792024-03-06T20:59:33.717Z27/30 Write block 269 len 24576 data: 5 4 3 3 3 3
37802024-03-06T20:59:33.717Z28/30 Read block 277 len 20480
37812024-03-06T20:59:33.717Z29/30 Write block 528 len 20480 data: 5 5 6 6 6
37822024-03-06T20:59:33.717Z30/30 Read block 508 len 16384
37832024-03-06T20:59:33.717Z----------------------------------------------------------------
37842024-03-06T20:59:33.717Z Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30
37852024-03-06T20:59:33.717ZStates: Active Active Active
37862024-03-06T20:59:33.717ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
37872024-03-06T20:59:33.717Z 1 Acked 1000 Write 4 Done Done Done false
37882024-03-06T20:59:33.717Z 2 Acked 1001 Write 9 Done Done Sent false
37892024-03-06T20:59:33.718Z 3 Acked 1002 Flush 0 Done Done Sent false
37902024-03-06T20:59:33.718Z 4 Acked 1003 Flush 0 Done Done Sent false
37912024-03-06T20:59:33.718Z 5 Acked 1004 Write 10 Done Done Sent false
37922024-03-06T20:59:33.718Z 6 Acked 1005 Read 8 Done Done Sent false
37932024-03-06T20:59:33.718Z 7 Acked 1006 Read 3 Done Done Sent false
37942024-03-06T20:59:33.718Z 8 Acked 1007 Write 5 Done Done Sent false
37952024-03-06T20:59:33.718Z 9 Acked 1008 Read 1 Done Done Sent false
37962024-03-06T20:59:33.718Z 10 Acked 1009 Write 10 Done Done Sent false
37972024-03-06T20:59:33.718Z 11 Acked 1010 Read 6 Done Done Sent false
37982024-03-06T20:59:33.718Z 12 Acked 1011 Read 8 Done Done Sent false
37992024-03-06T20:59:33.718Z 13 Acked 1012 Read 1 Done Done Sent false
38002024-03-06T20:59:33.718Z 14 Acked 1013 Write 3 Done Done Sent false
38012024-03-06T20:59:33.718Z 15 Acked 1014 Write 7 Done Done Sent false
38022024-03-06T20:59:33.718Z 16 Acked 1015 Read 5 Done Done Sent false
38032024-03-06T20:59:33.718Z 17 Acked 1016 Write 6 Done Done Sent false
38042024-03-06T20:59:33.718Z 18 Acked 1017 Read 8 Done Done Sent false
38052024-03-06T20:59:33.718Z 19 Acked 1018 Read 10 Done Done Sent false
38062024-03-06T20:59:33.718Z 20 Acked 1019 Read 4 Done Done Sent false
38072024-03-06T20:59:33.718Z 21 Acked 1020 Read 6 Done Done Sent false
38082024-03-06T20:59:33.718Z 22 Acked 1021 Write 2 Done Done Sent false
38092024-03-06T20:59:33.719Z 23 Acked 1022 Read 1 Done Done Sent false
38102024-03-06T20:59:33.719Z 24 Acked 1023 Flush 0 Done Done Sent false
38112024-03-06T20:59:33.719Z 25 Acked 1024 Write 9 Done Done Sent false
38122024-03-06T20:59:33.719Z 26 Acked 1025 Read 6 Done Done Sent false
38132024-03-06T20:59:33.719Z 27 Acked 1026 Write 6 Done Done Sent false
38142024-03-06T20:59:33.719Z 28 Acked 1027 Read 5 Done Done Sent false
38152024-03-06T20:59:33.719Z 29 Acked 1028 Write 5 Done Done Sent false
38162024-03-06T20:59:33.719Z 30 Acked 1029 Read 4 Sent Done Sent false
38172024-03-06T20:59:33.719Z STATES DS:0 DS:1 DS:2 TOTAL
38182024-03-06T20:59:33.719Z New 0 0 0 0
38192024-03-06T20:59:33.719Z Sent 1 0 29 30
38202024-03-06T20:59:33.719Z Done 29 30 1 60
38212024-03-06T20:59:33.719Z Skipped 0 0 0 0
38222024-03-06T20:59:33.719Z Error 0 0 0 0
38232024-03-06T20:59:33.719ZLast Flush: 1023 1023 0
38242024-03-06T20:59:33.719ZDownstairs last five completed:
38252024-03-06T20:59:33.719ZUpstairs last five completed: 30 29 28 27 26
38262024-03-06T20:59:33.719Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T20:59:31.032184874Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1275}
38272024-03-06T20:59:33.719ZWrote out file "/tmp/test_repair_verify.data"
38282024-03-06T20:59:33.719Z
38292024-03-06T20:59:33.719ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
38302024-03-06T20:59:33.719ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38312024-03-06T20:59:33.720Z 0 000-019 34 34 32 53 53 48 F F F <---
38322024-03-06T20:59:33.720Z 1 020-039 30 30 30 45 45 45 F F F
38332024-03-06T20:59:33.720Z 2 040-059 33 33 33 50 50 50 F F F
38342024-03-06T20:59:33.720Z 3 060-079 31 31 31 46 46 46 F F F
38352024-03-06T20:59:33.720Z 4 080-099 33 33 33 50 50 50 F F F
38362024-03-06T20:59:33.720Z 5 100-119 33 33 33 50 50 50 F F F
38372024-03-06T20:59:33.720Z 6 120-139 30 30 30 45 45 45 F F F
38382024-03-06T20:59:33.720Z 7 140-159 30 30 30 44 44 44 T T F <---
38392024-03-06T20:59:33.720Z 8 160-179 34 34 32 53 53 47 T T T <---
38402024-03-06T20:59:33.720Z 9 180-199 34 34 32 53 53 47 F F F <---
38412024-03-06T20:59:33.720Z 10 200-219 22 22 22 31 31 31 F F F
38422024-03-06T20:59:33.720Z 11 220-239 26 26 26 36 36 36 F F F
38432024-03-06T20:59:33.721Z 12 240-259 26 26 26 36 36 36 F F F
38442024-03-06T20:59:33.721Z 13 260-279 29 29 29 42 42 42 T T F <---
38452024-03-06T20:59:33.721Z 14 280-299 33 33 33 50 50 50 F F F
38462024-03-06T20:59:33.721Z 15 300-319 31 31 31 46 46 46 F F F
38472024-03-06T20:59:33.721Z 16 320-339 32 32 32 47 47 47 F F F
38482024-03-06T20:59:33.721Z 17 340-359 34 34 32 51 51 49 F F F <---
38492024-03-06T20:59:33.721Z 18 360-379 32 32 32 47 47 47 F F F
38502024-03-06T20:59:33.721Z 19 380-399 29 29 29 42 42 42 F F F
38512024-03-06T20:59:33.721Z 20 400-419 34 34 20 53 53 27 F F F <---
38522024-03-06T20:59:33.771Z 21 420-439 26 26 26 34 34 34 F F F
38532024-03-06T20:59:33.771Z 22 440-459 29 29 29 42 42 42 F F F
38542024-03-06T20:59:33.771Z 23 460-479 24 24 24 33 33 33 F F F
38552024-03-06T20:59:33.772Z 24 480-499 34 34 32 53 53 48 F F F <---
38562024-03-06T20:59:33.772Z 25 500-519 30 30 30 44 44 44 F F F
38572024-03-06T20:59:33.772Z 26 520-539 34 34 30 53 53 44 T T F <---
38582024-03-06T20:59:33.772Z 27 540-559 34 34 23 53 53 32 F F F <---
38592024-03-06T20:59:33.772Z 28 560-579 26 26 26 36 36 36 F F F
38602024-03-06T20:59:33.772Z 29 580-599 32 32 32 48 48 48 F F F
38612024-03-06T20:59:33.772ZMax gen: 34, Max flush: 53
38622024-03-06T20:59:33.772ZA Difference in extent metadata is expected here
38632024-03-06T20:59:33.772ZOn loop 17
38642024-03-06T20:59:33.772Z
38652024-03-06T20:59:33.772ZVerifying data now
38662024-03-06T20:59:36.847ZNo change after verify
38672024-03-06T20:59:36.847ZLoop: 17 Downstairs dump after verify (and repair):
38682024-03-06T20:59:36.864ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38692024-03-06T20:59:36.864Z 0 000-019 34 34 34 53 53 53 F F F
38702024-03-06T20:59:36.865Z 1 020-039 30 30 30 45 45 45 F F F
38712024-03-06T20:59:36.865Z 2 040-059 33 33 33 50 50 50 F F F
38722024-03-06T20:59:36.865Z 3 060-079 31 31 31 46 46 46 F F F
38732024-03-06T20:59:36.865Z 4 080-099 33 33 33 50 50 50 F F F
38742024-03-06T20:59:36.865Z 5 100-119 33 33 33 50 50 50 F F F
38752024-03-06T20:59:36.866Z 6 120-139 30 30 30 45 45 45 F F F
38762024-03-06T20:59:36.866Z 7 140-159 35 35 35 54 54 54 F F F
38772024-03-06T20:59:36.866Z 8 160-179 35 35 35 54 54 54 F F F
38782024-03-06T20:59:36.866Z 9 180-199 34 34 34 53 53 53 F F F
38792024-03-06T20:59:36.867Z 10 200-219 22 22 22 31 31 31 F F F
38802024-03-06T20:59:36.867Z 11 220-239 26 26 26 36 36 36 F F F
38812024-03-06T20:59:36.867Z 12 240-259 26 26 26 36 36 36 F F F
38822024-03-06T20:59:36.867Z 13 260-279 35 35 35 54 54 54 F F F
38832024-03-06T20:59:36.885Z 14 280-299 33 33 33 50 50 50 F F F
38842024-03-06T20:59:36.885Z 15 300-319 31 31 31 46 46 46 F F F
38852024-03-06T20:59:36.885Z 16 320-339 32 32 32 47 47 47 F F F
38862024-03-06T20:59:36.886Z 17 340-359 34 34 34 51 51 51 F F F
38872024-03-06T20:59:36.886Z 18 360-379 32 32 32 47 47 47 F F F
38882024-03-06T20:59:36.886Z 19 380-399 29 29 29 42 42 42 F F F
38892024-03-06T20:59:36.886Z 20 400-419 34 34 34 53 53 53 F F F
38902024-03-06T20:59:36.886Z 21 420-439 26 26 26 34 34 34 F F F
38912024-03-06T20:59:36.887Z 22 440-459 29 29 29 42 42 42 F F F
38922024-03-06T20:59:36.887Z 23 460-479 24 24 24 33 33 33 F F F
38932024-03-06T20:59:36.887Z 24 480-499 34 34 34 53 53 53 F F F
38942024-03-06T20:59:36.887Z 25 500-519 30 30 30 44 44 44 F F F
38952024-03-06T20:59:36.887Z 26 520-539 35 35 35 54 54 54 F F F
38962024-03-06T20:59:36.887Z 27 540-559 34 34 34 53 53 53 F F F
38972024-03-06T20:59:36.888Z 28 560-579 26 26 26 36 36 36 F F F
38982024-03-06T20:59:36.888Z 29 580-599 32 32 32 48 48 48 F F F
38992024-03-06T20:59:36.888ZMax gen: 35, Max flush: 54
39002024-03-06T20:59:36.888Z
39012024-03-06T20:59:36.888ZBegin loop 18
39022024-03-06T20:59:36.888ZDownstairs to restart: 2
39032024-03-06T20:59:36.936Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.650366893Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283}
39042024-03-06T20:59:36.936Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\nCrucible runtime is spawned
39052024-03-06T20:59:36.936Z cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.650676571Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283}
39062024-03-06T20:59:36.936Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.650708989Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283}
39072024-03-06T20:59:36.937Z{"msg":"Crucible bb01985c-93d0-40ce-9b8a-980ec4df8322 has session id: 2c1a0834-5ebd-425b-abf0-65ecddbe4f67","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.650754106Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39082024-03-06T20:59:36.937Z{"msg":"Upstairs opts: Upstairs UUID: bb01985c-93d0-40ce-9b8a-980ec4df8322, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.650786624Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39092024-03-06T20:59:36.937Z{"msg":"Crucible stats registered with UUID: bb01985c-93d0-40ce-9b8a-980ec4df8322","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.650813512Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39102024-03-06T20:59:36.937Z{"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.65084097Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283}
39112024-03-06T20:59:36.937Z{"msg":"bb01985c-93d0-40ce-9b8a-980ec4df8322 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.650866238Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39122024-03-06T20:59:36.937Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.650891037Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"client":"0","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39132024-03-06T20:59:36.937Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.650910085Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"client":"0","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39142024-03-06T20:59:36.937Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.650942143Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"client":"1","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39152024-03-06T20:59:36.938Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.650960322Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"client":"1","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39162024-03-06T20:59:36.938Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.65097708Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"client":"2","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39172024-03-06T20:59:36.938Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.651009338Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"client":"2","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39182024-03-06T20:59:36.938Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.651034967Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"":"io task","client":"2","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39192024-03-06T20:59:36.938Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.651064265Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"":"io task","client":"0","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39202024-03-06T20:59:36.938Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.651082533Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"":"io task","client":"1","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39212024-03-06T20:59:36.938Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.6512736Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"":"io task","client":"1","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39222024-03-06T20:59:36.938Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.651369393Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"":"io task","client":"2","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39232024-03-06T20:59:36.938Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.65141525Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"":"io task","client":"0","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39242024-03-06T20:59:36.939Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.652063536Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"client":"0","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39252024-03-06T20:59:36.939Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.652127041Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"client":"0","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39262024-03-06T20:59:36.939Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.65215325Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"client":"1","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39272024-03-06T20:59:36.939Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.652193607Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"client":"1","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39282024-03-06T20:59:36.939Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.652636236Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"client":"2","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39292024-03-06T20:59:36.939Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.652692732Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"client":"2","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39302024-03-06T20:59:36.939Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.652782016Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"client":"1","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39312024-03-06T20:59:36.939Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.652820833Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"client":"0","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39322024-03-06T20:59:36.940Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.653316699Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"client":"1","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39332024-03-06T20:59:36.940Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.653364976Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"client":"1","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39342024-03-06T20:59:36.940Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.653384595Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39352024-03-06T20:59:36.940Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.653407183Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"client":"2","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39362024-03-06T20:59:36.940Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.653441741Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"client":"0","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39372024-03-06T20:59:36.940Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.653468789Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"client":"0","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39382024-03-06T20:59:36.940Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.653495677Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39392024-03-06T20:59:36.940Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.653915248Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"client":"2","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39402024-03-06T20:59:36.940Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.653951435Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"client":"2","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39412024-03-06T20:59:36.940Z{"msg":"[0]R flush_numbers[0..12]: [53, 45, 50, 46, 50, 50, 45, 54, 54, 53, 31, 36]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.653968554Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"":"downstairs","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39422024-03-06T20:59:36.941Z{"msg":"[0]R generation[0..12]: [34, 30, 33, 31, 33, 33, 30, 35, 35, 34, 22, 26]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.653990393Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"":"downstairs","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39432024-03-06T20:59:36.941Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.654012011Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"":"downstairs","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39442024-03-06T20:59:36.941Z{"msg":"[1]R flush_numbers[0..12]: [53, 45, 50, 46, 50, 50, 45, 54, 54, 53, 31, 36]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.65402663Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"":"downstairs","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39452024-03-06T20:59:36.941Z{"msg":"[1]R generation[0..12]: [34, 30, 33, 31, 33, 33, 30, 35, 35, 34, 22, 26]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.654047259Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"":"downstairs","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39462024-03-06T20:59:36.941Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
39472024-03-06T20:59:36.941Z,"time":"2024-03-06T20:59:35.654062298Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"":"downstairs","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39482024-03-06T20:59:36.941Z{"msg":"[2]R flush_numbers[0..12]: [53, 45, 50, 46, 50, 50, 45, 54, 54, 53, 31, 36]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.654088226Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"":"downstairs","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39492024-03-06T20:59:36.942Z{"msg":"[2]R generation[0..12]: [34, 30, 33, 31, 33, 33, 30, 35, 35, 34, 22, 26]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.654102955Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"":"downstairs","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39502024-03-06T20:59:36.942Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.654123974Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"":"downstairs","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39512024-03-06T20:59:36.942Z{"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.654137383Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"":"downstairs","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39522024-03-06T20:59:36.942Z{"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.654157481Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"":"downstairs","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39532024-03-06T20:59:36.942Z{"msg":"Next flush: 55","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.654170751Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"":"downstairs","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39542024-03-06T20:59:36.942Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.654191489Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"":"downstairs","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39552024-03-06T20:59:36.942Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.654205098Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39562024-03-06T20:59:36.942Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.654223737Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"client":"0","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39572024-03-06T20:59:36.942Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.654237606Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"client":"1","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39582024-03-06T20:59:36.942Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.654250945Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"client":"2","":"downstairs","session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39592024-03-06T20:59:36.942Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.654265124Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39602024-03-06T20:59:36.943Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.654277303Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39612024-03-06T20:59:36.943Z{"msg":"bb01985c-93d0-40ce-9b8a-980ec4df8322 is now active with session: 2c1a0834-5ebd-425b-abf0-65ecddbe4f67","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.654297832Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39622024-03-06T20:59:36.943Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.654310511Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283,"session_id":"2c1a0834-5ebd-425b-abf0-65ecddbe4f67"}
39632024-03-06T20:59:36.943Z{"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:35.65432391Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283}
39642024-03-06T20:59:36.943ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
39652024-03-06T20:59:36.943ZLoading write count information from file "/tmp/test_repair_verify.data"
39662024-03-06T20:59:36.943ZRun Repair workload
39672024-03-06T20:59:36.943Z01/30 Write block 214 len 20480 data: 4 5 5 5 5
39682024-03-06T20:59:36.943Z02/30 Flush
39692024-03-06T20:59:36.943Z03/30 Write block 537 len 8192 data: 4 4
39702024-03-06T20:59:36.943Z04/30 Flush
39712024-03-06T20:59:36.943Z05/30 Write block 55 len 8192 data: 6 5
39722024-03-06T20:59:36.943Z06/30 Read block 275 len 12288
39732024-03-06T20:59:36.943Z07/30 Write block 312 len 20480 data: 5 4 3 3 3
39742024-03-06T20:59:36.944Z08/30 Flush
39752024-03-06T20:59:36.963Z09/30 Write block 125 len 32768 data: 4 3 3 4 4 4 3 4
39762024-03-06T20:59:36.964Z10/30 Read block 474 len 20480
39772024-03-06T20:59:36.964Z11/30 Read block 109 len 32768
39782024-03-06T20:59:36.964Z12/30 Read block 231 len 4096
39792024-03-06T20:59:36.964Z13/30 Read block 490 len 32768
39802024-03-06T20:59:36.964Z14/30 Read block 146 len 40960
39812024-03-06T20:59:36.964Z15/30 Read block 326 len 28672
39822024-03-06T20:59:36.964Z16/30 Write block 63 len 24576 data: 10 8 8 8 8 6
39832024-03-06T20:59:36.964Z17/30 Read block 69 len 24576
39842024-03-06T20:59:36.964Z18/30 Flush
39852024-03-06T20:59:36.965Z19/30 Write block 388 len 16384 data: 3 3 4 4
39862024-03-06T20:59:36.965Z20/30 Read block 252 len 24576
39872024-03-06T20:59:36.965Z21/30 Write block 143 len 32768 data: 3 3 3 3 4 5 5 5
39882024-03-06T20:59:36.965Z22/30 Write block 15 len 8192 data: 3 2
39892024-03-06T20:59:36.965Z23/30 Write block 58 len 16384 data: 6 7 8 9
39902024-03-06T20:59:36.965Z24/30 Read block 534 len 12288
39912024-03-06T20:59:36.965Z25/30 Write block 542 len 16384 data: 6 6 6 6
39922024-03-06T20:59:36.965Z26/30 Read block 532 len 32768
39932024-03-06T20:59:36.965Z27/30 Write block 465 len 32768 data: 5 5 4 4 4 2 2 2
39942024-03-06T20:59:36.965Z28/30 Read block 404 len 32768
39952024-03-06T20:59:36.965Z29/30 Write block 366 len 12288 data: 3 4 3
39962024-03-06T20:59:36.966Z30/30 Read block 551 len 20480
39972024-03-06T20:59:36.966Z----------------------------------------------------------------
39982024-03-06T20:59:36.966Z Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:28
39992024-03-06T20:59:36.966ZStates: Active Active Active
40002024-03-06T20:59:36.966ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
40012024-03-06T20:59:36.966Z 3 Acked 1002 Write 2 Done Done Sent false
40022024-03-06T20:59:36.966Z 4 Acked 1003 Flush 0 Done Done Sent false
40032024-03-06T20:59:36.966Z 5 Acked 1004 Write 2 Done Done Sent false
40042024-03-06T20:59:36.966Z 6 Acked 1005 Read 3 Done Done Sent false
40052024-03-06T20:59:36.966Z 7 Acked 1006 Write 5 Done Done Sent false
40062024-03-06T20:59:36.966Z 8 Acked 1007 Flush 0 Done Done Sent false
40072024-03-06T20:59:36.967Z 9 Acked 1008 Write 8 Done Done Sent false
40082024-03-06T20:59:36.967Z 10 Acked 1009 Read 5 Done Done Sent false
40092024-03-06T20:59:36.967Z 11 Acked 1010 Read 8 Done Done Sent false
40102024-03-06T20:59:36.967Z 12 Acked 1011 Read 1 Done Done Sent false
40112024-03-06T20:59:36.967Z 13 Acked 1012 Read 8 Done Done Sent false
40122024-03-06T20:59:36.967Z 14 Acked 1013 Read 10 Done Done Sent false
40132024-03-06T20:59:36.967Z 15 Acked 1014 Read 7 Done Done Sent false
40142024-03-06T20:59:36.967Z 16 Acked 1015 Write 6 Done Done Sent false
40152024-03-06T20:59:36.967Z 17 Acked 1016 Read 6 Done Done Sent false
40162024-03-06T20:59:36.968Z 18 Acked 1017 Flush 0 Done Done Sent false
40172024-03-06T20:59:36.968Z 19 Acked 1018 Write 4 Done Done Sent false
40182024-03-06T20:59:36.968Z 20 Acked 1019 Read 6 Done Done Sent false
40192024-03-06T20:59:36.968Z 21 Acked 1020 Write 8 Done Done Sent false
40202024-03-06T20:59:36.968Z 22 Acked 1021 Write 2 Done Done Sent false
40212024-03-06T20:59:36.968Z 23 Acked 1022 Write 4 Done Done Sent false
40222024-03-06T20:59:36.968Z 24 Acked 1023 Read 3 Done Done Sent false
40232024-03-06T20:59:36.968Z 25 Acked 1024 Write 4 Done Done Sent false
40242024-03-06T20:59:36.969Z 26 Acked 1025 Read 8 Done Done Sent false
40252024-03-06T20:59:36.969Z 27 Acked 1026 Write 8 Done Done Sent false
40262024-03-06T20:59:36.969Z 28 Acked 1027 Read 8 Done Done Sent false
40272024-03-06T20:59:36.969Z 29 Acked 1028 Write 3 Done Done Sent false
40282024-03-06T20:59:36.969Z 30 Acked 1029 Read 5 Done Sent Sent false
40292024-03-06T20:59:36.969Z STATES DS:0 DS:1 DS:2 TOTAL
40302024-03-06T20:59:36.969Z New 0 0 0 0
40312024-03-06T20:59:36.969Z Sent 0 1 28 29
40322024-03-06T20:59:36.970Z Done 28 27 0 55
40332024-03-06T20:59:36.970Z Skipped 0 0 0 0
40342024-03-06T20:59:36.970Z Error 0 0 0 0
40352024-03-06T20:59:36.970ZLast Flush: 1017 1017 1001
40362024-03-06T20:59:36.970ZDownstairs last five completed: 1001 1000
40372024-03-06T20:59:36.970ZUpstairs last five completed: 30 29 28 27 26
40382024-03-06T20:59:36.970Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T20:59:35.688823382Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1283}
40392024-03-06T20:59:36.970ZWrote out file "/tmp/test_repair_verify.data"
40402024-03-06T20:59:36.970Z
40412024-03-06T20:59:36.970ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
40422024-03-06T20:59:36.970ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
40432024-03-06T20:59:36.971Z 0 000-019 34 34 34 53 53 53 T T F <---
40442024-03-06T20:59:36.971Z 1 020-039 30 30 30 45 45 45 F F F
40452024-03-06T20:59:36.971Z 2 040-059 36 36 33 57 57 50 T T F <---
40462024-03-06T20:59:36.971Z 3 060-079 36 36 31 58 58 46 T T F <---
40472024-03-06T20:59:36.971Z 4 080-099 33 33 33 50 50 50 F F F
40482024-03-06T20:59:36.971Z 5 100-119 33 33 33 50 50 50 F F F
40492024-03-06T20:59:36.971Z 6 120-139 36 36 30 58 58 45 F F F <---
40502024-03-06T20:59:36.972Z 7 140-159 35 35 35 54 54 54 T T F <---
40512024-03-06T20:59:36.972Z 8 160-179 35 35 35 54 54 54 F F F
40522024-03-06T20:59:36.972Z 9 180-199 34 34 34 53 53 53 F F F
40532024-03-06T20:59:36.972Z 10 200-219 36 36 36 55 55 55 F F F
40542024-03-06T20:59:36.972Z 11 220-239 26 26 26 36 36 36 F F F
40552024-03-06T20:59:36.972Z 12 240-259 26 26 26 36 36 36 F F F
40562024-03-06T20:59:36.972Z 13 260-279 35 35 35 54 54 54 F F F
40572024-03-06T20:59:36.972Z 14 280-299 33 33 33 50 50 50 F F F
40582024-03-06T20:59:36.973Z 15 300-319 36 36 31 57 57 46 F F F <---
40592024-03-06T20:59:36.973Z 16 320-339 32 32 32 47 47 47 F F F
40602024-03-06T20:59:36.973Z 17 340-359 34 34 34 51 51 51 F F F
40612024-03-06T20:59:36.973Z 18 360-379 32 32 32 47 47 47 T T F <---
40622024-03-06T20:59:36.973Z 19 380-399 29 29 29 42 42 42 T T F <---
40632024-03-06T20:59:36.973Z 20 400-419 34 34 34 53 53 53 F F F
40642024-03-06T20:59:36.973Z 21 420-439 26 26 26 34 34 34 F F F
40652024-03-06T20:59:36.974Z 22 440-459 29 29 29 42 42 42 F F F
40662024-03-06T20:59:36.974Z 23 460-479 24 24 24 33 33 33 T T F <---
40672024-03-06T20:59:36.974Z 24 480-499 34 34 34 53 53 53 F F F
40682024-03-06T20:59:36.974Z 25 500-519 30 30 30 44 44 44 F F F
40692024-03-06T20:59:36.974Z 26 520-539 36 36 35 56 56 54 F F F <---
40702024-03-06T20:59:36.975Z 27 540-559 34 34 34 53 53 53 T T F <---
40712024-03-06T20:59:36.975Z 28 560-579 26 26 26 36 36 36 F F F
40722024-03-06T20:59:36.975Z 29 580-599 32 32 32 48 48 48 F F F
40732024-03-06T20:59:36.975ZMax gen: 36, Max flush: 58
40742024-03-06T20:59:36.975ZError: Difference in extent metadata found!
40752024-03-06T20:59:36.986ZA Difference in extent metadata is expected here
40762024-03-06T20:59:36.986ZOn loop 18
40772024-03-06T20:59:36.987Z
40782024-03-06T20:59:36.987ZVerifying data now
40792024-03-06T20:59:41.562ZNo change after verify
40802024-03-06T20:59:41.562ZLoop: 18 Downstairs dump after verify (and repair):
40812024-03-06T20:59:42.524ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
40822024-03-06T20:59:42.524Z 0 000-019 37 37 37 59 59 59 F F F
40832024-03-06T20:59:42.524Z 1 020-039 30 30 30 45 45 45 F F F
40842024-03-06T20:59:42.524Z 2 040-059 37 37 37 59 59 59 F F F
40852024-03-06T20:59:42.524Z 3 060-079 37 37 37 59 59 59 F F F
40862024-03-06T20:59:42.524Z 4 080-099 33 33 33 50 50 50 F F F
40872024-03-06T20:59:42.525Z 5 100-119 33 33 33 50 50 50 F F F
40882024-03-06T20:59:42.525Z 6 120-139 36 36 36 58 58 58 F F F
40892024-03-06T20:59:42.525Z 7 140-159 37 37 37 59 59 59 F F F
40902024-03-06T20:59:42.525Z 8 160-179 35 35 35 54 54 54 F F F
40912024-03-06T20:59:42.525Z 9 180-199 34 34 34 53 53 53 F F F
40922024-03-06T20:59:42.525Z 10 200-219 36 36 36 55 55 55 F F F
40932024-03-06T20:59:42.525Z 11 220-239 26 26 26 36 36 36 F F F
40942024-03-06T20:59:42.525Z 12 240-259 26 26 26 36 36 36 F F F
40952024-03-06T20:59:42.526Z 13 260-279 35 35 35 54 54 54 F F F
40962024-03-06T20:59:42.526Z 14 280-299 33 33 33 50 50 50 F F F
40972024-03-06T20:59:42.526Z 15 300-319 36 36 36 57 57 57 F F F
40982024-03-06T20:59:42.526Z 16 320-339 32 32 32 47 47 47 F F F
40992024-03-06T20:59:42.526Z 17 340-359 34 34 34 51 51 51 F F F
41002024-03-06T20:59:42.526Z 18 360-379 37 37 37 59 59 59 F F F
41012024-03-06T20:59:42.526Z 19 380-399 37 37 37 59 59 59 F F F
41022024-03-06T20:59:42.526Z 20 400-419 34 34 34 53 53 53 F F F
41032024-03-06T20:59:42.526Z 21 420-439 26 26 26 34 34 34 F F F
41042024-03-06T20:59:42.526Z 22 440-459 29 29 29 42 42 42 F F F
41052024-03-06T20:59:42.526Z 23 460-479 37 37 37 59 59 59 F F F
41062024-03-06T20:59:42.526Z 24 480-499 34 34 34 53 53 53 F F F
41072024-03-06T20:59:42.526Z 25 500-519 30 30 30 44 44 44 F F F
41082024-03-06T20:59:42.527Z 26 520-539 36 36 36 56 56 56 F F F
41092024-03-06T20:59:42.527Z 27 540-559 37 37 37 59 59 59 F F F
41102024-03-06T20:59:42.527Z 28 560-579 26 26 26 36 36 36 F F F
41112024-03-06T20:59:42.527Z 29 580-599 32 32 32 48 48 48 F F F
41122024-03-06T20:59:42.527ZMax gen: 37, Max flush: 59
41132024-03-06T20:59:42.527Z
41142024-03-06T20:59:42.527ZBegin loop 19
41152024-03-06T20:59:42.527ZDownstairs to restart: 0
41162024-03-06T20:59:42.576Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.344120284Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291}
41172024-03-06T20:59:42.577Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.344365617Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291}
41182024-03-06T20:59:42.577Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.344394335Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291}
41192024-03-06T20:59:42.577Z{"msg":"Crucible 45f729e5-1058-44e0-b352-a6646d2a94de has session id: b3648dee-1b8f-47d0-a303-d40cde8826e0","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.344418053Z","hostname":"Crucible runtime is spawned
41202024-03-06T20:59:42.577Zw-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41212024-03-06T20:59:42.577Z{"msg":"Upstairs opts: Upstairs UUID: 45f729e5-1058-44e0-b352-a6646d2a94de, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.344489668Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41222024-03-06T20:59:42.577Z{"msg":"Crucible stats registered with UUID: 45f729e5-1058-44e0-b352-a6646d2a94de","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.344519036Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41232024-03-06T20:59:42.578Z{"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.344534255Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291}
41242024-03-06T20:59:42.578Z{"msg":"45f729e5-1058-44e0-b352-a6646d2a94de active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.344600761Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41252024-03-06T20:59:42.578Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.344627359Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"client":"0","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41262024-03-06T20:59:42.578Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.344643638Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"client":"0","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41272024-03-06T20:59:42.578Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.344665476Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"client":"1","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41282024-03-06T20:59:42.579Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.344737951Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"client":"1","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41292024-03-06T20:59:42.579Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.344785898Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"client":"2","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41302024-03-06T20:59:42.579Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.344811806Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"client":"2","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41312024-03-06T20:59:42.579Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.344844644Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"":"io task","client":"0","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41322024-03-06T20:59:42.579Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.34489341Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"":"io task","client":"1","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41332024-03-06T20:59:42.580Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.344919459Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"":"io task","client":"2","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41342024-03-06T20:59:42.580Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.344966465Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"":"io task","client":"1","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41352024-03-06T20:59:42.580Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.345018662Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"":"io task","client":"2","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41362024-03-06T20:59:42.580Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.34504217Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"":"io task","client":"0","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41372024-03-06T20:59:42.580Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.345882092Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"client":"1","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41382024-03-06T20:59:42.581Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.346029042Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"client":"1","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41392024-03-06T20:59:42.581Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.346088278Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"client":"2","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41402024-03-06T20:59:42.581Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.346137435Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"client":"2","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41412024-03-06T20:59:42.581Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.346570135Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"client":"1","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41422024-03-06T20:59:42.581Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.346613522Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"client":"0","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41432024-03-06T20:59:42.581Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.346629901Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"client":"0","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41442024-03-06T20:59:42.581Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.346665088Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"client":"2","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41452024-03-06T20:59:42.581Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.347354681Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"client":"2","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41462024-03-06T20:59:42.582Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.347391578Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"client":"2","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41472024-03-06T20:59:42.582Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.347409037Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41482024-03-06T20:59:42.582Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.347423696Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"client":"1","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41492024-03-06T20:59:42.582Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.347438135Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"client":"1","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41502024-03-06T20:59:42.582Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.347451924Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41512024-03-06T20:59:42.582Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.347476742Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"client":"0","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41522024-03-06T20:59:42.582Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.347930701Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"client":"0","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41532024-03-06T20:59:42.583Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.347971948Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"client":"0","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41542024-03-06T20:59:42.583Z{"msg":"[0]R flush_numbers[0..12]: [59, 45, 59, 59, 50, 50, 58, 59, 54, 53, 55, 36]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.347987537Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"":"downstairs","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41552024-03-06T20:59:42.583Z{"msg":"[0]R generation[0..12]: [37, 30, 37, 37, 33, 33, 36, 37, 35, 34, 36, 26]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.348002286Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"":"downstairs","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41562024-03-06T20:59:42.583Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.348025205Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"":"downstairs","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41572024-03-06T20:59:42.583Z{"msg":"[1]R flush_numbers[0..12]: [59, 45, 59, 59, 50, 50, 58, 59, 54, 53, 55, 36]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.348039904Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"":"downstairs","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41582024-03-06T20:59:42.584Z{"msg":"[1]R generation[0..12]: [37, 30, 37, 37, 33, 33, 36, 37, 35, 34, 36, 26]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.348079391Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"":"downstairs","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41592024-03-06T20:59:42.584Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.348110209Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"":"downstairs","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41602024-03-06T20:59:42.584Z{"msg":"[2]R flush_numbers[0..12]: [59, 45, 59, 59, 50, 50, 58, 59, 54, 53, 55, 36]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.348131437Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"":"downstairs","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41612024-03-06T20:59:42.584Z{"msg":"[2]R generation[0..12]: [37, 30, 37, 37, 33, 33, 36, 37, 35, 34, 36, 26]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
41622024-03-06T20:59:42.585Z,"time":"2024-03-06T20:59:40.348151736Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"":"downstairs","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41632024-03-06T20:59:42.585Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.348172275Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"":"downstairs","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41642024-03-06T20:59:42.585Z{"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.348198733Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"":"downstairs","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41652024-03-06T20:59:42.586Z{"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.348217411Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"":"downstairs","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41662024-03-06T20:59:42.586Z{"msg":"Next flush: 60","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.34823523Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"":"downstairs","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41672024-03-06T20:59:42.586Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.348278057Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"":"downstairs","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41682024-03-06T20:59:42.586Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.348293726Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41692024-03-06T20:59:42.586Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.348308835Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"client":"0","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41702024-03-06T20:59:42.587Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.348328954Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"client":"1","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41712024-03-06T20:59:42.587Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.348342783Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"client":"2","":"downstairs","session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41722024-03-06T20:59:42.587Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.348356182Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41732024-03-06T20:59:42.587Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.348368401Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41742024-03-06T20:59:42.587Z{"msg":"45f729e5-1058-44e0-b352-a6646d2a94de is now active with session: b3648dee-1b8f-47d0-a303-d40cde8826e0","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.3483874Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41752024-03-06T20:59:42.587Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:40.348421887Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291,"session_id":"b3648dee-1b8f-47d0-a303-d40cde8826e0"}
41762024-03-06T20:59:42.587Z{"msg":"The guest has finished waiting for activation with:38","v":0,"name":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
41772024-03-06T20:59:42.587Zcrucible","level":30,"time":"2024-03-06T20:59:40.348445126Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291}
41782024-03-06T20:59:42.588ZLoading write count information from file "/tmp/test_repair_verify.data"
41792024-03-06T20:59:42.588ZRun Repair workload
41802024-03-06T20:59:42.588Z01/30 Write block 386 len 4096 data: 5
41812024-03-06T20:59:42.588Z02/30 Flush
41822024-03-06T20:59:42.588Z03/30 Write block 167 len 12288 data: 2 2 3
41832024-03-06T20:59:42.588Z04/30 Write block 217 len 4096 data: 6
41842024-03-06T20:59:42.589Z05/30 Write block 450 len 28672 data: 3 4 4 4 3 4 3
41852024-03-06T20:59:42.589Z06/30 Read block 40 len 4096
41862024-03-06T20:59:42.589Z07/30 Read block 130 len 28672
41872024-03-06T20:59:42.589Z08/30 Read block 271 len 8192
41882024-03-06T20:59:42.589Z09/30 Write block 421 len 16384 data: 6 4 3 3
41892024-03-06T20:59:42.589Z10/30 Read block 376 len 16384
41902024-03-06T20:59:42.589Z11/30 Write block 150 len 36864 data: 6 5 2 2 2 2 3 4 5
41912024-03-06T20:59:42.590Z12/30 Flush
41922024-03-06T20:59:42.590Z13/30 Write block 1 len 36864 data: 2 2 2 2 2 3 4 3 3
41932024-03-06T20:59:42.590Z14/30 Write block 476 len 12288 data: 3 3 3
41942024-03-06T20:59:42.590Z15/30 Read block 440 len 40960
41952024-03-06T20:59:42.590Z16/30 Write block 491 len 16384 data: 6 6 7 6
41962024-03-06T20:59:42.590Z17/30 Read block 590 len 20480
41972024-03-06T20:59:42.590Z18/30 Read block 20 len 12288
41982024-03-06T20:59:42.590Z19/30 Read block 483 len 8192
41992024-03-06T20:59:42.591Z20/30 Write block 585 len 32768 data: 5 4 5 4 4 5 4 5
42002024-03-06T20:59:42.591Z21/30 Write block 58 len 36864 data: 7 8 9 10 10 11 9 9 9
42012024-03-06T20:59:42.591Z22/30 Write block 284 len 4096 data: 5
42022024-03-06T20:59:42.591Z23/30 Read block 235 len 4096
42032024-03-06T20:59:42.591Z24/30 Read block 233 len 24576
42042024-03-06T20:59:42.591Z25/30 Read block 298 len 12288
42052024-03-06T20:59:42.592Z26/30 Read block 447 len 40960
42062024-03-06T20:59:42.592Z27/30 Write block 510 len 32768 data: 2 2 3 2 2 2 4 4
42072024-03-06T20:59:42.592Z28/30 Read block 299 len 20480
42082024-03-06T20:59:42.592Z29/30 Read block 189 len 16384
42092024-03-06T20:59:42.592Z30/30 Read block 559 len 28672
42102024-03-06T20:59:42.593Z----------------------------------------------------------------
42112024-03-06T20:59:42.593Z Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:28
42122024-03-06T20:59:42.593ZStates: Active Active Active
42132024-03-06T20:59:42.593ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
42142024-03-06T20:59:42.593Z 3 Acked 1002 Write 3 Done Done Done false
42152024-03-06T20:59:42.593Z 4 Acked 1003 Write 1 Done Done Done false
42162024-03-06T20:59:42.617Z 5 Acked 1004 Write 7 Done Done Done false
42172024-03-06T20:59:42.618Z 6 Acked 1005 Read 1 Done Done Done false
42182024-03-06T20:59:42.618Z 7 Acked 1006 Read 7 Sent Done Done false
42192024-03-06T20:59:42.619Z 8 Acked 1007 Read 2 Sent Done Done false
42202024-03-06T20:59:42.619Z 9 Acked 1008 Write 4 Sent Done Done false
42212024-03-06T20:59:42.619Z 10 Acked 1009 Read 4 Sent Done Done false
42222024-03-06T20:59:42.619Z 11 Acked 1010 Write 9 Sent Done Done false
42232024-03-06T20:59:42.620Z 12 Acked 1011 Flush 0 Sent Done Done false
42242024-03-06T20:59:42.620Z 13 Acked 1012 Write 9 Sent Done Done false
42252024-03-06T20:59:42.620Z 14 Acked 1013 Write 3 Sent Done Done false
42262024-03-06T20:59:42.620Z 15 Acked 1014 Read 10 Sent Done Done false
42272024-03-06T20:59:42.620Z 16 Acked 1015 Write 4 Sent Done Done false
42282024-03-06T20:59:42.620Z 17 Acked 1016 Read 5 Sent Done Done false
42292024-03-06T20:59:42.621Z 18 Acked 1017 Read 3 Sent Done Done false
42302024-03-06T20:59:42.621Z 19 Acked 1018 Read 2 Sent Done Done false
42312024-03-06T20:59:42.621Z 20 Acked 1019 Write 8 Sent Done Done false
42322024-03-06T20:59:42.621Z 21 Acked 1020 Write 9 Sent Done Done false
42332024-03-06T20:59:42.622Z 22 Acked 1021 Write 1 Sent Done Done false
42342024-03-06T20:59:42.622Z 23 Acked 1022 Read 1 Sent Done Done false
42352024-03-06T20:59:42.622Z 24 Acked 1023 Read 6 Sent Done Done false
42362024-03-06T20:59:42.622Z 25 Acked 1024 Read 3 Sent Done Done false
42372024-03-06T20:59:42.622Z 26 Acked 1025 Read 10 Sent Done Done false
42382024-03-06T20:59:42.622Z 27 Acked 1026 Write 8 Sent Done Done false
42392024-03-06T20:59:42.623Z 28 Acked 1027 Read 5 Sent Done Done false
42402024-03-06T20:59:42.623Z 29 Acked 1028 Read 4 Sent Done Done false
42412024-03-06T20:59:42.623Z 30 Acked 1029 Read 7 Sent Done Sent false
42422024-03-06T20:59:42.623Z STATES DS:0 DS:1 DS:2 TOTAL
42432024-03-06T20:59:42.623Z New 0 0 0 0
42442024-03-06T20:59:42.623Z Sent 24 0 1 25
42452024-03-06T20:59:42.624Z Done 4 28 27 59
42462024-03-06T20:59:42.624Z Skipped 0 0 0 0
42472024-03-06T20:59:42.624Z Error 0 0 0 0
42482024-03-06T20:59:42.624ZLast Flush: 1001 1011 1011
42492024-03-06T20:59:42.624ZDownstairs last five completed: 1001 1000
42502024-03-06T20:59:42.625ZUpstairs last five completed: 30 29 28 27 26
42512024-03-06T20:59:42.625Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T20:59:40.382561814Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1291}
42522024-03-06T20:59:42.625ZWrote out file "/tmp/test_repair_verify.data"
42532024-03-06T20:59:42.625Z
42542024-03-06T20:59:42.626ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
42552024-03-06T20:59:42.626ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
42562024-03-06T20:59:42.626Z 0 000-019 37 37 37 59 59 59 F T T <---
42572024-03-06T20:59:42.626Z 1 020-039 30 30 30 45 45 45 F F F
42582024-03-06T20:59:42.626Z 2 040-059 37 37 37 59 59 59 F T T <---
42592024-03-06T20:59:42.627Z 3 060-079 37 37 37 59 59 59 F T T <---
42602024-03-06T20:59:42.627Z 4 080-099 33 33 33 50 50 50 F F F
42612024-03-06T20:59:42.627Z 5 100-119 33 33 33 50 50 50 F F F
42622024-03-06T20:59:42.627Z 6 120-139 36 36 36 58 58 58 F F F
42632024-03-06T20:59:42.627Z 7 140-159 37 38 38 59 61 61 F F F <---
42642024-03-06T20:59:42.628Z 8 160-179 35 38 38 54 61 61 T F F <---
42652024-03-06T20:59:42.628Z 9 180-199 34 34 34 53 53 53 F F F
42662024-03-06T20:59:42.628Z 10 200-219 36 38 38 55 61 61 T F F <---
42672024-03-06T20:59:42.628Z 11 220-239 26 26 26 36 36 36 F F F
42682024-03-06T20:59:42.628Z 12 240-259 26 26 26 36 36 36 F F F
42692024-03-06T20:59:42.629Z 13 260-279 35 35 35 54 54 54 F F F
42702024-03-06T20:59:42.629Z 14 280-299 33 33 33 50 50 50 F T T <---
42712024-03-06T20:59:42.629Z 15 300-319 36 36 36 57 57 57 F F F
42722024-03-06T20:59:42.629Z 16 320-339 32 32 32 47 47 47 F F F
42732024-03-06T20:59:42.629Z 17 340-359 34 34 34 51 51 51 F F F
42742024-03-06T20:59:42.630Z 18 360-379 37 37 37 59 59 59 F F F
42752024-03-06T20:59:42.630Z 19 380-399 38 38 38 60 60 60 F F F
42762024-03-06T20:59:42.630Z 20 400-419 34 34 34 53 53 53 F F F
42772024-03-06T20:59:42.630Z 21 420-439 26 38 38 34 61 61 F F F <---
42782024-03-06T20:59:42.630Z 22 440-459 29 38 38 42 61 61 T F F <---
42792024-03-06T20:59:42.630Z 23 460-479 37 37 37 59 59 59 F T T <---
42802024-03-06T20:59:42.631Z 24 480-499 34 34 34 53 53 53 F T T <---
42812024-03-06T20:59:42.631Z 25 500-519 30 30 30 44 44 44 F T T <---
42822024-03-06T20:59:42.631Z 26 520-539 36 36 36 56 56 56 F F F
42832024-03-06T20:59:42.631Z 27 540-559 37 37 37 59 59 59 F F F
42842024-03-06T20:59:42.631Z 28 560-579 26 26 26 36 36 36 F F F
42852024-03-06T20:59:42.631Z 29 580-599 32 32 32 48 48 48 F T T <---
42862024-03-06T20:59:42.631ZMax gen: 38, Max flush: 61
42872024-03-06T20:59:42.632ZError: Difference in extent metadata found!
42882024-03-06T20:59:42.632ZA Difference in extent metadata is expected here
42892024-03-06T20:59:42.632ZOn loop 19
42902024-03-06T20:59:42.632Z
42912024-03-06T20:59:42.632ZVerifying data now
42922024-03-06T20:59:47.343ZNo change after verify
42932024-03-06T20:59:47.343ZLoop: 19 Downstairs dump after verify (and repair):
42942024-03-06T20:59:47.363ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
42952024-03-06T20:59:47.363Z 0 000-019 39 39 39 62 62 62 F F F
42962024-03-06T20:59:47.364Z 1 020-039 30 30 30 45 45 45 F F F
42972024-03-06T20:59:47.364Z 2 040-059 39 39 39 62 62 62 F F F
42982024-03-06T20:59:47.364Z 3 060-079 39 39 39 62 62 62 F F F
42992024-03-06T20:59:47.365Z 4 080-099 33 33 33 50 50 50 F F F
43002024-03-06T20:59:47.365Z 5 100-119 33 33 33 50 50 50 F F F
43012024-03-06T20:59:47.365Z 6 120-139 36 36 36 58 58 58 F F F
43022024-03-06T20:59:47.366Z 7 140-159 38 38 38 61 61 61 F F F
43032024-03-06T20:59:47.366Z 8 160-179 38 38 38 61 61 61 F F F
43042024-03-06T20:59:47.367Z 9 180-199 34 34 34 53 53 53 F F F
43052024-03-06T20:59:47.367Z 10 200-219 38 38 38 61 61 61 F F F
43062024-03-06T20:59:47.367Z 11 220-239 26 26 26 36 36 36 F F F
43072024-03-06T20:59:47.368Z 12 240-259 26 26 26 36 36 36 F F F
43082024-03-06T20:59:47.368Z 13 260-279 35 35 35 54 54 54 F F F
43092024-03-06T20:59:47.368Z 14 280-299 39 39 39 62 62 62 F F F
43102024-03-06T20:59:47.368Z 15 300-319 36 36 36 57 57 57 F F F
43112024-03-06T20:59:47.368Z 16 320-339 32 32 32 47 47 47 F F F
43122024-03-06T20:59:47.369Z 17 340-359 34 34 34 51 51 51 F F F
43132024-03-06T20:59:47.369Z 18 360-379 37 37 37 59 59 59 F F F
43142024-03-06T20:59:47.369Z 19 380-399 38 38 38 60 60 60 F F F
43152024-03-06T20:59:47.370Z 20 400-419 34 34 34 53 53 53 F F F
43162024-03-06T20:59:47.370Z 21 420-439 38 38 38 61 61 61 F F F
43172024-03-06T20:59:47.371Z 22 440-459 38 38 38 61 61 61 F F F
43182024-03-06T20:59:47.371Z 23 460-479 39 39 39 62 62 62 F F F
43192024-03-06T20:59:47.371Z 24 480-499 39 39 39 62 62 62 F F F
43202024-03-06T20:59:47.372Z 25 500-519 39 39 39 62 62 62 F F F
43212024-03-06T20:59:47.372Z 26 520-539 36 36 36 56 56 56 F F F
43222024-03-06T20:59:47.372Z 27 540-559 37 37 37 59 59 59 F F F
43232024-03-06T20:59:47.372Z 28 560-579 26 26 26 36 36 36 F F F
43242024-03-06T20:59:47.372Z 29 580-599 39 39 39 62 62 62 F F F
43252024-03-06T20:59:47.373ZMax gen: 39, Max flush: 62
43262024-03-06T20:59:47.423Z
43272024-03-06T20:59:47.423ZBegin loop 20
43282024-03-06T20:59:47.423ZDownstairs to restart: 2
43292024-03-06T20:59:47.423Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.142869272Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299}
43302024-03-06T20:59:47.424Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.14318989Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299}
43312024-03-06T20:59:47.424Z{"msg":"Crucible runtime is spawned
43322024-03-06T20:59:47.424ZUpstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.143224908Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299}
43332024-03-06T20:59:47.424Z{"msg":"Crucible 301bfc10-e81f-4e23-8fb5-fde96a0c725c has session id: 55ace3fe-9074-4e08-a867-5ef01eb97f8e","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.143276524Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43342024-03-06T20:59:47.424Z{"msg":"Upstairs opts: Upstairs UUID: 301bfc10-e81f-4e23-8fb5-fde96a0c725c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.143319221Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43352024-03-06T20:59:47.425Z{"msg":"Crucible stats registered with UUID: 301bfc10-e81f-4e23-8fb5-fde96a0c725c","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.14333727Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43362024-03-06T20:59:47.425Z{"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.143362198Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299}
43372024-03-06T20:59:47.425Z{"msg":"301bfc10-e81f-4e23-8fb5-fde96a0c725c active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.143377337Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43382024-03-06T20:59:47.425Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.143401016Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"client":"0","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43392024-03-06T20:59:47.426Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.143419664Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"client":"0","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43402024-03-06T20:59:47.426Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.143437713Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"client":"1","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43412024-03-06T20:59:47.426Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.143462422Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"client":"1","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43422024-03-06T20:59:47.426Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.143528787Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"client":"2","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43432024-03-06T20:59:47.426Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.143559055Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"client":"2","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43442024-03-06T20:59:47.427Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.143597652Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"":"io task","client":"0","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43452024-03-06T20:59:47.427Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.14363467Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"":"io task","client":"1","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43462024-03-06T20:59:47.427Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.143675127Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"":"io task","client":"2","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43472024-03-06T20:59:47.427Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.143705295Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"":"io task","client":"2","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43482024-03-06T20:59:47.427Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.143750222Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"":"io task","client":"1","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43492024-03-06T20:59:47.427Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.143935169Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"":"io task","client":"0","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43502024-03-06T20:59:47.428Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.144363319Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"client":"1","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43512024-03-06T20:59:47.428Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.144428325Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"client":"1","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43522024-03-06T20:59:47.428Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.144471132Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"client":"0","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43532024-03-06T20:59:47.428Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.144490251Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"client":"0","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43542024-03-06T20:59:47.428Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.144839856Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"client":"1","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43552024-03-06T20:59:47.428Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.144944579Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"client":"0","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43562024-03-06T20:59:47.428Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.145355961Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"client":"1","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43572024-03-06T20:59:47.428Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.145403728Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"client":"1","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43582024-03-06T20:59:47.429Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.145488512Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43592024-03-06T20:59:47.429Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.145570676Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"client":"2","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43602024-03-06T20:59:47.429Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.145627022Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"client":"2","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43612024-03-06T20:59:47.429Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.145680849Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"client":"0","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43622024-03-06T20:59:47.429Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.145707887Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"client":"0","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43632024-03-06T20:59:47.429Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.145736955Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43642024-03-06T20:59:47.429Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.146118778Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"client":"2","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43652024-03-06T20:59:47.429Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.146486213Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"client":"2","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43662024-03-06T20:59:47.429Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.146522351Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"client":"2","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43672024-03-06T20:59:47.430Z{"msg":"[0]R flush_numbers[0..12]: [62, 45, 62, 62, 50, 50, 58, 61, 61, 53, 61, 36]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.146537879Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"":"downstairs","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43682024-03-06T20:59:47.430Z{"msg":"[0]R generation[0..12]: [39, 30, 39, 39, 33, 33, 36, 38, 38, 34, 38, 26]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.146559568Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"":"downstairs","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43692024-03-06T20:59:47.430Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.146574197Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"":"downstairs","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43702024-03-06T20:59:47.430Z{"msg":"[1]R flush_numbers[0..12]: [62, 45, 62, 62, 50, 50, 58, 61, 61, 53, 61, 36]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.146594795Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"":"downstairs","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43712024-03-06T20:59:47.430Z{"msg":"[1]R generation[0..12]: [39, 30, 39, 39, 33, 33, 36, 38, 38, 34, 38, 26]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.146623913Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"":"downstairs","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43722024-03-06T20:59:47.431Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.146638432Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"":"downstairs","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43732024-03-06T20:59:47.431Z{"msg":"[2]R flush_numbers[0..12]: [62, 45, 62, 62, 50, 50, 58, 61, 61, 53, 61, 36]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.146652192Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"":"downstairs","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43742024-03-06T20:59:47.431ZWait for a query_work_queue command to finish before sending IO
43752024-03-06T20:59:47.431Z{"msg":"[2]R generation[0..12]: [39, 30, 39, 39, 33, 33, 36, 38, 38, 34, 38, 26]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.14667663Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"":"downstairs","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43762024-03-06T20:59:47.431Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.146691929Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"":"downstairs","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43772024-03-06T20:59:47.431Z{"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.146711838Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"":"downstairs","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43782024-03-06T20:59:47.431Z{"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.146725437Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"":"downstairs","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43792024-03-06T20:59:47.431Z{"msg":"Next flush: 63","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.146749745Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"":"downstairs","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43802024-03-06T20:59:47.432Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.146762984Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"":"downstairs","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43812024-03-06T20:59:47.432Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.146776183Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43822024-03-06T20:59:47.432Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.146790222Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"client":"0","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43832024-03-06T20:59:47.432Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.146809781Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"client":"1","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43842024-03-06T20:59:47.432Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.14682358Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"client":"2","":"downstairs","session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43852024-03-06T20:59:47.432Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.146837899Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43862024-03-06T20:59:47.433Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.146850078Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43872024-03-06T20:59:47.433Z{"msg":"301bfc10-e81f-4e23-8fb5-fde96a0c725c is now active with session: 55ace3fe-9074-4e08-a867-5ef01eb97f8e","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.146863677Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43882024-03-06T20:59:47.433Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.146875676Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299,"session_id":"55ace3fe-9074-4e08-a867-5ef01eb97f8e"}
43892024-03-06T20:59:47.433Z{"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:46.146895435Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299}
43902024-03-06T20:59:47.433ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
43912024-03-06T20:59:47.433ZLoading write count information from file "/tmp/test_repair_verify.data"
43922024-03-06T20:59:47.433ZRun Repair workload
43932024-03-06T20:59:47.434Z01/30 Write block 363 len 12288 data: 6 5 4
43942024-03-06T20:59:47.434Z02/30 Flush
43952024-03-06T20:59:47.460Z03/30 Write block 304 len 32768 data: 3 5 5 5 5 5 5 5
43962024-03-06T20:59:47.460Z04/30 Write block 93 len 28672 data: 3 4 3 3 4 5 5
43972024-03-06T20:59:47.461Z05/30 Write block 235 len 28672 data: 6 6 7 7 7 6 7
43982024-03-06T20:59:47.461Z06/30 Read block 137 len 36864
43992024-03-06T20:59:47.461Z07/30 Read block 328 len 8192
44002024-03-06T20:59:47.461Z08/30 Read block 134 len 28672
44012024-03-06T20:59:47.461Z09/30 Write block 274 len 24576 data: 4 2 2 2 2 2
44022024-03-06T20:59:47.462Z10/30 Read block 278 len 16384
44032024-03-06T20:59:47.462Z11/30 Read block 48 len 4096
44042024-03-06T20:59:47.462Z12/30 Flush
44052024-03-06T20:59:47.462Z13/30 Write block 513 len 28672 data: 3 3 3 5 5 4 4
44062024-03-06T20:59:47.462Z14/30 Write block 274 len 12288 data: 5 3 3
44072024-03-06T20:59:47.463Z15/30 Write block 5 len 4096 data: 3
44082024-03-06T20:59:47.463Z16/30 Write block 372 len 16384 data: 4 3 3 4
44092024-03-06T20:59:47.463Z17/30 Read block 295 len 40960
44102024-03-06T20:59:47.463Z18/30 Write block 560 len 28672 data: 3 3 3 2 2 2 3
44112024-03-06T20:59:47.463Z19/30 Read block 106 len 24576
44122024-03-06T20:59:47.463Z20/30 Write block 434 len 32768 data: 4 3 2 3 3 4 4 2
44132024-03-06T20:59:47.464Z21/30 Write block 306 len 12288 data: 6 6 6
44142024-03-06T20:59:47.464Z22/30 Write block 289 len 32768 data: 10 10 7 4 3 3 5 4
44152024-03-06T20:59:47.464Z23/30 Read block 27 len 20480
44162024-03-06T20:59:47.465Z24/30 Flush
44172024-03-06T20:59:47.465Z25/30 Write block 448 len 20480 data: 4 3 4 5 5
44182024-03-06T20:59:47.465Z26/30 Read block 269 len 40960
44192024-03-06T20:59:47.465Z27/30 Write block 19 len 8192 data: 2 3
44202024-03-06T20:59:47.465Z28/30 Read block 224 len 36864
44212024-03-06T20:59:47.466Z29/30 Write block 173 len 28672 data: 7 7 7 5 5 4 7
44222024-03-06T20:59:47.466Z30/30 Read block 562 len 8192
44232024-03-06T20:59:47.466Z----------------------------------------------------------------
44242024-03-06T20:59:47.466Z Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:28
44252024-03-06T20:59:47.466ZStates: Active Active Active
44262024-03-06T20:59:47.466ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
44272024-03-06T20:59:47.466Z 3 Acked 1002 Write 8 Done Done Done false
44282024-03-06T20:59:47.467Z 4 Acked 1003 Write 7 Done Done Sent false
44292024-03-06T20:59:47.467Z 5 Acked 1004 Write 7 Done Done Sent false
44302024-03-06T20:59:47.467Z 6 Acked 1005 Read 9 Done Done Sent false
44312024-03-06T20:59:47.467Z 7 Acked 1006 Read 2 Done Done Sent false
44322024-03-06T20:59:47.467Z 8 Acked 1007 Read 7 Done Done Sent false
44332024-03-06T20:59:47.467Z 9 Acked 1008 Write 6 Done Done Sent false
44342024-03-06T20:59:47.468Z 10 Acked 1009 Read 4 Done Done Sent false
44352024-03-06T20:59:47.468Z 11 Acked 1010 Read 1 Done Done Sent false
44362024-03-06T20:59:47.468Z 12 Acked 1011 Flush 0 Done Done Sent false
44372024-03-06T20:59:47.468Z 13 Acked 1012 Write 7 Done Done Sent false
44382024-03-06T20:59:47.468Z 14 Acked 1013 Write 3 Done Done Sent false
44392024-03-06T20:59:47.469Z 15 Acked 1014 Write 1 Done Done Sent false
44402024-03-06T20:59:47.469Z 16 Acked 1015 Write 4 Done Done Sent false
44412024-03-06T20:59:47.469Z 17 Acked 1016 Read 10 Done Done Sent false
44422024-03-06T20:59:47.469Z 18 Acked 1017 Write 7 Done Done Sent false
44432024-03-06T20:59:47.469Z 19 Acked 1018 Read 6 Done Done Sent false
44442024-03-06T20:59:47.470Z 20 Acked 1019 Write 8 Done Done Sent false
44452024-03-06T20:59:47.470Z 21 Acked 1020 Write 3 Done Done Sent false
44462024-03-06T20:59:47.470Z 22 Acked 1021 Write 8 Done Done Sent false
44472024-03-06T20:59:47.470Z 23 Acked 1022 Read 5 Done Done Sent false
44482024-03-06T20:59:47.470Z 24 Acked 1023 Flush 0 Done Done Sent false
44492024-03-06T20:59:47.471Z 25 Acked 1024 Write 5 Done Done Sent false
44502024-03-06T20:59:47.471Z 26 Acked 1025 Read 10 Done Done Sent false
44512024-03-06T20:59:47.471Z 27 Acked 1026 Write 2 Done Done Sent false
44522024-03-06T20:59:47.471Z 28 Acked 1027 Read 9 Done Done Sent false
44532024-03-06T20:59:47.471Z 29 Acked 1028 Write 7 Done Done Sent false
44542024-03-06T20:59:47.472Z 30 Acked 1029 Read 2 Done Sent Sent false
44552024-03-06T20:59:47.472Z STATES DS:0 DS:1 DS:2 TOTAL
44562024-03-06T20:59:47.472Z New 0 0 0 0
44572024-03-06T20:59:47.473Z Sent 0 1 27 28
44582024-03-06T20:59:47.473Z Done 28 27 1 56
44592024-03-06T20:59:47.473Z Skipped 0 0 0 0
44602024-03-06T20:59:47.473Z Error 0 0 0 0
44612024-03-06T20:59:47.473ZLast Flush: 1023 1023 1001
44622024-03-06T20:59:47.474ZDownstairs last five completed: 1001 1000
44632024-03-06T20:59:47.474ZUpstairs last five completed: 30 29 28 27 26
44642024-03-06T20:59:47.474Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T20:59:46.183126098Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1299}
44652024-03-06T20:59:47.474ZWrote out file "/tmp/test_repair_verify.data"
44662024-03-06T20:59:47.475Z
44672024-03-06T20:59:47.475ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
44682024-03-06T20:59:47.475ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
44692024-03-06T20:59:47.475Z 0 000-019 40 40 39 65 65 62 T T F <---
44702024-03-06T20:59:47.476Z 1 020-039 30 30 30 45 45 45 T T F <---
44712024-03-06T20:59:47.476Z 2 040-059 39 39 39 62 62 62 F F F
44722024-03-06T20:59:47.476Z 3 060-079 39 39 39 62 62 62 F F F
44732024-03-06T20:59:47.476Z 4 080-099 40 40 33 64 64 50 F F F <---
44742024-03-06T20:59:47.477Z 5 100-119 33 33 33 50 50 50 F F F
44752024-03-06T20:59:47.477Z 6 120-139 36 36 36 58 58 58 F F F
44762024-03-06T20:59:47.477Z 7 140-159 38 38 38 61 61 61 F F F
44772024-03-06T20:59:47.478Z 8 160-179 38 38 38 61 61 61 T T F <---
44782024-03-06T20:59:47.478Z 9 180-199 34 34 34 53 53 53 F F F
44792024-03-06T20:59:47.478Z 10 200-219 38 38 38 61 61 61 F F F
44802024-03-06T20:59:47.478Z 11 220-239 40 40 26 64 64 36 F F F <---
44812024-03-06T20:59:47.478Z 12 240-259 40 40 26 64 64 36 F F F <---
44822024-03-06T20:59:47.479Z 13 260-279 40 40 35 65 65 54 F F F <---
44832024-03-06T20:59:47.479Z 14 280-299 40 40 39 65 65 62 F F F <---
44842024-03-06T20:59:47.479Z 15 300-319 40 40 36 65 65 57 F F T <---
44852024-03-06T20:59:47.479Z 16 320-339 32 32 32 47 47 47 F F F
44862024-03-06T20:59:47.479Z 17 340-359 34 34 34 51 51 51 F F F
44872024-03-06T20:59:47.480Z 18 360-379 40 40 40 65 65 63 F F F <---
44882024-03-06T20:59:47.480Z 19 380-399 38 38 38 60 60 60 F F F
44892024-03-06T20:59:47.480Z 20 400-419 34 34 34 53 53 53 F F F
44902024-03-06T20:59:47.480Z 21 420-439 40 40 38 65 65 61 F F F <---
44912024-03-06T20:59:47.481Z 22 440-459 40 40 38 65 65 61 T T F <---
44922024-03-06T20:59:47.481Z 23 460-479 39 39 39 62 62 62 F F F
44932024-03-06T20:59:47.481Z 24 480-499 39 39 39 62 62 62 F F F
44942024-03-06T20:59:47.481Z 25 500-519 40 40 39 65 65 62 F F F <---
44952024-03-06T20:59:47.502Z 26 520-539 36 36 36 56 56 56 F F F
44962024-03-06T20:59:47.503Z 27 540-559 37 37 37 59 59 59 F F F
44972024-03-06T20:59:47.503Z 28 560-579 40 40 26 65 65 36 F F F <---
44982024-03-06T20:59:47.503Z 29 580-599 39 39 39 62 62 62 F F F
44992024-03-06T20:59:47.504ZMax gen: 40, Max flush: 65
45002024-03-06T20:59:47.504ZError: Difference in extent metadata found!
45012024-03-06T20:59:47.504ZA Difference in extent metadata is expected here
45022024-03-06T20:59:47.504ZOn loop 20
45032024-03-06T20:59:47.504Z
45042024-03-06T20:59:47.504ZVerifying data now
45052024-03-06T20:59:52.182ZNo change after verify
45062024-03-06T20:59:52.182ZLoop: 20 Downstairs dump after verify (and repair):
45072024-03-06T20:59:52.637ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
45082024-03-06T20:59:52.637Z 0 000-019 41 41 41 66 66 66 F F F
45092024-03-06T20:59:52.638Z 1 020-039 41 41 41 66 66 66 F F F
45102024-03-06T20:59:52.638Z 2 040-059 39 39 39 62 62 62 F F F
45112024-03-06T20:59:52.638Z 3 060-079 39 39 39 62 62 62 F F F
45122024-03-06T20:59:52.638Z 4 080-099 40 40 40 64 64 64 F F F
45132024-03-06T20:59:52.638Z 5 100-119 33 33 33 50 50 50 F F F
45142024-03-06T20:59:52.638Z 6 120-139 36 36 36 58 58 58 F F F
45152024-03-06T20:59:52.639Z 7 140-159 38 38 38 61 61 61 F F F
45162024-03-06T20:59:52.639Z 8 160-179 41 41 41 66 66 66 F F F
45172024-03-06T20:59:52.639Z 9 180-199 34 34 34 53 53 53 F F F
45182024-03-06T20:59:52.639Z 10 200-219 38 38 38 61 61 61 F F F
45192024-03-06T20:59:52.639Z 11 220-239 40 40 40 64 64 64 F F F
45202024-03-06T20:59:52.639Z 12 240-259 40 40 40 64 64 64 F F F
45212024-03-06T20:59:52.639Z 13 260-279 40 40 40 65 65 65 F F F
45222024-03-06T20:59:52.640Z 14 280-299 40 40 40 65 65 65 F F F
45232024-03-06T20:59:52.640Z 15 300-319 40 40 40 65 65 65 F F F
45242024-03-06T20:59:52.640Z 16 320-339 32 32 32 47 47 47 F F F
45252024-03-06T20:59:52.640Z 17 340-359 34 34 34 51 51 51 F F F
45262024-03-06T20:59:52.640Z 18 360-379 40 40 40 65 65 65 F F F
45272024-03-06T20:59:52.640Z 19 380-399 38 38 38 60 60 60 F F F
45282024-03-06T20:59:52.640Z 20 400-419 34 34 34 53 53 53 F F F
45292024-03-06T20:59:52.640Z 21 420-439 40 40 40 65 65 65 F F F
45302024-03-06T20:59:52.640Z 22 440-459 41 41 41 66 66 66 F F F
45312024-03-06T20:59:52.640Z 23 460-479 39 39 39 62 62 62 F F F
45322024-03-06T20:59:52.641Z 24 480-499 39 39 39 62 62 62 F F F
45332024-03-06T20:59:52.641Z 25 500-519 40 40 40 65 65 65 F F F
45342024-03-06T20:59:52.641Z 26 520-539 36 36 36 56 56 56 F F F
45352024-03-06T20:59:52.641Z 27 540-559 37 37 37 59 59 59 F F F
45362024-03-06T20:59:52.641Z 28 560-579 40 40 40 65 65 65 F F F
45372024-03-06T20:59:52.641Z 29 580-599 39 39 39 62 62 62 F F F
45382024-03-06T20:59:52.641ZMax gen: 41, Max flush: 66
45392024-03-06T20:59:52.641Z
45402024-03-06T20:59:52.642ZBegin loop 21
45412024-03-06T20:59:52.642ZDownstairs to restart: 0
45422024-03-06T20:59:52.671Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.972979215Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307}
45432024-03-06T20:59:52.671Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.97320831Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307}
45442024-03-06T20:59:52.671Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.973226489Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307}
45452024-03-06T20:59:52.671Z{"msg":"Crucible 71c5884b-15ad-483c-a34a-b16dd05767f4 has session id: d69f8e29-b584-4bed-9896-aa8337165680","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.973249657Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45462024-03-06T20:59:52.672Z{"msg":"Upstairs opts: Upstairs UUID: 71c5884b-15ad-483c-a34a-b16dd05767f4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.973294954Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45472024-03-06T20:59:52.672ZCrucible runtime is spawned
45482024-03-06T20:59:52.672Z{"msg":"Crucible stats registered with UUID: 71c5884b-15ad-483c-a34a-b16dd05767f4","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.97334616Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45492024-03-06T20:59:52.672Z{"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.973407136Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307}
45502024-03-06T20:59:52.673Z{"msg":"71c5884b-15ad-483c-a34a-b16dd05767f4 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.973444574Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45512024-03-06T20:59:52.673Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.973465992Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"client":"0","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45522024-03-06T20:59:52.673Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.973504989Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"client":"0","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45532024-03-06T20:59:52.673Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.973528608Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"client":"1","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45542024-03-06T20:59:52.674Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.973557086Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"client":"1","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45552024-03-06T20:59:52.674Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.973573234Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"client":"2","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45562024-03-06T20:59:52.674Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.973595893Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"client":"2","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45572024-03-06T20:59:52.674Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.973615242Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"":"io task","client":"1","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45582024-03-06T20:59:52.674Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.97364525Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"":"io task","client":"0","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45592024-03-06T20:59:52.674Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.973671558Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"":"io task","client":"2","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45602024-03-06T20:59:52.675Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.973822777Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"":"io task","client":"1","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45612024-03-06T20:59:52.675Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.973880264Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"":"io task","client":"2","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45622024-03-06T20:59:52.675Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.973906232Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"":"io task","client":"0","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45632024-03-06T20:59:52.675Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.974491391Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"client":"1","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45642024-03-06T20:59:52.675Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.974539468Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"client":"1","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45652024-03-06T20:59:52.676Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.974562966Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"client":"2","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45662024-03-06T20:59:52.676Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.974588285Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"client":"2","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45672024-03-06T20:59:52.676Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.975136597Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"client":"1","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45682024-03-06T20:59:52.676Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.975296806Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"client":"2","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45692024-03-06T20:59:52.677Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.975434136Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"client":"0","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45702024-03-06T20:59:52.677Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.975478703Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"client":"0","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45712024-03-06T20:59:52.677Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.975640762Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"client":"1","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45722024-03-06T20:59:52.677Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.975696358Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"client":"1","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45732024-03-06T20:59:52.678Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.975730096Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45742024-03-06T20:59:52.678Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.975779002Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"client":"2","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45752024-03-06T20:59:52.678Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.975795951Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"client":"2","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45762024-03-06T20:59:52.678Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.975810311Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45772024-03-06T20:59:52.678Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.976007187Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"client":"0","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45782024-03-06T20:59:52.678Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.976378881Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"client":"0","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45792024-03-06T20:59:52.679Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.976422648Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"client":"0","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45802024-03-06T20:59:52.679Z{"msg":"[0]R flush_numbers[0..12]: [66, 66, 62, 62, 64, 50, 58, 61, 66, 53, 61, 64]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.976441737Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"":"downstairs","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45812024-03-06T20:59:52.679Z{"msg":"[0]R generation[0..12]: [41, 41, 39, 39, 40, 33, 36, 38, 41, 34, 38, 40]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.976457026Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"":"downstairs","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45822024-03-06T20:59:52.679Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.976478844Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"":"downstairs","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45832024-03-06T20:59:52.679Z{"msg":"[1]R flush_numbers[0..12]: [66, 66, 62, 62, 64, 50, 58, 61, 66, 53, 61, 64]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.976493093Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"":"downstairs","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45842024-03-06T20:59:52.679Z{"msg":"[1]R generation[0..12]: [41, 41, 39, 39, 40, 33, 36, 38, 41, 34, 38, 40]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.976513672Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"":"downstairs","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45852024-03-06T20:59:52.680Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.976528361Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"":"downstairs","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45862024-03-06T20:59:52.680ZWait for a query_work_queue command to finish before sending IO
45872024-03-06T20:59:52.680Z{"msg":"[2]R flush_numbers[0..12]: [66, 66, 62, 62, 64, 50, 58, 61, 66, 53, 61, 64]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.976550489Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"":"downstairs","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45882024-03-06T20:59:52.680Z{"msg":"[2]R generation[0..12]: [41, 41, 39, 39, 40, 33, 36, 38, 41, 34, 38, 40]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.976565008Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"":"downstairs","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45892024-03-06T20:59:52.680Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.976585537Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"":"downstairs","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45902024-03-06T20:59:52.680Z{"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.976599096Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"":"downstairs","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45912024-03-06T20:59:52.681Z{"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.976618685Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"":"downstairs","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45922024-03-06T20:59:52.681Z{"msg":"Next flush: 67","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.976631904Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"":"downstairs","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45932024-03-06T20:59:52.681Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.976651993Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"":"downstairs","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45942024-03-06T20:59:52.681Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.976665192Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45952024-03-06T20:59:52.681Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.976677061Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"client":"0","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45962024-03-06T20:59:52.681Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.976696619Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"client":"1","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45972024-03-06T20:59:52.682Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.976710258Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"client":"2","":"downstairs","session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45982024-03-06T20:59:52.682Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.976724017Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
45992024-03-06T20:59:52.682Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.976736137Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
46002024-03-06T20:59:52.682Z{"msg":"71c5884b-15ad-483c-a34a-b16dd05767f4 is now active with session: d69f8e29-b584-4bed-9896-aa8337165680","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.976748976Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
46012024-03-06T20:59:52.682Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.976767514Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307,"session_id":"d69f8e29-b584-4bed-9896-aa8337165680"}
46022024-03-06T20:59:52.682Z{"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:50.976779884Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307}
46032024-03-06T20:59:52.682ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
46042024-03-06T20:59:52.682ZLoading write count information from file "/tmp/test_repair_verify.data"
46052024-03-06T20:59:52.683ZRun Repair workload
46062024-03-06T20:59:52.683Z01/30 Write block 433 len 16384 data: 4 5 4 3
46072024-03-06T20:59:52.683Z02/30 Write block 449 len 28672 data: 4 5 6 6 5 4 5
46082024-03-06T20:59:52.683Z03/30 Write block 560 len 32768 data: 4 4 4 3 3 3 4 3
46092024-03-06T20:59:52.683Z04/30 Write block 354 len 12288 data: 4 3 5
46102024-03-06T20:59:52.683Z05/30 Write block 492 len 32768 data: 7 8 7 7 7 6 6 5
46112024-03-06T20:59:52.683Z06/30 Read block 331 len 16384
46122024-03-06T20:59:52.683Z07/30 Read block 319 len 16384
46132024-03-06T20:59:52.683Z08/30 Read block 250 len 16384
46142024-03-06T20:59:52.683Z09/30 Read block 71 len 8192
46152024-03-06T20:59:52.683Z10/30 Read block 71 len 8192
46162024-03-06T20:59:52.684Z11/30 Read block 231 len 16384
46172024-03-06T20:59:52.684Z12/30 Write block 200 len 28672 data: 2 2 2 2 2 2 2
46182024-03-06T20:59:52.684Z13/30 Write block 63 len 8192 data: 12 10
46192024-03-06T20:59:52.684Z14/30 Write block 416 len 32768 data: 5 4 4 4 4 7 5 4
46202024-03-06T20:59:52.684Z15/30 Write block 501 len 28672 data: 2 2 3 3 3 3 3
46212024-03-06T20:59:52.684Z16/30 Read block 391 len 12288
46222024-03-06T20:59:52.684Z17/30 Read block 398 len 28672
46232024-03-06T20:59:52.684Z18/30 Write block 510 len 24576 data: 3 3 4 4 4 4
46242024-03-06T20:59:52.685Z19/30 Write block 382 len 28672 data: 2 2 5 5 6 4 4
46252024-03-06T20:59:52.685Z20/30 Read block 86 len 16384
46262024-03-06T20:59:52.685Z21/30 Read block 295 len 28672
46272024-03-06T20:59:52.685Z22/30 Read block 382 len 12288
46282024-03-06T20:59:52.685Z23/30 Read block 365 len 4096
46292024-03-06T20:59:52.685Z24/30 Write block 45 len 12288 data: 3 3 5
46302024-03-06T20:59:52.685Z25/30 Flush
46312024-03-06T20:59:52.685Z26/30 Flush
46322024-03-06T20:59:52.685Z27/30 Write block 52 len 4096 data: 5
46332024-03-06T20:59:52.685Z28/30 Write block 463 len 28672 data: 7 6 6 6 5 5 5
46342024-03-06T20:59:52.686Z29/30 Read block 234 len 12288
46352024-03-06T20:59:52.686Z30/30 Write block 205 len 32768 data: 3 3 2 3 3 4 4 4
46362024-03-06T20:59:52.686Z----------------------------------------------------------------
46372024-03-06T20:59:52.686Z Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30
46382024-03-06T20:59:52.686ZStates: Active Active Active
46392024-03-06T20:59:52.686ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
46402024-03-06T20:59:52.686Z 1 Acked 1000 Write 4 Done Done Done false
46412024-03-06T20:59:52.686Z 2 Acked 1001 Write 7 Sent Done Done false
46422024-03-06T20:59:53.955Z 3 Acked 1002 Write 8 Sent Done Done false
46432024-03-06T20:59:53.955Z 4 Acked 1003 Write 3 Sent Done Done false
46442024-03-06T20:59:53.956Z 5 Acked 1004 Write 8 Sent Done Done false
46452024-03-06T20:59:53.956Z 6 Acked 1005 Read 4 Sent Done Done false
46462024-03-06T20:59:53.956Z 7 Acked 1006 Read 4 Sent Done Done false
46472024-03-06T20:59:53.956Z 8 Acked 1007 Read 4 Sent Done Done false
46482024-03-06T20:59:53.956Z 9 Acked 1008 Read 2 Sent Done Done false
46492024-03-06T20:59:53.956Z 10 Acked 1009 Read 2 Sent Done Done false
46502024-03-06T20:59:53.956Z 11 Acked 1010 Read 4 Sent Done Done false
46512024-03-06T20:59:53.956Z 12 Acked 1011 Write 7 Sent Done Done false
46522024-03-06T20:59:53.956Z 13 Acked 1012 Write 2 Sent Done Done false
46532024-03-06T20:59:53.956Z 14 Acked 1013 Write 8 Sent Done Done false
46542024-03-06T20:59:53.957Z 15 Acked 1014 Write 7 Sent Done Done false
46552024-03-06T20:59:53.957Z 16 Acked 1015 Read 3 Sent Done Done false
46562024-03-06T20:59:53.957Z 17 Acked 1016 Read 7 Sent Done Done false
46572024-03-06T20:59:53.957Z 18 Acked 1017 Write 6 Sent Done Done false
46582024-03-06T20:59:53.957Z 19 Acked 1018 Write 7 Sent Done Done false
46592024-03-06T20:59:53.957Z 20 Acked 1019 Read 4 Sent Done Done false
46602024-03-06T20:59:53.957Z 21 Acked 1020 Read 7 Sent Done Done false
46612024-03-06T20:59:53.957Z 22 Acked 1021 Read 3 Sent Done Done false
46622024-03-06T20:59:53.957Z 23 Acked 1022 Read 1 Sent Done Done false
46632024-03-06T20:59:53.957Z 24 Acked 1023 Write 3 Sent Done Done false
46642024-03-06T20:59:53.957Z 25 Acked 1024 Flush 0 Sent Done Done false
46652024-03-06T20:59:53.957Z 26 Acked 1025 Flush 0 Sent Done Done false
46662024-03-06T20:59:53.957Z 27 Acked 1026 Write 1 Sent Done Done false
46672024-03-06T20:59:53.958Z 28 Acked 1027 Write 7 Sent Done Done false
46682024-03-06T20:59:53.958Z 29 Acked 1028 Read 3 Sent Done Done false
46692024-03-06T20:59:53.958Z 30 Acked 1029 Write 8 Sent Sent Sent false
46702024-03-06T20:59:53.958Z STATES DS:0 DS:1 DS:2 TOTAL
46712024-03-06T20:59:53.958Z New 0 0 0 0
46722024-03-06T20:59:53.958Z Sent 29 1 1 31
46732024-03-06T20:59:53.958Z Done 1 29 29 59
46742024-03-06T20:59:53.958Z Skipped 0 0 0 0
46752024-03-06T20:59:53.958Z Error 0 0 0 0
46762024-03-06T20:59:53.958ZLast Flush: 0 1025 1025
46772024-03-06T20:59:53.958ZDownstairs last five completed:
46782024-03-06T20:59:53.958ZUpstairs last five completed: 30 29 28 27 26
46792024-03-06T20:59:53.958Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T20:59:51.008632129Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1307}
46802024-03-06T20:59:53.958ZWrote out file "/tmp/test_repair_verify.data"
46812024-03-06T20:59:53.958Z
46822024-03-06T20:59:53.959ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
46832024-03-06T20:59:53.959ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
46842024-03-06T20:59:53.959Z 0 000-019 41 41 41 66 66 66 F F F
46852024-03-06T20:59:53.959Z 1 020-039 41 41 41 66 66 66 F F F
46862024-03-06T20:59:53.959Z 2 040-059 39 42 42 62 67 67 F T T <---
46872024-03-06T20:59:53.959Z 3 060-079 39 42 42 62 67 67 F F F <---
46882024-03-06T20:59:53.959Z 4 080-099 40 40 40 64 64 64 F F F
46892024-03-06T20:59:53.959Z 5 100-119 33 33 33 50 50 50 F F F
46902024-03-06T20:59:53.959Z 6 120-139 36 36 36 58 58 58 F F F
46912024-03-06T20:59:53.959Z 7 140-159 38 38 38 61 61 61 F F F
46922024-03-06T20:59:53.959Z 8 160-179 41 41 41 66 66 66 F F F
46932024-03-06T20:59:53.959Z 9 180-199 34 34 34 53 53 53 F F F
46942024-03-06T20:59:53.959Z 10 200-219 38 42 42 61 67 67 F T F <---
46952024-03-06T20:59:53.959Z 11 220-239 40 40 40 64 64 64 F F F
46962024-03-06T20:59:53.959Z 12 240-259 40 40 40 64 64 64 F F F
46972024-03-06T20:59:53.960Z 13 260-279 40 40 40 65 65 65 F F F
46982024-03-06T20:59:53.960Z 14 280-299 40 40 40 65 65 65 F F F
46992024-03-06T20:59:53.960Z 15 300-319 40 40 40 65 65 65 F F F
47002024-03-06T20:59:53.960Z 16 320-339 32 32 32 47 47 47 F F F
47012024-03-06T20:59:53.960Z 17 340-359 34 42 42 51 67 67 F F F <---
47022024-03-06T20:59:53.960Z 18 360-379 40 40 40 65 65 65 F F F
47032024-03-06T20:59:53.960Z 19 380-399 38 42 42 60 67 67 F F F <---
47042024-03-06T20:59:53.960Z 20 400-419 34 42 42 53 67 67 F F F <---
47052024-03-06T20:59:53.960Z 21 420-439 40 42 42 65 67 67 T F F <---
47062024-03-06T20:59:53.960Z 22 440-459 41 42 42 66 67 67 F F F <---
47072024-03-06T20:59:53.960Z 23 460-479 39 39 39 62 62 62 F T T <---
47082024-03-06T20:59:53.960Z 24 480-499 39 42 42 62 67 67 F F F <---
47092024-03-06T20:59:53.960Z 25 500-519 40 42 42 65 67 67 F F F <---
47102024-03-06T20:59:53.960Z 26 520-539 36 36 36 56 56 56 F F F
47112024-03-06T20:59:53.961Z 27 540-559 37 37 37 59 59 59 F F F
47122024-03-06T20:59:53.961Z 28 560-579 40 42 42 65 67 67 F F F <---
47132024-03-06T20:59:53.961Z 29 580-599 39 39 39 62 62 62 F F F
47142024-03-06T20:59:53.961ZMax gen: 42, Max flush: 67
47152024-03-06T20:59:53.961ZError: Difference in extent metadata found!
47162024-03-06T20:59:53.961ZA Difference in extent metadata is expected here
47172024-03-06T20:59:53.961ZOn loop 21
47182024-03-06T20:59:53.961Z
47192024-03-06T20:59:53.961ZVerifying data now
47202024-03-06T20:59:56.891ZNo change after verify
47212024-03-06T20:59:56.891ZLoop: 21 Downstairs dump after verify (and repair):
47222024-03-06T20:59:56.901ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
47232024-03-06T20:59:56.901Z 0 000-019 41 41 41 66 66 66 F F F
47242024-03-06T20:59:56.901Z 1 020-039 41 41 41 66 66 66 F F F
47252024-03-06T20:59:56.901Z 2 040-059 43 43 43 68 68 68 F F F
47262024-03-06T20:59:56.901Z 3 060-079 42 42 42 67 67 67 F F F
47272024-03-06T20:59:56.901Z 4 080-099 40 40 40 64 64 64 F F F
47282024-03-06T20:59:56.902Z 5 100-119 33 33 33 50 50 50 F F F
47292024-03-06T20:59:56.902Z 6 120-139 36 36 36 58 58 58 F F F
47302024-03-06T20:59:56.902Z 7 140-159 38 38 38 61 61 61 F F F
47312024-03-06T20:59:56.902Z 8 160-179 41 41 41 66 66 66 F F F
47322024-03-06T20:59:56.902Z 9 180-199 34 34 34 53 53 53 F F F
47332024-03-06T20:59:56.902Z 10 200-219 43 43 43 68 68 68 F F F
47342024-03-06T20:59:56.902Z 11 220-239 40 40 40 64 64 64 F F F
47352024-03-06T20:59:56.903Z 12 240-259 40 40 40 64 64 64 F F F
47362024-03-06T20:59:56.910Z 13 260-279 40 40 40 65 65 65 F F F
47372024-03-06T20:59:56.910Z 14 280-299 40 40 40 65 65 65 F F F
47382024-03-06T20:59:56.911Z 15 300-319 40 40 40 65 65 65 F F F
47392024-03-06T20:59:56.911Z 16 320-339 32 32 32 47 47 47 F F F
47402024-03-06T20:59:56.911Z 17 340-359 42 42 42 67 67 67 F F F
47412024-03-06T20:59:56.911Z 18 360-379 40 40 40 65 65 65 F F F
47422024-03-06T20:59:56.911Z 19 380-399 42 42 42 67 67 67 F F F
47432024-03-06T20:59:56.911Z 20 400-419 42 42 42 67 67 67 F F F
47442024-03-06T20:59:56.911Z 21 420-439 42 42 42 67 67 67 F F F
47452024-03-06T20:59:56.911Z 22 440-459 42 42 42 67 67 67 F F F
47462024-03-06T20:59:56.912Z 23 460-479 43 43 43 68 68 68 F F F
47472024-03-06T20:59:56.912Z 24 480-499 42 42 42 67 67 67 F F F
47482024-03-06T20:59:56.912Z 25 500-519 42 42 42 67 67 67 F F F
47492024-03-06T20:59:56.912Z 26 520-539 36 36 36 56 56 56 F F F
47502024-03-06T20:59:56.912Z 27 540-559 37 37 37 59 59 59 F F F
47512024-03-06T20:59:56.912Z 28 560-579 42 42 42 67 67 67 F F F
47522024-03-06T20:59:56.912Z 29 580-599 39 39 39 62 62 62 F F F
47532024-03-06T20:59:56.912ZMax gen: 43, Max flush: 68
47542024-03-06T20:59:56.913Z
47552024-03-06T20:59:56.913ZBegin loop 22
47562024-03-06T20:59:56.913ZDownstairs to restart: 1
47572024-03-06T20:59:57.351Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.694125178Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315}
47582024-03-06T20:59:57.352Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.694434287Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315}
47592024-03-06T20:59:57.352Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.694454095Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315}
47602024-03-06T20:59:57.352Z{"msg":"Crucible 837cc9f5-e00f-47d9-84c0-0052632e2704 has session id: f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.694477144Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47612024-03-06T20:59:57.352Z{"msg":"Upstairs opts: Upstairs UUID: 837cc9f5-e00f-47d9-84c0-0052632e2704, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.694521881Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47622024-03-06T20:59:57.814Z{"msg":"Crucible stats registered with UUID: 837cc9f5-e00f-47d9-84c0-0052632e2704","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.694546119Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47632024-03-06T20:59:57.815ZCrucible runtime is spawned
47642024-03-06T20:59:57.815Z{"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.694688319Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315}
47652024-03-06T20:59:57.816Z{"msg":"837cc9f5-e00f-47d9-84c0-0052632e2704 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.694802032Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47662024-03-06T20:59:57.816Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.694848238Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"client":"0","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47672024-03-06T20:59:57.816Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.694876506Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"client":"0","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47682024-03-06T20:59:57.816Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.694902585Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"client":"1","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47692024-03-06T20:59:57.816Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.694926583Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"client":"1","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47702024-03-06T20:59:57.816Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.69496532Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"client":"2","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47712024-03-06T20:59:57.816Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.694989779Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"client":"2","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47722024-03-06T20:59:57.816Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.695010807Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"":"io task","client":"0","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47732024-03-06T20:59:57.816Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.695035975Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"":"io task","client":"1","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47742024-03-06T20:59:57.816Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.695060703Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"":"io task","client":"2","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47752024-03-06T20:59:57.816Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.69526374Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"":"io task","client":"2","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47762024-03-06T20:59:57.817Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.695315246Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"":"io task","client":"1","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47772024-03-06T20:59:57.817Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.695348324Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"":"io task","client":"0","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47782024-03-06T20:59:57.817Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.696071974Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"client":"0","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47792024-03-06T20:59:57.817Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.696113301Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"client":"0","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47802024-03-06T20:59:57.817Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.696138959Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"client":"2","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47812024-03-06T20:59:57.817Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.696164358Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"client":"2","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47822024-03-06T20:59:57.817Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.696754747Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"client":"0","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47832024-03-06T20:59:57.817Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.69686137Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"client":"2","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47842024-03-06T20:59:57.818Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.697347286Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"client":"2","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47852024-03-06T20:59:57.818Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.697391703Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"client":"2","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47862024-03-06T20:59:57.818Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.697408122Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47872024-03-06T20:59:57.818Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.697425141Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"client":"1","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47882024-03-06T20:59:57.818Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.697459188Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"client":"1","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47892024-03-06T20:59:57.818Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.697487016Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"client":"0","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47902024-03-06T20:59:57.819Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.697512555Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"client":"0","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47912024-03-06T20:59:57.819Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.697526864Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47922024-03-06T20:59:57.819Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.69801272Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"client":"1","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47932024-03-06T20:59:57.819Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.69844771Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"client":"1","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47942024-03-06T20:59:57.819Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.698492267Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"client":"1","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47952024-03-06T20:59:57.820Z{"msg":"[0]R flush_numbers[0..12]: [66, 66, 68, 67, 64, 50, 58, 61, 66, 53, 68, 64]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.698511476Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"":"downstairs","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47962024-03-06T20:59:57.820Z{"msg":"[0]R generation[0..12]: [41, 41, 43, 42, 40, 33, 36, 38, 41, 34, 43, 40]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.698533994Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"":"downstairs","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47972024-03-06T20:59:57.820Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.698548723Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"":"downstairs","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47982024-03-06T20:59:57.820Z{"msg":"[1]R flush_numbers[0..12]: [66, 66, 68, 67, 64, 50, 58, 61, 66, 53, 68, 64]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.698569322Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"":"downstairs","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
47992024-03-06T20:59:57.820Z{"msg":"[1]R generation[0..12]: [41, 41, 43, 42, 40, 33, 36, 38, 41, 34, 43, 40]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.698583101Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"":"downstairs","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
48002024-03-06T20:59:57.820Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.698604339Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"":"downstairs","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
48012024-03-06T20:59:57.820Z{"msg":"[2]R flush_numbers[0..12]: [66, 66, 68, 67, 64, 50, 58, 61, 66, 53, 68, 64]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.698625448Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"":"downstairs","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
48022024-03-06T20:59:57.821Z{"msg":"[2]R generation[0..12]: [41, 41, 43, 42, 40, 33, 36, 38, 41, 34, 43, 40]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
48032024-03-06T20:59:57.821Z,"time":"2024-03-06T20:59:55.698639837Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"":"downstairs","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
48042024-03-06T20:59:57.821Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.698662735Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"":"downstairs","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
48052024-03-06T20:59:57.821Z{"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.698677894Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"":"downstairs","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
48062024-03-06T20:59:57.821Z{"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.698696673Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"":"downstairs","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
48072024-03-06T20:59:57.821Z{"msg":"Next flush: 69","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.698715072Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"":"downstairs","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
48082024-03-06T20:59:57.821Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.698729581Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"":"downstairs","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
48092024-03-06T20:59:57.821Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.698748889Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
48102024-03-06T20:59:57.822Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.698771118Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"client":"0","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
48112024-03-06T20:59:57.822Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.698818245Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"client":"1","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
48122024-03-06T20:59:57.822Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.698844643Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"client":"2","":"downstairs","session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
48132024-03-06T20:59:57.822Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.698874411Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
48142024-03-06T20:59:57.822Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.698894359Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
48152024-03-06T20:59:57.822Z{"msg":"837cc9f5-e00f-47d9-84c0-0052632e2704 is now active with session: f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.698923697Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
48162024-03-06T20:59:57.822Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.698940136Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315,"session_id":"f15e34b7-9a55-4c6d-8cae-db2eb8f2e4e7"}
48172024-03-06T20:59:57.823Z{"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-03-06T20:59:55.698971134Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315}
48182024-03-06T20:59:57.823ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
48192024-03-06T20:59:57.823ZLoading write count information from file "/tmp/test_repair_verify.data"
48202024-03-06T20:59:57.823ZRun Repair workload
48212024-03-06T20:59:57.823Z01/30 Write block 423 len 40960 data: 5 4 4 5 5 5 5 6 4 4
48222024-03-06T20:59:57.824Z02/30 Read block 95 len 20480
48232024-03-06T20:59:57.824Z03/30 Flush
48242024-03-06T20:59:57.824Z04/30 Write block 102 len 36864 data: 6 5 5 4 6 7 4 4 4
48252024-03-06T20:59:57.824Z05/30 Flush
48262024-03-06T20:59:57.824Z06/30 Write block 176 len 8192 data: 6 6
48272024-03-06T20:59:57.824Z07/30 Read block 135 len 32768
48282024-03-06T20:59:57.824Z08/30 Read block 284 len 24576
48292024-03-06T20:59:57.824Z09/30 Read block 65 len 32768
48302024-03-06T20:59:57.824Z10/30 Read block 108 len 12288
48312024-03-06T20:59:57.824Z11/30 Write block 433 len 24576 data: 5 6 5 4 4 4
48322024-03-06T20:59:57.825Z12/30 Write block 127 len 36864 data: 4 5 5 5 4 5 3 2 2
48332024-03-06T20:59:57.825Z13/30 Read block 39 len 4096
48342024-03-06T20:59:57.825Z14/30 Read block 323 len 24576
48352024-03-06T20:59:57.825Z15/30 Read block 545 len 20480
48362024-03-06T20:59:57.825Z16/30 Read block 263 len 28672
48372024-03-06T20:59:57.826Z17/30 Read block 524 len 28672
48382024-03-06T20:59:57.826Z18/30 Write block 562 len 24576 data: 5 4 4 4 5 4
48392024-03-06T20:59:57.826Z19/30 Read block 481 len 24576
48402024-03-06T20:59:57.826Z20/30 Read block 268 len 16384
48412024-03-06T20:59:57.826Z21/30 Read block 219 len 16384
48422024-03-06T20:59:57.826Z22/30 Write block 293 len 20480 data: 4 4 6 5 4
48432024-03-06T20:59:57.827Z23/30 Write block 271 len 40960 data: 4 4 4 6 4 4 3 3 3 2
48442024-03-06T20:59:57.827Z24/30 Read block 88 len 4096
48452024-03-06T20:59:57.827Z25/30 Write block 379 len 4096 data: 4
48462024-03-06T20:59:57.827Z26/30 Write block 448 len 20480 data: 5 5 6 7 7
48472024-03-06T20:59:57.827Z27/30 Write block 561 len 16384 data: 5 6 5 5
48482024-03-06T20:59:57.828Z28/30 Read block 343 len 36864
48492024-03-06T20:59:57.828Z29/30 Read block 46 len 16384
48502024-03-06T20:59:57.828Z30/30 Read block 537 len 8192
48512024-03-06T20:59:57.828Z----------------------------------------------------------------
48522024-03-06T20:59:57.828Z Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:30
48532024-03-06T20:59:57.828ZStates: Active Active Active
48542024-03-06T20:59:57.828ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
48552024-03-06T20:59:57.828Z 1 Acked 1000 Write 10 Done Done Done false
48562024-03-06T20:59:57.829Z 2 Acked 1001 Read 5 Done Done Done false
48572024-03-06T20:59:57.829Z 3 Acked 1002 Flush 0 Done Sent Done false
48582024-03-06T20:59:57.829Z 4 Acked 1003 Write 9 Done Sent Done false
48592024-03-06T20:59:57.829Z 5 Acked 1004 Flush 0 Done Sent Done false
48602024-03-06T20:59:57.829Z 6 Acked 1005 Write 2 Done Sent Done false
48612024-03-06T20:59:57.829Z 7 Acked 1006 Read 8 Done Sent Done false
48622024-03-06T20:59:57.856Z 8 Acked 1007 Read 6 Done Sent Done false
48632024-03-06T20:59:57.856Z 9 Acked 1008 Read 8 Done Sent Done false
48642024-03-06T20:59:57.856Z 10 Acked 1009 Read 3 Done Sent Done false
48652024-03-06T20:59:57.856Z 11 Acked 1010 Write 6 Done Sent Done false
48662024-03-06T20:59:57.856Z 12 Acked 1011 Write 9 Done Sent Done false
48672024-03-06T20:59:57.856Z 13 Acked 1012 Read 1 Done Sent Done false
48682024-03-06T20:59:57.857Z 14 Acked 1013 Read 6 Done Sent Done false
48692024-03-06T20:59:57.857Z 15 Acked 1014 Read 5 Done Sent Done false
48702024-03-06T20:59:57.857Z 16 Acked 1015 Read 7 Done Sent Done false
48712024-03-06T20:59:57.857Z 17 Acked 1016 Read 7 Done Sent Done false
48722024-03-06T20:59:57.857Z 18 Acked 1017 Write 6 Done Sent Done false
48732024-03-06T20:59:57.857Z 19 Acked 1018 Read 6 Done Sent Done false
48742024-03-06T20:59:57.857Z 20 Acked 1019 Read 4 Done Sent Done false
48752024-03-06T20:59:57.857Z 21 Acked 1020 Read 4 Done Sent Done false
48762024-03-06T20:59:57.857Z 22 Acked 1021 Write 5 Done Sent Done false
48772024-03-06T20:59:57.858Z 23 Acked 1022 Write 10 Done Sent Done false
48782024-03-06T20:59:57.858Z 24 Acked 1023 Read 1 Done Sent Done false
48792024-03-06T20:59:57.858Z 25 Acked 1024 Write 1 Done Sent Done false
48802024-03-06T20:59:57.858Z 26 Acked 1025 Write 5 Done Sent Done false
48812024-03-06T20:59:57.858Z 27 Acked 1026 Write 4 Done Sent Done false
48822024-03-06T20:59:57.858Z 28 Acked 1027 Read 9 Done Sent Done false
48832024-03-06T20:59:57.859Z 29 Acked 1028 Read 4 Done Sent Done false
48842024-03-06T20:59:57.859Z 30 Acked 1029 Read 2 Done Sent Done false
48852024-03-06T20:59:57.859Z STATES DS:0 DS:1 DS:2 TOTAL
48862024-03-06T20:59:57.859Z New 0 0 0 0
48872024-03-06T20:59:57.859Z Sent 0 28 0 28
48882024-03-06T20:59:57.859Z Done 30 2 30 62
48892024-03-06T20:59:57.859Z Skipped 0 0 0 0
48902024-03-06T20:59:57.859Z Error 0 0 0 0
48912024-03-06T20:59:57.859ZLast Flush: 1004 0 1004
48922024-03-06T20:59:57.859ZDownstairs last five completed:
48932024-03-06T20:59:57.860ZUpstairs last five completed: 30 29 28 27 26
48942024-03-06T20:59:57.860Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T20:59:55.733258441Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1315}
48952024-03-06T20:59:57.860ZWrote out file "/tmp/test_repair_verify.data"
48962024-03-06T20:59:57.860Z
48972024-03-06T20:59:57.861ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
48982024-03-06T20:59:57.861ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
48992024-03-06T20:59:57.861Z 0 000-019 41 41 41 66 66 66 F F F
49002024-03-06T20:59:57.861Z 1 020-039 41 41 41 66 66 66 F F F
49012024-03-06T20:59:57.861Z 2 040-059 43 43 43 68 68 68 F F F
49022024-03-06T20:59:57.861Z 3 060-079 42 42 42 67 67 67 F F F
49032024-03-06T20:59:57.861Z 4 080-099 40 40 40 64 64 64 F F F
49042024-03-06T20:59:57.861Z 5 100-119 44 33 44 70 50 70 F F F <---
49052024-03-06T20:59:57.861Z 6 120-139 36 36 36 58 58 58 T F T <---
49062024-03-06T20:59:57.861Z 7 140-159 38 38 38 61 61 61 F F F
49072024-03-06T20:59:57.861Z 8 160-179 41 41 41 66 66 66 T F T <---
49082024-03-06T20:59:57.861Z 9 180-199 34 34 34 53 53 53 F F F
49092024-03-06T20:59:57.862Z 10 200-219 43 43 43 68 68 68 F F F
49102024-03-06T20:59:57.862Z 11 220-239 40 40 40 64 64 64 F F F
49112024-03-06T20:59:57.862Z 12 240-259 40 40 40 64 64 64 F F F
49122024-03-06T20:59:57.862Z 13 260-279 40 40 40 65 65 65 T F T <---
49132024-03-06T20:59:57.862Z 14 280-299 40 40 40 65 65 65 T F T <---
49142024-03-06T20:59:57.862Z 15 300-319 40 40 40 65 65 65 F F F
49152024-03-06T20:59:57.862Z 16 320-339 32 32 32 47 47 47 F F F
49162024-03-06T20:59:57.862Z 17 340-359 42 42 42 67 67 67 F F F
49172024-03-06T20:59:57.862Z 18 360-379 40 40 40 65 65 65 T F T <---
49182024-03-06T20:59:57.862Z 19 380-399 42 42 42 67 67 67 F F F
49192024-03-06T20:59:57.863Z 20 400-419 42 42 42 67 67 67 F F F
49202024-03-06T20:59:57.863Z 21 420-439 44 42 44 69 67 69 T T T <---
49212024-03-06T20:59:57.863Z 22 440-459 42 42 42 67 67 67 T F T <---
49222024-03-06T20:59:57.863Z 23 460-479 43 43 43 68 68 68 F F F
49232024-03-06T20:59:57.863Z 24 480-499 42 42 42 67 67 67 F F F
49242024-03-06T20:59:57.863Z 25 500-519 42 42 42 67 67 67 F F F
49252024-03-06T20:59:57.863Z 26 520-539 36 36 36 56 56 56 F F F
49262024-03-06T20:59:57.863Z 27 540-559 37 37 37 59 59 59 F F F
49272024-03-06T20:59:57.863Z 28 560-579 42 42 42 67 67 67 T F T <---
49282024-03-06T20:59:57.863Z 29 580-599 39 39 39 62 62 62 F F F
49292024-03-06T20:59:57.863ZMax gen: 44, Max flush: 70
49302024-03-06T20:59:57.863ZError: Difference in extent metadata found!
49312024-03-06T20:59:57.863ZA Difference in extent metadata is expected here
49322024-03-06T20:59:57.864ZOn loop 22
49332024-03-06T20:59:57.864Z
49342024-03-06T20:59:57.864ZVerifying data now
49352024-03-06T21:00:01.518ZNo change after verify
49362024-03-06T21:00:01.518ZLoop: 22 Downstairs dump after verify (and repair):
49372024-03-06T21:00:01.529ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
49382024-03-06T21:00:01.530Z 0 000-019 41 41 41 66 66 66 F F F
49392024-03-06T21:00:01.530Z 1 020-039 41 41 41 66 66 66 F F F
49402024-03-06T21:00:01.530Z 2 040-059 43 43 43 68 68 68 F F F
49412024-03-06T21:00:01.530Z 3 060-079 42 42 42 67 67 67 F F F
49422024-03-06T21:00:01.530Z 4 080-099 40 40 40 64 64 64 F F F
49432024-03-06T21:00:01.530Z 5 100-119 44 44 44 70 70 70 F F F
49442024-03-06T21:00:01.530Z 6 120-139 45 45 45 71 71 71 F F F
49452024-03-06T21:00:01.530Z 7 140-159 38 38 38 61 61 61 F F F
49462024-03-06T21:00:01.531Z 8 160-179 45 45 45 71 71 71 F F F
49472024-03-06T21:00:01.540Z 9 180-199 34 34 34 53 53 53 F F F
49482024-03-06T21:00:01.540Z 10 200-219 43 43 43 68 68 68 F F F
49492024-03-06T21:00:01.541Z 11 220-239 40 40 40 64 64 64 F F F
49502024-03-06T21:00:01.541Z 12 240-259 40 40 40 64 64 64 F F F
49512024-03-06T21:00:01.541Z 13 260-279 45 45 45 71 71 71 F F F
49522024-03-06T21:00:01.541Z 14 280-299 45 45 45 71 71 71 F F F
49532024-03-06T21:00:01.541Z 15 300-319 40 40 40 65 65 65 F F F
49542024-03-06T21:00:01.541Z 16 320-339 32 32 32 47 47 47 F F F
49552024-03-06T21:00:01.541Z 17 340-359 42 42 42 67 67 67 F F F
49562024-03-06T21:00:01.542Z 18 360-379 45 45 45 71 71 71 F F F
49572024-03-06T21:00:01.542Z 19 380-399 42 42 42 67 67 67 F F F
49582024-03-06T21:00:01.542Z 20 400-419 42 42 42 67 67 67 F F F
49592024-03-06T21:00:01.542Z 21 420-439 45 45 45 71 71 71 F F F
49602024-03-06T21:00:01.542Z 22 440-459 45 45 45 71 71 71 F F F
49612024-03-06T21:00:01.542Z 23 460-479 43 43 43 68 68 68 F F F
49622024-03-06T21:00:01.543Z 24 480-499 42 42 42 67 67 67 F F F
49632024-03-06T21:00:01.543Z 25 500-519 42 42 42 67 67 67 F F F
49642024-03-06T21:00:01.543Z 26 520-539 36 36 36 56 56 56 F F F
49652024-03-06T21:00:01.543Z 27 540-559 37 37 37 59 59 59 F F F
49662024-03-06T21:00:01.543Z 28 560-579 45 45 45 71 71 71 F F F
49672024-03-06T21:00:01.543Z 29 580-599 39 39 39 62 62 62 F F F
49682024-03-06T21:00:01.543ZMax gen: 45, Max flush: 71
49692024-03-06T21:00:01.544Z
49702024-03-06T21:00:01.544ZBegin loop 23
49712024-03-06T21:00:01.544ZDownstairs to restart: 2
49722024-03-06T21:00:01.558Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.31705211Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323}
49732024-03-06T21:00:01.558Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.317373768Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323}
49742024-03-06T21:00:01.558Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":Crucible runtime is spawned
49752024-03-06T21:00:01.583Z"crucible","level":30,"time":"2024-03-06T21:00:00.317433244Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323}
49762024-03-06T21:00:01.584Z{"msg":"Crucible d5e536d8-eafb-4cde-b8c0-c215d3b28602 has session id: a9051de4-b067-4665-8d56-c4288ac9287c","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.31749133Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
49772024-03-06T21:00:01.584Z{"msg":"Upstairs opts: Upstairs UUID: d5e536d8-eafb-4cde-b8c0-c215d3b28602, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.317543836Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
49782024-03-06T21:00:01.584Z{"msg":"Crucible stats registered with UUID: d5e536d8-eafb-4cde-b8c0-c215d3b28602","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.317567755Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
49792024-03-06T21:00:01.584Z{"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.317595113Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323}
49802024-03-06T21:00:01.584Z{"msg":"d5e536d8-eafb-4cde-b8c0-c215d3b28602 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.317610412Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
49812024-03-06T21:00:01.585Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.317626151Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"client":"0","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
49822024-03-06T21:00:01.585Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.317653299Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"client":"0","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
49832024-03-06T21:00:01.585Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.317683387Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"client":"1","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
49842024-03-06T21:00:01.585Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.317717604Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"client":"1","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
49852024-03-06T21:00:01.585Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.317768811Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"client":"2","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
49862024-03-06T21:00:01.585Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.317813938Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"client":"2","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
49872024-03-06T21:00:01.586Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.317856885Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"":"io task","client":"0","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
49882024-03-06T21:00:01.586Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.317907831Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"":"io task","client":"1","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
49892024-03-06T21:00:01.586Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.317943369Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"":"io task","client":"2","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
49902024-03-06T21:00:01.586Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.318013734Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"":"io task","client":"2","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
49912024-03-06T21:00:01.586Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.31806865Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"":"io task","client":"1","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
49922024-03-06T21:00:01.586Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.318117987Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"":"io task","client":"0","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
49932024-03-06T21:00:01.587Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.318870965Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"client":"0","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
49942024-03-06T21:00:01.587Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.318916762Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"client":"0","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
49952024-03-06T21:00:01.587Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.318967898Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"client":"1","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
49962024-03-06T21:00:01.587Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.319002476Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"client":"1","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
49972024-03-06T21:00:01.587Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.319310675Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"client":"2","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
49982024-03-06T21:00:01.587Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.319363591Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"client":"2","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
49992024-03-06T21:00:01.587Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.319625863Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"client":"1","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50002024-03-06T21:00:01.588Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.319661301Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"client":"0","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50012024-03-06T21:00:01.588Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.319995247Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"client":"2","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50022024-03-06T21:00:01.588Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.320268069Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"client":"0","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50032024-03-06T21:00:01.588Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.320318115Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"client":"0","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50042024-03-06T21:00:01.588Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.320368732Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50052024-03-06T21:00:01.589Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.320401839Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"client":"1","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50062024-03-06T21:00:01.589Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.320421538Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"client":"1","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50072024-03-06T21:00:01.589Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.320443027Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50082024-03-06T21:00:01.589Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.320527861Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"client":"2","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50092024-03-06T21:00:01.589Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.320552559Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"client":"2","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50102024-03-06T21:00:01.589Z{"msg":"[0]R flush_numbers[0..12]: [66, 66, 68, 67, 64, 70, 71, 61, 71, 53, 68, 64]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.320566968Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"":"downstairs","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50112024-03-06T21:00:01.589Z{"msg":"[0]R generation[0..12]: [41, 41, 43, 42, 40, 44, 45, 38, 45, 34, 43, 40]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.320581447Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"":"downstairs","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50122024-03-06T21:00:01.590Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.320602536Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"":"downstairs","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50132024-03-06T21:00:01.590Z{"msg":"[1]R flush_numbers[0..12]: [66, 66, 68, 67, 64, 70, 71, 61, 71, 53, 68, 64]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.320616705Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"":"downstairs","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50142024-03-06T21:00:01.590Z{"msg":"[1]R generation[0..12]: [41, 41, 43, 42, 40, 44, 45, 38, 45, 34, 43, 40]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.320630424Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"":"downstairs","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50152024-03-06T21:00:01.590Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.320644633Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"":"downstairs","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50162024-03-06T21:00:01.590Z{"msg":"[2]R flush_numbers[0..12]: [66, 66, 68, 67, 64, 70, 71, 61, 71, 53, 68, 64]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.320658352Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"":"downstairs","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50172024-03-06T21:00:01.591Z{"msg":"[2]R generation[0..12]: [41, 41, 43, 42, 40, 44, 45, 38, 45, 34, 43, 40]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.320678571Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"":"downstairs","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50182024-03-06T21:00:01.591Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.320692819Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"":"downstairs","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50192024-03-06T21:00:01.591Z{"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.320705718Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"":"downstairs","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50202024-03-06T21:00:01.591Z{"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.320725337Z","hostname":"Wait for a query_work_queue command to finish before sending IO
50212024-03-06T21:00:01.591Zw-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"":"downstairs","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50222024-03-06T21:00:01.591Z{"msg":"Next flush: 72","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.320741526Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"":"downstairs","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50232024-03-06T21:00:01.592Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.320754485Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"":"downstairs","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50242024-03-06T21:00:01.592Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.320774264Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50252024-03-06T21:00:01.592Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.320786383Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"client":"0","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50262024-03-06T21:00:01.592Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.320799532Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"client":"1","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50272024-03-06T21:00:01.592Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.320812771Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"client":"2","":"downstairs","session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50282024-03-06T21:00:01.593Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.32082675Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50292024-03-06T21:00:01.593Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.320839019Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50302024-03-06T21:00:01.593Z{"msg":"d5e536d8-eafb-4cde-b8c0-c215d3b28602 is now active with session: a9051de4-b067-4665-8d56-c4288ac9287c","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.320856078Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50312024-03-06T21:00:01.593Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.320888536Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323,"session_id":"a9051de4-b067-4665-8d56-c4288ac9287c"}
50322024-03-06T21:00:01.593Z{"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:00.320921324Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323}
50332024-03-06T21:00:01.593ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
50342024-03-06T21:00:01.593ZLoading write count information from file "/tmp/test_repair_verify.data"
50352024-03-06T21:00:01.593ZRun Repair workload
50362024-03-06T21:00:01.593Z01/30 Write block 41 len 12288 data: 4 5 4
50372024-03-06T21:00:01.594Z02/30 Write block 586 len 40960 data: 5 6 5 5 6 5 6 3 4 4
50382024-03-06T21:00:01.594Z03/30 Read block 507 len 8192
50392024-03-06T21:00:01.594Z04/30 Read block 528 len 28672
50402024-03-06T21:00:01.594Z05/30 Read block 376 len 16384
50412024-03-06T21:00:01.594Z06/30 Write block 461 len 20480 data: 7 7 8 7 7
50422024-03-06T21:00:01.594Z07/30 Write block 15 len 24576 data: 4 3 2 2 3 4
50432024-03-06T21:00:01.594Z08/30 Write block 537 len 4096 data: 5
50442024-03-06T21:00:01.594Z09/30 Flush
50452024-03-06T21:00:01.613Z10/30 Write block 363 len 20480 data: 7 6 5 4 5
50462024-03-06T21:00:01.613Z11/30 Write block 306 len 40960 data: 7 7 7 6 6 6 6 5 4 4
50472024-03-06T21:00:01.613Z12/30 Write block 109 len 12288 data: 5 5 5
50482024-03-06T21:00:01.613Z13/30 Read block 156 len 24576
50492024-03-06T21:00:01.614Z14/30 Read block 236 len 28672
50502024-03-06T21:00:01.614Z15/30 Write block 546 len 40960 data: 6 6 5 5 4 4 3 2 3 3
50512024-03-06T21:00:01.614Z16/30 Write block 274 len 32768 data: 7 5 5 4 4 4 3 3
50522024-03-06T21:00:01.614Z17/30 Write block 69 len 8192 data: 7 6
50532024-03-06T21:00:01.614Z18/30 Read block 371 len 4096
50542024-03-06T21:00:01.614Z19/30 Read block 26 len 4096
50552024-03-06T21:00:01.614Z20/30 Read block 295 len 24576
50562024-03-06T21:00:01.615Z21/30 Write block 236 len 28672 data: 7 8 8 8 7 8 6
50572024-03-06T21:00:01.615Z22/30 Read block 568 len 32768
50582024-03-06T21:00:01.615Z23/30 Read block 382 len 20480
50592024-03-06T21:00:01.615Z24/30 Write block 463 len 20480 data: 9 8 8 7 6
50602024-03-06T21:00:01.615Z25/30 Read block 399 len 28672
50612024-03-06T21:00:01.615Z26/30 Read block 123 len 20480
50622024-03-06T21:00:01.616Z27/30 Read block 385 len 40960
50632024-03-06T21:00:01.616Z28/30 Read block 491 len 28672
50642024-03-06T21:00:01.616Z29/30 Write block 224 len 4096 data: 2
50652024-03-06T21:00:01.616Z30/30 Read block 335 len 40960
50662024-03-06T21:00:01.616Z----------------------------------------------------------------
50672024-03-06T21:00:01.616Z Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:30
50682024-03-06T21:00:01.616ZStates: Active Active Active
50692024-03-06T21:00:01.617ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
50702024-03-06T21:00:01.617Z 1 Acked 1000 Write 3 Done Done Sent false
50712024-03-06T21:00:01.617Z 2 Acked 1001 Write 10 Done Done Sent false
50722024-03-06T21:00:01.617Z 3 Acked 1002 Read 2 Done Done Sent false
50732024-03-06T21:00:01.618Z 4 Acked 1003 Read 7 Done Done Sent false
50742024-03-06T21:00:01.618Z 5 Acked 1004 Read 4 Done Done Sent false
50752024-03-06T21:00:01.618Z 6 Acked 1005 Write 5 Done Done Sent false
50762024-03-06T21:00:01.618Z 7 Acked 1006 Write 6 Done Done Sent false
50772024-03-06T21:00:01.618Z 8 Acked 1007 Write 1 Done Done Sent false
50782024-03-06T21:00:01.619Z 9 Acked 1008 Flush 0 Done Done Sent false
50792024-03-06T21:00:01.619Z 10 Acked 1009 Write 5 Done Done Sent false
50802024-03-06T21:00:01.619Z 11 Acked 1010 Write 10 Done Done Sent false
50812024-03-06T21:00:01.619Z 12 Acked 1011 Write 3 Done Done Sent false
50822024-03-06T21:00:01.619Z 13 Acked 1012 Read 6 Done Done Sent false
50832024-03-06T21:00:01.619Z 14 Acked 1013 Read 7 Done Done Sent false
50842024-03-06T21:00:01.619Z 15 Acked 1014 Write 10 Done Done Sent false
50852024-03-06T21:00:01.620Z 16 Acked 1015 Write 8 Done Done Sent false
50862024-03-06T21:00:01.620Z 17 Acked 1016 Write 2 Done Done Sent false
50872024-03-06T21:00:01.620Z 18 Acked 1017 Read 1 Done Done Sent false
50882024-03-06T21:00:01.620Z 19 Acked 1018 Read 1 Done Done Sent false
50892024-03-06T21:00:01.620Z 20 Acked 1019 Read 6 Done Done Sent false
50902024-03-06T21:00:01.621Z 21 Acked 1020 Write 7 Done Done Sent false
50912024-03-06T21:00:01.621Z 22 Acked 1021 Read 8 Done Done Sent false
50922024-03-06T21:00:01.621Z 23 Acked 1022 Read 5 Done Done Sent false
50932024-03-06T21:00:01.621Z 24 Acked 1023 Write 5 Done Done Sent false
50942024-03-06T21:00:01.621Z 25 Acked 1024 Read 7 Done Done Sent false
50952024-03-06T21:00:01.621Z 26 Acked 1025 Read 5 Done Done Sent false
50962024-03-06T21:00:01.621Z 27 Acked 1026 Read 10 Done Done Sent false
50972024-03-06T21:00:01.622Z 28 Acked 1027 Read 7 Done Done Sent false
50982024-03-06T21:00:01.622Z 29 Acked 1028 Write 1 Done Done Sent false
50992024-03-06T21:00:01.622Z 30 Acked 1029 Read 10 Done Sent Sent false
51002024-03-06T21:00:01.622Z STATES DS:0 DS:1 DS:2 TOTAL
51012024-03-06T21:00:01.622Z New 0 0 0 0
51022024-03-06T21:00:01.622Z Sent 0 1 30 31
51032024-03-06T21:00:01.622Z Done 30 29 0 59
51042024-03-06T21:00:01.622Z Skipped 0 0 0 0
51052024-03-06T21:00:01.623Z Error 0 0 0 0
51062024-03-06T21:00:01.623ZLast Flush: 1008 1008 0
51072024-03-06T21:00:01.623ZDownstairs last five completed:
51082024-03-06T21:00:01.623ZUpstairs last five completed: 30 29 28 27 26
51092024-03-06T21:00:01.623Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:00:00.353145593Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1323}
51102024-03-06T21:00:01.623ZWrote out file "/tmp/test_repair_verify.data"
51112024-03-06T21:00:01.623Z
51122024-03-06T21:00:01.623ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
51132024-03-06T21:00:01.644ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51142024-03-06T21:00:01.645Z 0 000-019 46 46 41 72 72 66 F F F <---
51152024-03-06T21:00:01.645Z 1 020-039 46 46 41 72 72 66 F F F <---
51162024-03-06T21:00:01.645Z 2 040-059 46 46 43 72 72 68 F F F <---
51172024-03-06T21:00:01.645Z 3 060-079 42 42 42 67 67 67 T T F <---
51182024-03-06T21:00:01.645Z 4 080-099 40 40 40 64 64 64 F F F
51192024-03-06T21:00:01.645Z 5 100-119 44 44 44 70 70 70 T T F <---
51202024-03-06T21:00:01.645Z 6 120-139 45 45 45 71 71 71 F F F
51212024-03-06T21:00:01.645Z 7 140-159 38 38 38 61 61 61 F F F
51222024-03-06T21:00:01.646Z 8 160-179 45 45 45 71 71 71 F F F
51232024-03-06T21:00:01.646Z 9 180-199 34 34 34 53 53 53 F F F
51242024-03-06T21:00:01.646Z 10 200-219 43 43 43 68 68 68 F F F
51252024-03-06T21:00:01.646Z 11 220-239 40 40 40 64 64 64 T T F <---
51262024-03-06T21:00:01.646Z 12 240-259 40 40 40 64 64 64 T T F <---
51272024-03-06T21:00:01.646Z 13 260-279 45 45 45 71 71 71 T T F <---
51282024-03-06T21:00:01.646Z 14 280-299 45 45 45 71 71 71 T T F <---
51292024-03-06T21:00:01.646Z 15 300-319 40 40 40 65 65 65 T T F <---
51302024-03-06T21:00:01.646Z 16 320-339 32 32 32 47 47 47 F F F
51312024-03-06T21:00:01.646Z 17 340-359 42 42 42 67 67 67 F F F
51322024-03-06T21:00:01.646Z 18 360-379 45 45 45 71 71 71 T T F <---
51332024-03-06T21:00:01.647Z 19 380-399 42 42 42 67 67 67 F F F
51342024-03-06T21:00:01.647Z 20 400-419 42 42 42 67 67 67 F F F
51352024-03-06T21:00:01.647Z 21 420-439 45 45 45 71 71 71 F F F
51362024-03-06T21:00:01.647Z 22 440-459 45 45 45 71 71 71 F F F
51372024-03-06T21:00:01.647Z 23 460-479 46 46 43 72 72 68 T T F <---
51382024-03-06T21:00:01.647Z 24 480-499 42 42 42 67 67 67 F F F
51392024-03-06T21:00:01.647Z 25 500-519 42 42 42 67 67 67 F F F
51402024-03-06T21:00:01.647Z 26 520-539 46 46 36 72 72 56 F F F <---
51412024-03-06T21:00:01.647Z 27 540-559 37 37 37 59 59 59 T T F <---
51422024-03-06T21:00:01.647Z 28 560-579 45 45 45 71 71 71 F F F
51432024-03-06T21:00:01.647Z 29 580-599 46 46 39 72 72 62 F F F <---
51442024-03-06T21:00:01.648ZMax gen: 46, Max flush: 72
51452024-03-06T21:00:01.648ZError: Difference in extent metadata found!
51462024-03-06T21:00:01.648ZA Difference in extent metadata is expected here
51472024-03-06T21:00:01.648ZOn loop 23
51482024-03-06T21:00:01.648Z
51492024-03-06T21:00:01.648ZVerifying data now
51502024-03-06T21:00:06.417ZNo change after verify
51512024-03-06T21:00:06.417ZLoop: 23 Downstairs dump after verify (and repair):
51522024-03-06T21:00:06.443ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51532024-03-06T21:00:06.443Z 0 000-019 46 46 46 72 72 72 F F F
51542024-03-06T21:00:06.443Z 1 020-039 46 46 46 72 72 72 F F F
51552024-03-06T21:00:06.443Z 2 040-059 46 46 46 72 72 72 F F F
51562024-03-06T21:00:06.443Z 3 060-079 47 47 47 73 73 73 F F F
51572024-03-06T21:00:06.444Z 4 080-099 40 40 40 64 64 64 F F F
51582024-03-06T21:00:06.444Z 5 100-119 47 47 47 73 73 73 F F F
51592024-03-06T21:00:06.444Z 6 120-139 45 45 45 71 71 71 F F F
51602024-03-06T21:00:06.444Z 7 140-159 38 38 38 61 61 61 F F F
51612024-03-06T21:00:07.162Z 8 160-179 45 45 45 71 71 71 F F F
51622024-03-06T21:00:07.162Z 9 180-199 34 34 34 53 53 53 F F F
51632024-03-06T21:00:07.162Z 10 200-219 43 43 43 68 68 68 F F F
51642024-03-06T21:00:07.162Z 11 220-239 47 47 47 73 73 73 F F F
51652024-03-06T21:00:07.162Z 12 240-259 47 47 47 73 73 73 F F F
51662024-03-06T21:00:07.162Z 13 260-279 47 47 47 73 73 73 F F F
51672024-03-06T21:00:07.162Z 14 280-299 47 47 47 73 73 73 F F F
51682024-03-06T21:00:07.162Z 15 300-319 47 47 47 73 73 73 F F F
51692024-03-06T21:00:07.163Z 16 320-339 32 32 32 47 47 47 F F F
51702024-03-06T21:00:07.163Z 17 340-359 42 42 42 67 67 67 F F F
51712024-03-06T21:00:07.163Z 18 360-379 47 47 47 73 73 73 F F F
51722024-03-06T21:00:07.163Z 19 380-399 42 42 42 67 67 67 F F F
51732024-03-06T21:00:07.163Z 20 400-419 42 42 42 67 67 67 F F F
51742024-03-06T21:00:07.163Z 21 420-439 45 45 45 71 71 71 F F F
51752024-03-06T21:00:07.164Z 22 440-459 45 45 45 71 71 71 F F F
51762024-03-06T21:00:07.164Z 23 460-479 47 47 47 73 73 73 F F F
51772024-03-06T21:00:07.164Z 24 480-499 42 42 42 67 67 67 F F F
51782024-03-06T21:00:07.164Z 25 500-519 42 42 42 67 67 67 F F F
51792024-03-06T21:00:07.164Z 26 520-539 46 46 46 72 72 72 F F F
51802024-03-06T21:00:07.164Z 27 540-559 47 47 47 73 73 73 F F F
51812024-03-06T21:00:07.164Z 28 560-579 45 45 45 71 71 71 F F F
51822024-03-06T21:00:07.164Z 29 580-599 46 46 46 72 72 72 F F F
51832024-03-06T21:00:07.164ZMax gen: 47, Max flush: 73
51842024-03-06T21:00:07.164Z
51852024-03-06T21:00:07.165ZBegin loop 24
51862024-03-06T21:00:07.165ZDownstairs to restart: 0
51872024-03-06T21:00:07.165Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.216669823Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331}
51882024-03-06T21:00:07.165Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
51892024-03-06T21:00:07.165Z,"time":"2024-03-06T21:00:05.21700407Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331}
51902024-03-06T21:00:07.165Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.217047557Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331}
51912024-03-06T21:00:07.165Z{"msg":"Crucible e766927e-74fe-4a2b-9cf9-dd1926de6a87 has session id: da178e7f-99dd-42a7-9010-019ca7eadabb","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.217079795Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
51922024-03-06T21:00:07.165Z{"msg":"Upstairs opts: Upstairs UUID: e766927e-74fe-4a2b-9cf9-dd1926de6a87, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.217124362Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
51932024-03-06T21:00:07.165Z{"msg":"Crucible stats registered with UUID: e766927e-74fe-4a2b-9cf9-dd1926de6a87","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.21715155Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
51942024-03-06T21:00:07.166Z{"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.217173438Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331}
51952024-03-06T21:00:07.166Z{"msg":"e766927e-74fe-4a2b-9cf9-dd1926de6a87 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.217210036Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
51962024-03-06T21:00:07.166Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.217240794Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"client":"0","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
51972024-03-06T21:00:07.166Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.217271341Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"client":"0","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
51982024-03-06T21:00:07.166Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.217320748Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"client":"1","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
51992024-03-06T21:00:07.166Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.217366425Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"client":"1","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52002024-03-06T21:00:07.166Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.217408392Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"client":"2","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52012024-03-06T21:00:07.167Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.217473548Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"client":"2","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52022024-03-06T21:00:07.167Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.217521684Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"":"io task","client":"2","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52032024-03-06T21:00:07.167Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.217648946Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"":"io task","client":"1","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52042024-03-06T21:00:07.167Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.217714211Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"":"io task","client":"0","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52052024-03-06T21:00:07.167Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.217782366Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"":"io task","client":"0","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52062024-03-06T21:00:07.167Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.217808544Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"":"io task","client":"2","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52072024-03-06T21:00:07.167Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.217826233Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"":"io task","client":"1","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52082024-03-06T21:00:07.167Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.218550903Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"client":"1","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52092024-03-06T21:00:07.167Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.21859239Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"client":"1","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52102024-03-06T21:00:07.167Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.218620469Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"client":"2","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52112024-03-06T21:00:07.168Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.218659106Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"client":"2","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52122024-03-06T21:00:07.669Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.219277713Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"client":"2","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52132024-03-06T21:00:07.670Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.21932975Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"client":"1","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52142024-03-06T21:00:07.670Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.219376207Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"client":"0","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52152024-03-06T21:00:07.670Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.219433712Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"client":"0","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52162024-03-06T21:00:07.670Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.219961486Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"client":"2","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52172024-03-06T21:00:07.670Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.220005543Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"client":"2","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52182024-03-06T21:00:07.670Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.220025422Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52192024-03-06T21:00:07.670Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.220065149Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"client":"1","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52202024-03-06T21:00:07.670Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.220092787Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"client":"1","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52212024-03-06T21:00:07.670Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.220124275Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52222024-03-06T21:00:07.671Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.220139204Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"client":"0","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52232024-03-06T21:00:07.671Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.220503709Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"client":"0","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52242024-03-06T21:00:07.671Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.220548126Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"client":"0","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52252024-03-06T21:00:07.671Z{"msg":"[0]R flush_numbers[0..12]: [72, 72, 72, 73, 64, 73, 71, 61, 71, 53, 68, 73]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.220567105Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"":"downstairs","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52262024-03-06T21:00:07.671Z{"msg":"[0]R generation[0..12]: [46, 46, 46, 47, 40, 47, 45, 38, 45, 34, 43, 47]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.220601212Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"":"downstairs","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52272024-03-06T21:00:07.671Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.220617961Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"":"downstairs","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52282024-03-06T21:00:07.671Z{"msg":"[1]R flush_numbers[0..12]: [72, 72, 72, 73, 64, 73, 71, 61, 71, 53, 68, 73]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.220651819Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"":"downstairs","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52292024-03-06T21:00:07.671Z{"msg":"Wait for a query_work_queue command to finish before sending IO
52302024-03-06T21:00:07.671Z[1]R generation[0..12]: [46, 46, 46, 47, 40, 47, 45, 38, 45, 34, 43, 47]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.220689966Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"":"downstairs","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52312024-03-06T21:00:07.671Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.220725363Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"":"downstairs","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52322024-03-06T21:00:07.671Z{"msg":"[2]R flush_numbers[0..12]: [72, 72, 72, 73, 64, 73, 71, 61, 71, 53, 68, 73]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.220759121Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"":"downstairs","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52332024-03-06T21:00:07.671Z{"msg":"[2]R generation[0..12]: [46, 46, 46, 47, 40, 47, 45, 38, 45, 34, 43, 47]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.220793409Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"":"downstairs","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52342024-03-06T21:00:07.671Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.220814388Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"":"downstairs","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52352024-03-06T21:00:07.672Z{"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.220844975Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"":"downstairs","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52362024-03-06T21:00:07.672Z{"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.220860884Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"":"downstairs","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52372024-03-06T21:00:07.672Z{"msg":"Next flush: 74","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.220893302Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"":"downstairs","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52382024-03-06T21:00:07.672Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.220912431Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"":"downstairs","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52392024-03-06T21:00:07.672ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
52402024-03-06T21:00:07.672Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.220946978Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52412024-03-06T21:00:07.672Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.220964147Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"client":"0","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52422024-03-06T21:00:07.672Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.220992405Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"client":"1","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52432024-03-06T21:00:07.672Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.221007174Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"client":"2","":"downstairs","session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52442024-03-06T21:00:07.672Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.221020553Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52452024-03-06T21:00:07.672Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.221050361Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52462024-03-06T21:00:07.672Z{"msg":"e766927e-74fe-4a2b-9cf9-dd1926de6a87 is now active with session: da178e7f-99dd-42a7-9010-019ca7eadabb","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.22107066Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52472024-03-06T21:00:07.673Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.221096598Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331,"session_id":"da178e7f-99dd-42a7-9010-019ca7eadabb"}
52482024-03-06T21:00:07.673Z{"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:05.221110617Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331}
52492024-03-06T21:00:07.673ZLoading write count information from file "/tmp/test_repair_verify.data"
52502024-03-06T21:00:07.673ZRun Repair workload
52512024-03-06T21:00:07.673Z01/30 Write block 469 len 40960 data: 6 3 3 3 3 3 3 4 4 4
52522024-03-06T21:00:07.673Z02/30 Flush
52532024-03-06T21:00:07.673Z03/30 Write block 357 len 32768 data: 5 5 5 6 7 5 8 7
52542024-03-06T21:00:07.673Z04/30 Read block 493 len 28672
52552024-03-06T21:00:07.673Z05/30 Read block 206 len 8192
52562024-03-06T21:00:07.673Z06/30 Read block 506 len 20480
52572024-03-06T21:00:07.673Z07/30 Write block 537 len 16384 data: 6 5 4 6
52582024-03-06T21:00:07.673Z08/30 Flush
52592024-03-06T21:00:07.673Z09/30 Flush
52602024-03-06T21:00:07.673Z10/30 Write block 422 len 4096 data: 6
52612024-03-06T21:00:07.673Z11/30 Write block 124 len 4096 data: 4
52622024-03-06T21:00:07.673Z12/30 Write block 369 len 28672 data: 5 6 5 5 4 4 5
52632024-03-06T21:00:07.674Z13/30 Read block 574 len 28672
52642024-03-06T21:00:07.674Z14/30 Flush
52652024-03-06T21:00:07.674Z15/30 Write block 405 len 28672 data: 3 3 3 3 3 3 3
52662024-03-06T21:00:07.674Z16/30 Read block 564 len 20480
52672024-03-06T21:00:07.674Z17/30 Write block 219 len 20480 data: 3 3 3 2 2
52682024-03-06T21:00:07.674Z18/30 Read block 390 len 4096
52692024-03-06T21:00:07.674Z19/30 Read block 239 len 28672
52702024-03-06T21:00:07.674Z20/30 Read block 416 len 16384
52712024-03-06T21:00:07.674Z21/30 Read block 331 len 32768
52722024-03-06T21:00:07.674Z22/30 Read block 89 len 8192
52732024-03-06T21:00:07.674Z23/30 Read block 14 len 12288
52742024-03-06T21:00:07.675Z24/30 Write block 180 len 16384 data: 8 10 7 5
52752024-03-06T21:00:07.675Z25/30 Read block 282 len 32768
52762024-03-06T21:00:07.675Z26/30 Read block 293 len 4096
52772024-03-06T21:00:07.675Z27/30 Write block 134 len 16384 data: 3 3 4 3
52782024-03-06T21:00:07.675Z28/30 Read block 219 len 4096
52792024-03-06T21:00:07.675Z29/30 Read block 418 len 16384
52802024-03-06T21:00:07.675Z30/30 Read block 252 len 32768
52812024-03-06T21:00:07.675Z----------------------------------------------------------------
52822024-03-06T21:00:07.675Z Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30
52832024-03-06T21:00:07.675ZStates: Active Active Active
52842024-03-06T21:00:07.675ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
52852024-03-06T21:00:07.675Z 1 Acked 1000 Write 10 Sent Done Done false
52862024-03-06T21:00:07.675Z 2 Acked 1001 Flush 0 Sent Done Done false
52872024-03-06T21:00:07.675Z 3 Acked 1002 Write 8 Sent Done Done false
52882024-03-06T21:00:07.675Z 4 Acked 1003 Read 7 Sent Done Done false
52892024-03-06T21:00:07.675Z 5 Acked 1004 Read 2 Sent Done Done false
52902024-03-06T21:00:07.675Z 6 Acked 1005 Read 5 Sent Done Done false
52912024-03-06T21:00:07.675Z 7 Acked 1006 Write 4 Sent Done Done false
52922024-03-06T21:00:07.675Z 8 Acked 1007 Flush 0 Sent Done Done false
52932024-03-06T21:00:07.675Z 9 Acked 1008 Flush 0 Sent Done Done false
52942024-03-06T21:00:07.675Z 10 Acked 1009 Write 1 Sent Done Done false
52952024-03-06T21:00:07.675Z 11 Acked 1010 Write 1 Sent Done Done false
52962024-03-06T21:00:07.675Z 12 Acked 1011 Write 7 Sent Done Done false
52972024-03-06T21:00:07.676Z 13 Acked 1012 Read 7 Sent Done Done false
52982024-03-06T21:00:07.676Z 14 Acked 1013 Flush 0 Sent Done Done false
52992024-03-06T21:00:07.676Z 15 Acked 1014 Write 7 Sent Done Done false
53002024-03-06T21:00:07.676Z 16 Acked 1015 Read 5 Sent Done Done false
53012024-03-06T21:00:07.676Z 17 Acked 1016 Write 5 Sent Done Done false
53022024-03-06T21:00:07.676Z 18 Acked 1017 Read 1 Sent Done Done false
53032024-03-06T21:00:07.676Z 19 Acked 1018 Read 7 Sent Done Done false
53042024-03-06T21:00:07.676Z 20 Acked 1019 Read 4 Sent Done Done false
53052024-03-06T21:00:07.676Z 21 Acked 1020 Read 8 Sent Done Done false
53062024-03-06T21:00:07.676Z 22 Acked 1021 Read 2 Sent Done Done false
53072024-03-06T21:00:07.676Z 23 Acked 1022 Read 3 Sent Done Done false
53082024-03-06T21:00:07.676Z 24 Acked 1023 Write 4 Sent Done Done false
53092024-03-06T21:00:07.676Z 25 Acked 1024 Read 8 Sent Done Done false
53102024-03-06T21:00:07.676Z 26 Acked 1025 Read 1 Sent Done Done false
53112024-03-06T21:00:07.676Z 27 Acked 1026 Write 4 Sent Done Done false
53122024-03-06T21:00:07.694Z 28 Acked 1027 Read 1 Sent Done Done false
53132024-03-06T21:00:07.694Z 29 Acked 1028 Read 4 Sent Done Done false
53142024-03-06T21:00:07.694Z 30 Acked 1029 Read 8 Sent Done Sent false
53152024-03-06T21:00:07.694Z STATES DS:0 DS:1 DS:2 TOTAL
53162024-03-06T21:00:07.694Z New 0 0 0 0
53172024-03-06T21:00:07.694Z Sent 30 0 1 31
53182024-03-06T21:00:07.694Z Done 0 30 29 59
53192024-03-06T21:00:07.694Z Skipped 0 0 0 0
53202024-03-06T21:00:07.695Z Error 0 0 0 0
53212024-03-06T21:00:07.695ZLast Flush: 0 1013 1013
53222024-03-06T21:00:07.695ZDownstairs last five completed:
53232024-03-06T21:00:07.695ZUpstairs last five completed: 30 29 28 27 26
53242024-03-06T21:00:07.695Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:00:05.254676354Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1331}
53252024-03-06T21:00:07.695ZWrote out file "/tmp/test_repair_verify.data"
53262024-03-06T21:00:07.695Z
53272024-03-06T21:00:07.695ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
53282024-03-06T21:00:07.695ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53292024-03-06T21:00:07.695Z 0 000-019 46 46 46 72 72 72 F F F
53302024-03-06T21:00:07.695Z 1 020-039 46 46 46 72 72 72 F F F
53312024-03-06T21:00:07.695Z 2 040-059 46 46 46 72 72 72 F F F
53322024-03-06T21:00:07.695Z 3 060-079 47 47 47 73 73 73 F F F
53332024-03-06T21:00:07.695Z 4 080-099 40 40 40 64 64 64 F F F
53342024-03-06T21:00:07.695Z 5 100-119 47 47 47 73 73 73 F F F
53352024-03-06T21:00:07.695Z 6 120-139 45 48 48 71 77 77 F T T <---
53362024-03-06T21:00:07.695Z 7 140-159 38 38 38 61 61 61 F F F
53372024-03-06T21:00:07.695Z 8 160-179 45 45 45 71 71 71 F F F
53382024-03-06T21:00:07.696Z 9 180-199 34 34 34 53 53 53 F T T <---
53392024-03-06T21:00:07.696Z 10 200-219 43 43 43 68 68 68 F T T <---
53402024-03-06T21:00:07.696Z 11 220-239 47 47 47 73 73 73 F T T <---
53412024-03-06T21:00:07.696Z 12 240-259 47 47 47 73 73 73 F F F
53422024-03-06T21:00:07.696Z 13 260-279 47 47 47 73 73 73 F F F
53432024-03-06T21:00:07.696Z 14 280-299 47 47 47 73 73 73 F F F
53442024-03-06T21:00:07.696Z 15 300-319 47 47 47 73 73 73 F F F
53452024-03-06T21:00:07.696Z 16 320-339 32 32 32 47 47 47 F F F
53462024-03-06T21:00:07.696Z 17 340-359 42 48 48 67 75 75 F F F <---
53472024-03-06T21:00:07.696Z 18 360-379 47 48 48 73 77 77 F F F <---
53482024-03-06T21:00:07.696Z 19 380-399 42 42 42 67 67 67 F F F
53492024-03-06T21:00:07.696Z 20 400-419 42 42 42 67 67 67 F T T <---
53502024-03-06T21:00:07.696Z 21 420-439 45 48 48 71 77 77 F F F <---
53512024-03-06T21:00:07.696Z 22 440-459 45 45 45 71 71 71 F F F
53522024-03-06T21:00:07.696Z 23 460-479 47 48 48 73 74 74 F F F <---
53532024-03-06T21:00:07.696Z 24 480-499 42 42 42 67 67 67 F F F
53542024-03-06T21:00:07.696Z 25 500-519 42 42 42 67 67 67 F F F
53552024-03-06T21:00:07.696Z 26 520-539 46 48 48 72 75 75 F F F <---
53562024-03-06T21:00:07.696Z 27 540-559 47 48 48 73 75 75 F F F <---
53572024-03-06T21:00:07.696Z 28 560-579 45 45 45 71 71 71 F F F
53582024-03-06T21:00:07.696Z 29 580-599 46 46 46 72 72 72 F F F
53592024-03-06T21:00:07.696ZMax gen: 48, Max flush: 77
53602024-03-06T21:00:07.696ZError: Difference in extent metadata found!
53612024-03-06T21:00:07.696ZA Difference in extent metadata is expected here
53622024-03-06T21:00:07.697ZOn loop 24
53632024-03-06T21:00:07.697Z
53642024-03-06T21:00:07.697ZVerifying data now
53652024-03-06T21:00:11.142ZNo change after verify
53662024-03-06T21:00:11.143ZLoop: 24 Downstairs dump after verify (and repair):
53672024-03-06T21:00:11.161ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53682024-03-06T21:00:11.161Z 0 000-019 46 46 46 72 72 72 F F F
53692024-03-06T21:00:11.161Z 1 020-039 46 46 46 72 72 72 F F F
53702024-03-06T21:00:11.161Z 2 040-059 46 46 46 72 72 72 F F F
53712024-03-06T21:00:11.161Z 3 060-079 47 47 47 73 73 73 F F F
53722024-03-06T21:00:11.162Z 4 080-099 40 40 40 64 64 64 F F F
53732024-03-06T21:00:11.162Z 5 100-119 47 47 47 73 73 73 F F F
53742024-03-06T21:00:11.162Z 6 120-139 49 49 49 78 78 78 F F F
53752024-03-06T21:00:11.162Z 7 140-159 38 38 38 61 61 61 F F F
53762024-03-06T21:00:11.162Z 8 160-179 45 45 45 71 71 71 F F F
53772024-03-06T21:00:11.162Z 9 180-199 49 49 49 78 78 78 F F F
53782024-03-06T21:00:11.162Z 10 200-219 49 49 49 78 78 78 F F F
53792024-03-06T21:00:11.162Z 11 220-239 49 49 49 78 78 78 F F F
53802024-03-06T21:00:11.163Z 12 240-259 47 47 47 73 73 73 F F F
53812024-03-06T21:00:11.163Z 13 260-279 47 47 47 73 73 73 F F F
53822024-03-06T21:00:11.163Z 14 280-299 47 47 47 73 73 73 F F F
53832024-03-06T21:00:11.163Z 15 300-319 47 47 47 73 73 73 F F F
53842024-03-06T21:00:11.163Z 16 320-339 32 32 32 47 47 47 F F F
53852024-03-06T21:00:11.163Z 17 340-359 48 48 48 75 75 75 F F F
53862024-03-06T21:00:11.163Z 18 360-379 48 48 48 77 77 77 F F F
53872024-03-06T21:00:11.163Z 19 380-399 42 42 42 67 67 67 F F F
53882024-03-06T21:00:11.164Z 20 400-419 49 49 49 78 78 78 F F F
53892024-03-06T21:00:11.164Z 21 420-439 48 48 48 77 77 77 F F F
53902024-03-06T21:00:11.164Z 22 440-459 45 45 45 71 71 71 F F F
53912024-03-06T21:00:11.164Z 23 460-479 48 48 48 74 74 74 F F F
53922024-03-06T21:00:11.164Z 24 480-499 42 42 42 67 67 67 F F F
53932024-03-06T21:00:11.164Z 25 500-519 42 42 42 67 67 67 F F F
53942024-03-06T21:00:11.164Z 26 520-539 48 48 48 75 75 75 F F F
53952024-03-06T21:00:11.164Z 27 540-559 48 48 48 75 75 75 F F F
53962024-03-06T21:00:11.165Z 28 560-579 45 45 45 71 71 71 F F F
53972024-03-06T21:00:11.165Z 29 580-599 46 46 46 72 72 72 F F F
53982024-03-06T21:00:11.165ZMax gen: 49, Max flush: 78
53992024-03-06T21:00:11.210Z
54002024-03-06T21:00:11.210ZBegin loop 25
54012024-03-06T21:00:11.210ZDownstairs to restart: 0
54022024-03-06T21:00:11.278Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.942618955Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339}
54032024-03-06T21:00:11.278Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.942911675Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339}
54042024-03-06T21:00:11.278Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.942947172Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339}
54052024-03-06T21:00:11.279Z{"msg":"Crucible 927aa8a5-cb6b-49ca-a708-c4ffd8ba28b3 has session id: aa1eee9f-ffbb-4c46-a7f7-7e26685eead0","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.94297678Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54062024-03-06T21:00:11.279Z{"msg":"Upstairs opts: Upstairs UUID: 927aa8a5-cb6b-49ca-a708-c4ffd8ba28b3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
54072024-03-06T21:00:11.279Z,"time":"2024-03-06T21:00:09.943033506Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54082024-03-06T21:00:11.279Z{"msg":"Crucible stats registered with UUID: 927aa8a5-cb6b-49ca-a708-c4ffd8ba28b3","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.943065384Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54092024-03-06T21:00:11.279Z{"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.94311844Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339}
54102024-03-06T21:00:11.280Z{"msg":"927aa8a5-cb6b-49ca-a708-c4ffd8ba28b3 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.943237432Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54112024-03-06T21:00:11.280Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.943275269Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"client":"0","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54122024-03-06T21:00:11.280Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.943297798Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"client":"0","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54132024-03-06T21:00:11.280Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.943322216Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"client":"1","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54142024-03-06T21:00:11.281Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.943364223Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"client":"1","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54152024-03-06T21:00:11.281Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.943390131Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"client":"2","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54162024-03-06T21:00:11.281Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.943435818Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"client":"2","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54172024-03-06T21:00:11.281Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.943481915Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"":"io task","client":"2","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54182024-03-06T21:00:11.282Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.943545731Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"":"io task","client":"0","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54192024-03-06T21:00:11.282Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.943588428Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"":"io task","client":"1","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54202024-03-06T21:00:11.282Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.943623265Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"":"io task","client":"2","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54212024-03-06T21:00:11.282Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.943647844Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"":"io task","client":"1","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54222024-03-06T21:00:11.282Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.943754766Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"":"io task","client":"0","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54232024-03-06T21:00:11.282Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.94442751Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"client":"2","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54242024-03-06T21:00:11.283Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.944466897Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"client":"2","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54252024-03-06T21:00:11.283Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.944485196Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"client":"1","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54262024-03-06T21:00:11.283Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.944502435Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"client":"1","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54272024-03-06T21:00:11.283Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.94500393Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"client":"1","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54282024-03-06T21:00:11.283Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.945144621Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"client":"2","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54292024-03-06T21:00:11.283Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.94558533Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"client":"1","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54302024-03-06T21:00:11.283Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.945652575Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"client":"1","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54312024-03-06T21:00:11.283Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.945683703Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54322024-03-06T21:00:11.283Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.94573207Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"client":"0","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54332024-03-06T21:00:11.284Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.945770438Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"client":"0","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54342024-03-06T21:00:11.284Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.945904308Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"client":"2","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54352024-03-06T21:00:11.284Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.945933946Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"client":"2","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54362024-03-06T21:00:11.284Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.945950515Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54372024-03-06T21:00:11.284Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.94631691Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"client":"0","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54382024-03-06T21:00:11.284Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.946736521Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"client":"0","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54392024-03-06T21:00:11.284Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.946778698Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"client":"0","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54402024-03-06T21:00:11.284Z{"msg":"[0]R flush_numbers[0..12]: [72, 72, 72, 73, 64, 73, 78, 61, 71, 78, 78, 78]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.946796237Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"":"downstairs","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54412024-03-06T21:00:11.284Z{"msg":"[0]R generation[0..12]: [46, 46, 46, 47, 40, 47, 49, 38, 45, 49, 49, 49]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.946827874Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"":"downstairs","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54422024-03-06T21:00:11.285Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.946849533Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"":"downstairs","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54432024-03-06T21:00:11.285Z{"msg":"[1]R flush_numbers[0..12]: [72, 72, 72, 73, 64, 73, 78, 61, 71, 78, 78, 78]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.946880101Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"":"downstairs","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54442024-03-06T21:00:11.285Z{"msg":"[1]R generation[0..12]: [46, 46, 46, 47, 40, 47, 49, 38, 45, 49, 49, 49]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
54452024-03-06T21:00:11.285Z,"time":"2024-03-06T21:00:09.94689689Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"":"downstairs","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54462024-03-06T21:00:11.285Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.946924148Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"":"downstairs","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54472024-03-06T21:00:11.285Z{"msg":"[2]R flush_numbers[0..12]: [72, 72, 72, 73, 64, 73, 78, 61, 71, 78, 78, 78]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.946952966Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"":"downstairs","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54482024-03-06T21:00:11.285Z{"msg":"[2]R generation[0..12]: [46, 46, 46, 47, 40, 47, 49, 38, 45, 49, 49, 49]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.946970735Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"":"downstairs","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54492024-03-06T21:00:11.285Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.947017301Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"":"downstairs","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54502024-03-06T21:00:11.285Z{"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.94703659Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"":"downstairs","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54512024-03-06T21:00:11.286Z{"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.947050889Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"":"downstairs","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54522024-03-06T21:00:11.286Z{"msg":"Next flush: 79","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.947068878Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"":"downstairs","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54532024-03-06T21:00:11.286Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.947103355Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"":"downstairs","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54542024-03-06T21:00:11.286Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.947130454Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54552024-03-06T21:00:11.286Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.947149123Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"client":"0","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54562024-03-06T21:00:11.286Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.947163151Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"client":"1","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54572024-03-06T21:00:11.286Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.947176601Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"client":"2","":"downstairs","session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54582024-03-06T21:00:11.287Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.947208878Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54592024-03-06T21:00:11.287Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.947233657Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54602024-03-06T21:00:11.287Z{"msg":"927aa8a5-cb6b-49ca-a708-c4ffd8ba28b3 is now active with session: aa1eee9f-ffbb-4c46-a7f7-7e26685eead0","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.947262155Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54612024-03-06T21:00:11.287Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.947275424Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339,"session_id":"aa1eee9f-ffbb-4c46-a7f7-7e26685eead0"}
54622024-03-06T21:00:11.287Z{"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:09.947288473Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339}
54632024-03-06T21:00:11.287ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
54642024-03-06T21:00:11.288ZLoading write count information from file "/tmp/test_repair_verify.data"
54652024-03-06T21:00:11.288ZRun Repair workload
54662024-03-06T21:00:11.288Z01/30 Write block 49 len 4096 data: 5
54672024-03-06T21:00:11.288Z02/30 Read block 46 len 8192
54682024-03-06T21:00:11.288Z03/30 Read block 344 len 32768
54692024-03-06T21:00:11.288Z04/30 Write block 96 len 4096 data: 4
54702024-03-06T21:00:11.288Z05/30 Flush
54712024-03-06T21:00:11.288Z06/30 Write block 281 len 40960 data: 4 4 4 6 8 10 10 10 11 11
54722024-03-06T21:00:11.288Z07/30 Read block 207 len 16384
54732024-03-06T21:00:11.288Z08/30 Write block 347 len 4096 data: 4
54742024-03-06T21:00:11.289Z09/30 Read block 361 len 12288
54752024-03-06T21:00:11.289Z10/30 Read block 523 len 28672
54762024-03-06T21:00:11.289Z11/30 Read block 121 len 40960
54772024-03-06T21:00:11.289Z12/30 Read block 234 len 32768
54782024-03-06T21:00:11.289Z13/30 Write block 313 len 28672 data: 6 5 5 4 3 3 3
54792024-03-06T21:00:11.289Z14/30 Read block 61 len 40960
54802024-03-06T21:00:11.289Z15/30 Write block 461 len 20480 data: 8 8 10 9 9
54812024-03-06T21:00:11.289Z16/30 Flush
54822024-03-06T21:00:11.289Z17/30 Write block 91 len 36864 data: 3 3 4 5 4 5 5 6 6
54832024-03-06T21:00:11.289Z18/30 Read block 144 len 20480
54842024-03-06T21:00:11.289Z19/30 Flush
54852024-03-06T21:00:11.290Z20/30 Write block 61 len 36864 data: 11 11 13 11 10 10 9 7 8
54862024-03-06T21:00:11.290Z21/30 Read block 253 len 8192
54872024-03-06T21:00:11.290Z22/30 Read block 514 len 32768
54882024-03-06T21:00:11.290Z23/30 Write block 529 len 36864 data: 6 7 7 7 7 5 6 5 7
54892024-03-06T21:00:11.290Z24/30 Read block 586 len 12288
54902024-03-06T21:00:11.290Z25/30 Read block 364 len 4096
54912024-03-06T21:00:11.290Z26/30 Write block 420 len 24576 data: 5 8 7 6 5 5
54922024-03-06T21:00:11.290Z27/30 Write block 103 len 24576 data: 6 6 5 7 8 5
54932024-03-06T21:00:11.290Z28/30 Write block 198 len 12288 data: 3 2 3
54942024-03-06T21:00:11.290Z29/30 Write block 372 len 16384 data: 6 5 5 6
54952024-03-06T21:00:11.290Z30/30 Read block 83 len 32768
54962024-03-06T21:00:11.290Z----------------------------------------------------------------
54972024-03-06T21:00:11.290Z Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:30
54982024-03-06T21:00:11.290ZStates: Active Active Active
54992024-03-06T21:00:11.291ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
55002024-03-06T21:00:11.291Z 1 Acked 1000 Write 1 Sent Done Done false
55012024-03-06T21:00:11.291Z 2 Acked 1001 Read 2 Sent Done Done false
55022024-03-06T21:00:11.300Z 3 Acked 1002 Read 8 Sent Done Done false
55032024-03-06T21:00:11.300Z 4 Acked 1003 Write 1 Sent Done Done false
55042024-03-06T21:00:11.300Z 5 Acked 1004 Flush 0 Sent Done Done false
55052024-03-06T21:00:11.300Z 6 Acked 1005 Write 10 Sent Done Done false
55062024-03-06T21:00:11.301Z 7 Acked 1006 Read 4 Sent Done Done false
55072024-03-06T21:00:11.301Z 8 Acked 1007 Write 1 Sent Done Done false
55082024-03-06T21:00:11.301Z 9 Acked 1008 Read 3 Sent Done Done false
55092024-03-06T21:00:11.301Z 10 Acked 1009 Read 7 Sent Done Done false
55102024-03-06T21:00:11.301Z 11 Acked 1010 Read 10 Sent Done Done false
55112024-03-06T21:00:11.301Z 12 Acked 1011 Read 8 Sent Done Done false
55122024-03-06T21:00:11.302Z 13 Acked 1012 Write 7 Sent Done Done false
55132024-03-06T21:00:11.302Z 14 Acked 1013 Read 10 Sent Done Done false
55142024-03-06T21:00:11.302Z 15 Acked 1014 Write 5 Sent Done Done false
55152024-03-06T21:00:11.302Z 16 Acked 1015 Flush 0 Sent Done Done false
55162024-03-06T21:00:11.302Z 17 Acked 1016 Write 9 Sent Done Done false
55172024-03-06T21:00:11.302Z 18 Acked 1017 Read 5 Sent Done Done false
55182024-03-06T21:00:11.302Z 19 Acked 1018 Flush 0 Sent Done Done false
55192024-03-06T21:00:11.302Z 20 Acked 1019 Write 9 Sent Done Done false
55202024-03-06T21:00:11.303Z 21 Acked 1020 Read 2 Sent Done Done false
55212024-03-06T21:00:11.303Z 22 Acked 1021 Read 8 Sent Done Done false
55222024-03-06T21:00:11.303Z 23 Acked 1022 Write 9 Sent Done Done false
55232024-03-06T21:00:11.303Z 24 Acked 1023 Read 3 Sent Done Done false
55242024-03-06T21:00:11.303Z 25 Acked 1024 Read 1 Sent Done Done false
55252024-03-06T21:00:11.303Z 26 Acked 1025 Write 6 Sent Done Done false
55262024-03-06T21:00:11.303Z 27 Acked 1026 Write 6 Sent Done Done false
55272024-03-06T21:00:11.304Z 28 Acked 1027 Write 3 Sent Done Done false
55282024-03-06T21:00:11.304Z 29 Acked 1028 Write 4 Sent Done Sent false
55292024-03-06T21:00:11.304Z 30 Acked 1029 Read 8 Sent Done Sent false
55302024-03-06T21:00:11.304Z STATES DS:0 DS:1 DS:2 TOTAL
55312024-03-06T21:00:11.304Z New 0 0 0 0
55322024-03-06T21:00:11.304Z Sent 30 0 2 32
55332024-03-06T21:00:11.305Z Done 0 30 28 58
55342024-03-06T21:00:11.305Z Skipped 0 0 0 0
55352024-03-06T21:00:11.305Z Error 0 0 0 0
55362024-03-06T21:00:11.305ZLast Flush: 0 1018 1018
55372024-03-06T21:00:11.305ZDownstairs last five completed:
55382024-03-06T21:00:11.305ZUpstairs last five completed: 30 29 28 27 26
55392024-03-06T21:00:11.305Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:00:09.983711533Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1339}
55402024-03-06T21:00:11.306ZWrote out file "/tmp/test_repair_verify.data"
55412024-03-06T21:00:11.306Z
55422024-03-06T21:00:11.306ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
55432024-03-06T21:00:11.306ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
55442024-03-06T21:00:11.306Z 0 000-019 46 46 46 72 72 72 F F F
55452024-03-06T21:00:11.306Z 1 020-039 46 46 46 72 72 72 F F F
55462024-03-06T21:00:11.306Z 2 040-059 46 50 50 72 79 79 F F F <---
55472024-03-06T21:00:11.307Z 3 060-079 47 47 47 73 73 73 F T T <---
55482024-03-06T21:00:11.307Z 4 080-099 40 50 50 64 81 81 F F F <---
55492024-03-06T21:00:11.307Z 5 100-119 47 47 47 73 73 73 F T T <---
55502024-03-06T21:00:11.307Z 6 120-139 49 49 49 78 78 78 F F F
55512024-03-06T21:00:11.307Z 7 140-159 38 38 38 61 61 61 F F F
55522024-03-06T21:00:11.307Z 8 160-179 45 45 45 71 71 71 F F F
55532024-03-06T21:00:11.307Z 9 180-199 49 49 49 78 78 78 F T T <---
55542024-03-06T21:00:11.308Z 10 200-219 49 49 49 78 78 78 F T T <---
55552024-03-06T21:00:11.308Z 11 220-239 49 49 49 78 78 78 F F F
55562024-03-06T21:00:11.308Z 12 240-259 47 47 47 73 73 73 F F F
55572024-03-06T21:00:11.308Z 13 260-279 47 47 47 73 73 73 F F F
55582024-03-06T21:00:11.308Z 14 280-299 47 50 50 73 80 80 F F F <---
55592024-03-06T21:00:11.308Z 15 300-319 47 50 50 73 80 80 F F F <---
55602024-03-06T21:00:11.308Z 16 320-339 32 32 32 47 47 47 F F F
55612024-03-06T21:00:11.308Z 17 340-359 48 50 50 75 80 80 F F F <---
55622024-03-06T21:00:11.309Z 18 360-379 48 48 48 77 77 77 F T T <---
55632024-03-06T21:00:11.309Z 19 380-399 42 42 42 67 67 67 F F F
55642024-03-06T21:00:11.309Z 20 400-419 49 49 49 78 78 78 F F F
55652024-03-06T21:00:11.309Z 21 420-439 48 48 48 77 77 77 F T T <---
55662024-03-06T21:00:11.309Z 22 440-459 45 45 45 71 71 71 F F F
55672024-03-06T21:00:11.309Z 23 460-479 48 50 50 74 80 80 F F F <---
55682024-03-06T21:00:11.310Z 24 480-499 42 42 42 67 67 67 F F F
55692024-03-06T21:00:11.310Z 25 500-519 42 42 42 67 67 67 F F F
55702024-03-06T21:00:11.310Z 26 520-539 48 48 48 75 75 75 F T T <---
55712024-03-06T21:00:11.310Z 27 540-559 48 48 48 75 75 75 F F F
55722024-03-06T21:00:11.310Z 28 560-579 45 45 45 71 71 71 F F F
55732024-03-06T21:00:11.310Z 29 580-599 46 46 46 72 72 72 F F F
55742024-03-06T21:00:11.311ZMax gen: 50, Max flush: 81
55752024-03-06T21:00:11.311ZError: Difference in extent metadata found!
55762024-03-06T21:00:11.311ZA Difference in extent metadata is expected here
55772024-03-06T21:00:11.311ZOn loop 25
55782024-03-06T21:00:11.311Z
55792024-03-06T21:00:11.311ZVerifying data now
55802024-03-06T21:00:15.949ZNo change after verify
55812024-03-06T21:00:15.949ZLoop: 25 Downstairs dump after verify (and repair):
55822024-03-06T21:00:15.959ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
55832024-03-06T21:00:15.959Z 0 000-019 46 46 46 72 72 72 F F F
55842024-03-06T21:00:15.959Z 1 020-039 46 46 46 72 72 72 F F F
55852024-03-06T21:00:15.959Z 2 040-059 50 50 50 79 79 79 F F F
55862024-03-06T21:00:15.959Z 3 060-079 51 51 51 82 82 82 F F F
55872024-03-06T21:00:15.959Z 4 080-099 50 50 50 81 81 81 F F F
55882024-03-06T21:00:15.959Z 5 100-119 51 51 51 82 82 82 F F F
55892024-03-06T21:00:15.959Z 6 120-139 49 49 49 78 78 78 F F F
55902024-03-06T21:00:15.959Z 7 140-159 38 38 38 61 61 61 F F F
55912024-03-06T21:00:15.959Z 8 160-179 45 45 45 71 71 71 F F F
55922024-03-06T21:00:15.959Z 9 180-199 51 51 51 82 82 82 F F F
55932024-03-06T21:00:15.959Z 10 200-219 51 51 51 82 82 82 F F F
55942024-03-06T21:00:15.959Z 11 220-239 49 49 49 78 78 78 F F F
55952024-03-06T21:00:15.959Z 12 240-259 47 47 47 73 73 73 F F F
55962024-03-06T21:00:15.959Z 13 260-279 47 47 47 73 73 73 F F F
55972024-03-06T21:00:15.959Z 14 280-299 50 50 50 80 80 80 F F F
55982024-03-06T21:00:15.959Z 15 300-319 50 50 50 80 80 80 F F F
55992024-03-06T21:00:15.959Z 16 320-339 32 32 32 47 47 47 F F F
56002024-03-06T21:00:15.959Z 17 340-359 50 50 50 80 80 80 F F F
56012024-03-06T21:00:15.959Z 18 360-379 51 51 51 82 82 82 F F F
56022024-03-06T21:00:15.959Z 19 380-399 42 42 42 67 67 67 F F F
56032024-03-06T21:00:15.959Z 20 400-419 49 49 49 78 78 78 F F F
56042024-03-06T21:00:15.960Z 21 420-439 51 51 51 82 82 82 F F F
56052024-03-06T21:00:15.960Z 22 440-459 45 45 45 71 71 71 F F F
56062024-03-06T21:00:15.963Z 23 460-479 50 50 50 80 80 80 F F F
56072024-03-06T21:00:15.963Z 24 480-499 42 42 42 67 67 67 F F F
56082024-03-06T21:00:15.963Z 25 500-519 42 42 42 67 67 67 F F F
56092024-03-06T21:00:15.963Z 26 520-539 51 51 51 82 82 82 F F F
56102024-03-06T21:00:15.963Z 27 540-559 48 48 48 75 75 75 F F F
56112024-03-06T21:00:15.964Z 28 560-579 45 45 45 71 71 71 F F F
56122024-03-06T21:00:15.964Z 29 580-599 46 46 46 72 72 72 F F F
56132024-03-06T21:00:15.964ZMax gen: 51, Max flush: 82
56142024-03-06T21:00:15.964Z
56152024-03-06T21:00:15.964ZBegin loop 26
56162024-03-06T21:00:15.964ZDownstairs to restart: 2
56172024-03-06T21:00:15.996Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.75166609Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347}
56182024-03-06T21:00:15.996Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.751976538Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347}
56192024-03-06T21:00:15.996Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.752003976Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347}
56202024-03-06T21:00:15.996Z{"msg":"Crucible ab959ffb-cb7c-45fa-a9af-09aa9ec0c5c7 has session id: 9615c55d-2169-496b-ada8-a040a8ad422d","v":0,"name":"crucible","level":30Crucible runtime is spawned
56212024-03-06T21:00:15.996Z,"time":"2024-03-06T21:00:14.752040004Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56222024-03-06T21:00:15.996Z{"msg":"Upstairs opts: Upstairs UUID: ab959ffb-cb7c-45fa-a9af-09aa9ec0c5c7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.752102029Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56232024-03-06T21:00:15.996Z{"msg":"Crucible stats registered with UUID: ab959ffb-cb7c-45fa-a9af-09aa9ec0c5c7","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.752131447Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56242024-03-06T21:00:15.997Z{"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.752157816Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347}
56252024-03-06T21:00:15.997Z{"msg":"ab959ffb-cb7c-45fa-a9af-09aa9ec0c5c7 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.752319714Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56262024-03-06T21:00:15.997Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.752354662Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"client":"0","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56272024-03-06T21:00:15.997Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.752377721Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"client":"0","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56282024-03-06T21:00:15.997Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.752394679Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"client":"1","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56292024-03-06T21:00:15.997Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.752424287Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"client":"1","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56302024-03-06T21:00:15.997Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.752448236Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"client":"2","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56312024-03-06T21:00:15.997Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.752474494Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"client":"2","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56322024-03-06T21:00:15.997Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.752498942Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"":"io task","client":"2","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56332024-03-06T21:00:15.997Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.75252981Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"":"io task","client":"0","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56342024-03-06T21:00:15.997Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.752562968Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"":"io task","client":"1","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56352024-03-06T21:00:15.997Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.75282495Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"":"io task","client":"0","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56362024-03-06T21:00:15.997Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.752868677Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"":"io task","client":"2","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56372024-03-06T21:00:15.997Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.752895615Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"":"io task","client":"1","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56382024-03-06T21:00:15.997Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.7536948Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"client":"1","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56392024-03-06T21:00:15.997Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.753727348Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"client":"1","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56402024-03-06T21:00:15.998Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.753743386Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"client":"0","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56412024-03-06T21:00:15.998Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.753757606Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"client":"0","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56422024-03-06T21:00:15.998Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.754348325Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"client":"0","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56432024-03-06T21:00:15.998Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.754506894Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"client":"1","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56442024-03-06T21:00:15.998Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.754937614Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"client":"0","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56452024-03-06T21:00:15.998Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.75500079Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"client":"0","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56462024-03-06T21:00:15.998Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.755030738Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56472024-03-06T21:00:15.998Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.75513538Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"client":"2","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56482024-03-06T21:00:15.998Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.755174348Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"client":"2","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56492024-03-06T21:00:15.999Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.755199796Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"client":"1","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56502024-03-06T21:00:15.999Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.755214975Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"client":"1","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56512024-03-06T21:00:15.999Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.755229414Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56522024-03-06T21:00:15.999Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.755736119Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"client":"2","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56532024-03-06T21:00:15.999Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.756176819Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"client":"2","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56542024-03-06T21:00:15.999Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.756228915Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"client":"2","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56552024-03-06T21:00:15.999Z{"msg":"[0]R flush_numbers[0..12]: [72, 72, 79, 82, 81, 82, 78, 61, 71, 82, 82, 78]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.756249064Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"":"downstairs","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56562024-03-06T21:00:15.999Z{"msg":"[0]R generation[0..12]: [46, 46, 50, 51, 50, 51, 49, 38, 45, 51, 51, 49]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.756265133Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"":"downstairs","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56572024-03-06T21:00:15.999Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.756288431Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"":"downstairs","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56582024-03-06T21:00:15.999Z{"msg":"[1]R flush_numbers[0..12]: [72, 72, 79, 82, 81, 82, 78, 61, 71, 82, 82, 78]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.7563034Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"":"downstairs","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56592024-03-06T21:00:15.999Z{"msg":"[1]R generation[0..12]: [46, 46, 50, 51, 50, 51, 49, 38, 45, 51, 51, 49]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.756326628Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"":"downstairs","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56602024-03-06T21:00:15.999Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.756341607Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"":"downstairs","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56612024-03-06T21:00:15.999Z{"msg":"Wait for a query_work_queue command to finish before sending IO
56622024-03-06T21:00:16.000Z[2]R flush_numbers[0..12]: [72, 72, 79, 82, 81, 82, 78, 61, 71, 82, 82, 78]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.756365356Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"":"downstairs","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56632024-03-06T21:00:16.000Z{"msg":"[2]R generation[0..12]: [46, 46, 50, 51, 50, 51, 49, 38, 45, 51, 51, 49]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.756380915Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"":"downstairs","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56642024-03-06T21:00:16.000Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.756402733Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"":"downstairs","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56652024-03-06T21:00:16.000Z{"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.756416952Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"":"downstairs","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56662024-03-06T21:00:16.000Z{"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.756430311Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"":"downstairs","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56672024-03-06T21:00:16.000Z{"msg":"Next flush: 83","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.756454069Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"":"downstairs","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56682024-03-06T21:00:16.000Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.756474688Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"":"downstairs","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56692024-03-06T21:00:16.001Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.756490547Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56702024-03-06T21:00:16.001Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.756503246Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"client":"0","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56712024-03-06T21:00:16.001Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.756517865Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"client":"1","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56722024-03-06T21:00:16.001Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.756532184Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"client":"2","":"downstairs","session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56732024-03-06T21:00:16.001Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.756545713Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56742024-03-06T21:00:16.001Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.756558282Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56752024-03-06T21:00:16.001Z{"msg":"ab959ffb-cb7c-45fa-a9af-09aa9ec0c5c7 is now active with session: 9615c55d-2169-496b-ada8-a040a8ad422d","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.756580381Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56762024-03-06T21:00:16.001Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.75659302Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347,"session_id":"9615c55d-2169-496b-ada8-a040a8ad422d"}
56772024-03-06T21:00:16.002Z{"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:14.756612899Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347}
56782024-03-06T21:00:16.015ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
56792024-03-06T21:00:16.015ZLoading write count information from file "/tmp/test_repair_verify.data"
56802024-03-06T21:00:16.015ZRun Repair workload
56812024-03-06T21:00:16.015Z01/30 Write block 547 len 16384 data: 7 6 6 5
56822024-03-06T21:00:16.015Z02/30 Write block 351 len 28672 data: 6 5 4 5 4 6 6
56832024-03-06T21:00:16.015Z03/30 Write block 279 len 40960 data: 5 4 5 5 5 7 9 11 11 11
56842024-03-06T21:00:16.015Z04/30 Read block 191 len 8192
56852024-03-06T21:00:16.015Z05/30 Read block 84 len 12288
56862024-03-06T21:00:16.016Z06/30 Write block 59 len 40960 data: 9 10 12 12 14 12 11 11 10 8
56872024-03-06T21:00:16.016Z07/30 Write block 402 len 8192 data: 2 2
56882024-03-06T21:00:16.016Z08/30 Write block 382 len 24576 data: 3 3 6 6 7 5
56892024-03-06T21:00:16.016Z09/30 Read block 362 len 28672
56902024-03-06T21:00:16.016Z10/30 Read block 436 len 12288
56912024-03-06T21:00:16.016Z11/30 Read block 558 len 28672
56922024-03-06T21:00:16.016Z12/30 Flush
56932024-03-06T21:00:16.016Z13/30 Write block 427 len 32768 data: 6 6 6 7 5 5 6 7
56942024-03-06T21:00:16.016Z14/30 Write block 11 len 40960 data: 6 5 5 4 5 4 3 3 4 5
56952024-03-06T21:00:16.016Z15/30 Read block 187 len 16384
56962024-03-06T21:00:16.025Z16/30 Write block 43 len 16384 data: 5 4 4 4
56972024-03-06T21:00:16.025Z17/30 Read block 555 len 28672
56982024-03-06T21:00:16.025Z18/30 Flush
56992024-03-06T21:00:16.025Z19/30 Write block 406 len 32768 data: 4 4 4 4 4 4 3 5
57002024-03-06T21:00:16.025Z20/30 Write block 403 len 24576 data: 3 2 4 5 5 5
57012024-03-06T21:00:16.025Z21/30 Write block 21 len 28672 data: 3 4 5 4 4 4 4
57022024-03-06T21:00:16.025Z22/30 Read block 339 len 8192
57032024-03-06T21:00:16.026Z23/30 Read block 570 len 4096
57042024-03-06T21:00:16.026Z24/30 Flush
57052024-03-06T21:00:16.038Z25/30 Write block 276 len 36864 data: 6 5 5 6 5 6 6 6 8
57062024-03-06T21:00:16.038Z26/30 Write block 583 len 28672 data: 3 5 6 6 7 6 6
57072024-03-06T21:00:16.038Z27/30 Write block 348 len 16384 data: 6 5 6 7
57082024-03-06T21:00:16.038Z28/30 Read block 533 len 16384
57092024-03-06T21:00:16.039Z29/30 Read block 454 len 24576
57102024-03-06T21:00:16.039Z30/30 Read block 575 len 24576
57112024-03-06T21:00:16.039Z----------------------------------------------------------------
57122024-03-06T21:00:16.039Z Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30
57132024-03-06T21:00:16.039ZStates: Active Active Active
57142024-03-06T21:00:16.039ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
57152024-03-06T21:00:16.040Z 1 Acked 1000 Write 4 Done Done Sent false
57162024-03-06T21:00:16.040Z 2 Acked 1001 Write 7 Done Done Sent false
57172024-03-06T21:00:16.040Z 3 Acked 1002 Write 10 Done Done Sent false
57182024-03-06T21:00:16.040Z 4 Acked 1003 Read 2 Done Done Sent false
57192024-03-06T21:00:16.040Z 5 Acked 1004 Read 3 Done Done Sent false
57202024-03-06T21:00:16.040Z 6 Acked 1005 Write 10 Done Done Sent false
57212024-03-06T21:00:16.040Z 7 Acked 1006 Write 2 Done Done Sent false
57222024-03-06T21:00:16.040Z 8 Acked 1007 Write 6 Done Done Sent false
57232024-03-06T21:00:16.040Z 9 Acked 1008 Read 7 Done Done Sent false
57242024-03-06T21:00:16.040Z 10 Acked 1009 Read 3 Done Done Sent false
57252024-03-06T21:00:16.041Z 11 Acked 1010 Read 7 Done Done Sent false
57262024-03-06T21:00:16.041Z 12 Acked 1011 Flush 0 Done Done Sent false
57272024-03-06T21:00:16.041Z 13 Acked 1012 Write 8 Done Done Sent false
57282024-03-06T21:00:16.041Z 14 Acked 1013 Write 10 Done Done Sent false
57292024-03-06T21:00:16.041Z 15 Acked 1014 Read 4 Done Done Sent false
57302024-03-06T21:00:16.041Z 16 Acked 1015 Write 4 Done Done Sent false
57312024-03-06T21:00:16.041Z 17 Acked 1016 Read 7 Done Done Sent false
57322024-03-06T21:00:16.041Z 18 Acked 1017 Flush 0 Done Done Sent false
57332024-03-06T21:00:16.041Z 19 Acked 1018 Write 8 Done Done Sent false
57342024-03-06T21:00:16.041Z 20 Acked 1019 Write 6 Done Done Sent false
57352024-03-06T21:00:16.042Z 21 Acked 1020 Write 7 Done Done Sent false
57362024-03-06T21:00:16.042Z 22 Acked 1021 Read 2 Done Done Sent false
57372024-03-06T21:00:16.042Z 23 Acked 1022 Read 1 Done Done Sent false
57382024-03-06T21:00:16.042Z 24 Acked 1023 Flush 0 Done Done Sent false
57392024-03-06T21:00:16.042Z 25 Acked 1024 Write 9 Done Done Sent false
57402024-03-06T21:00:16.042Z 26 Acked 1025 Write 7 Done Done Sent false
57412024-03-06T21:00:16.042Z 27 Acked 1026 Write 4 Done Done Sent false
57422024-03-06T21:00:16.042Z 28 Acked 1027 Read 4 Done Done Sent false
57432024-03-06T21:00:16.042Z 29 Acked 1028 Read 6 Done Done Sent false
57442024-03-06T21:00:16.042Z 30 Acked 1029 Read 6 Done Done Sent false
57452024-03-06T21:00:16.042Z STATES DS:0 DS:1 DS:2 TOTAL
57462024-03-06T21:00:16.042Z New 0 0 0 0
57472024-03-06T21:00:16.043Z Sent 0 0 30 30
57482024-03-06T21:00:16.043Z Done 30 30 0 60
57492024-03-06T21:00:16.043Z Skipped 0 0 0 0
57502024-03-06T21:00:16.043Z Error 0 0 0 0
57512024-03-06T21:00:16.043ZLast Flush: 1023 1023 0
57522024-03-06T21:00:16.043ZDownstairs last five completed:
57532024-03-06T21:00:16.043ZUpstairs last five completed: 30 29 28 27 26
57542024-03-06T21:00:16.043Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:00:14.794353998Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1347}
57552024-03-06T21:00:16.043ZWrote out file "/tmp/test_repair_verify.data"
57562024-03-06T21:00:16.486Z
57572024-03-06T21:00:16.486ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
57582024-03-06T21:00:16.500ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
57592024-03-06T21:00:16.500Z 0 000-019 52 52 46 84 84 72 F F F <---
57602024-03-06T21:00:16.500Z 1 020-039 52 52 46 85 85 72 F F F <---
57612024-03-06T21:00:16.500Z 2 040-059 52 52 50 84 84 79 F F F <---
57622024-03-06T21:00:16.500Z 3 060-079 52 52 51 83 83 82 F F F <---
57632024-03-06T21:00:16.500Z 4 080-099 50 50 50 81 81 81 F F F
57642024-03-06T21:00:16.500Z 5 100-119 51 51 51 82 82 82 F F F
57652024-03-06T21:00:16.501Z 6 120-139 49 49 49 78 78 78 F F F
57662024-03-06T21:00:16.501Z 7 140-159 38 38 38 61 61 61 F F F
57672024-03-06T21:00:16.501Z 8 160-179 45 45 45 71 71 71 F F F
57682024-03-06T21:00:16.501Z 9 180-199 51 51 51 82 82 82 F F F
57692024-03-06T21:00:16.501Z 10 200-219 51 51 51 82 82 82 F F F
57702024-03-06T21:00:16.501Z 11 220-239 49 49 49 78 78 78 F F F
57712024-03-06T21:00:16.501Z 12 240-259 47 47 47 73 73 73 F F F
57722024-03-06T21:00:16.501Z 13 260-279 52 52 47 83 83 73 T T F <---
57732024-03-06T21:00:16.501Z 14 280-299 52 52 50 83 83 80 T T F <---
57742024-03-06T21:00:16.501Z 15 300-319 50 50 50 80 80 80 F F F
57752024-03-06T21:00:16.501Z 16 320-339 32 32 32 47 47 47 F F F
57762024-03-06T21:00:16.501Z 17 340-359 52 52 50 83 83 80 T T F <---
57772024-03-06T21:00:16.501Z 18 360-379 51 51 51 82 82 82 F F F
57782024-03-06T21:00:16.501Z 19 380-399 52 52 42 83 83 67 F F F <---
57792024-03-06T21:00:16.502Z 20 400-419 52 52 49 85 85 78 F F F <---
57802024-03-06T21:00:16.502Z 21 420-439 52 52 51 84 84 82 F F F <---
57812024-03-06T21:00:16.502Z 22 440-459 45 45 45 71 71 71 F F F
57822024-03-06T21:00:16.502Z 23 460-479 50 50 50 80 80 80 F F F
57832024-03-06T21:00:16.502Z 24 480-499 42 42 42 67 67 67 F F F
57842024-03-06T21:00:16.502Z 25 500-519 42 42 42 67 67 67 F F F
57852024-03-06T21:00:16.502Z 26 520-539 51 51 51 82 82 82 F F F
57862024-03-06T21:00:16.502Z 27 540-559 52 52 48 83 83 75 F F F <---
57872024-03-06T21:00:16.502Z 28 560-579 45 45 45 71 71 71 F F F
57882024-03-06T21:00:16.502Z 29 580-599 46 46 46 72 72 72 T T F <---
57892024-03-06T21:00:16.502ZMax gen: 52, Max flush: 85
57902024-03-06T21:00:16.502ZError: Difference in extent metadata found!
57912024-03-06T21:00:16.502ZA Difference in extent metadata is expected here
57922024-03-06T21:00:16.502ZOn loop 26
57932024-03-06T21:00:16.503Z
57942024-03-06T21:00:16.503ZVerifying data now
57952024-03-06T21:00:20.714ZNo change after verify
57962024-03-06T21:00:20.714ZLoop: 26 Downstairs dump after verify (and repair):
57972024-03-06T21:00:20.733ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
57982024-03-06T21:00:20.733Z 0 000-019 52 52 52 84 84 84 F F F
57992024-03-06T21:00:20.733Z 1 020-039 52 52 52 85 85 85 F F F
58002024-03-06T21:00:20.733Z 2 040-059 52 52 52 84 84 84 F F F
58012024-03-06T21:00:20.734Z 3 060-079 52 52 52 83 83 83 F F F
58022024-03-06T21:00:20.734Z 4 080-099 50 50 50 81 81 81 F F F
58032024-03-06T21:00:20.734Z 5 100-119 51 51 51 82 82 82 F F F
58042024-03-06T21:00:20.734Z 6 120-139 49 49 49 78 78 78 F F F
58052024-03-06T21:00:20.734Z 7 140-159 38 38 38 61 61 61 F F F
58062024-03-06T21:00:20.734Z 8 160-179 45 45 45 71 71 71 F F F
58072024-03-06T21:00:20.734Z 9 180-199 51 51 51 82 82 82 F F F
58082024-03-06T21:00:20.735Z 10 200-219 51 51 51 82 82 82 F F F
58092024-03-06T21:00:20.735Z 11 220-239 49 49 49 78 78 78 F F F
58102024-03-06T21:00:20.735Z 12 240-259 47 47 47 73 73 73 F F F
58112024-03-06T21:00:20.735Z 13 260-279 53 53 53 86 86 86 F F F
58122024-03-06T21:00:20.735Z 14 280-299 53 53 53 86 86 86 F F F
58132024-03-06T21:00:20.735Z 15 300-319 50 50 50 80 80 80 F F F
58142024-03-06T21:00:20.735Z 16 320-339 32 32 32 47 47 47 F F F
58152024-03-06T21:00:20.748Z 17 340-359 53 53 53 86 86 86 F F F
58162024-03-06T21:00:20.748Z 18 360-379 51 51 51 82 82 82 F F F
58172024-03-06T21:00:20.748Z 19 380-399 52 52 52 83 83 83 F F F
58182024-03-06T21:00:20.748Z 20 400-419 52 52 52 85 85 85 F F F
58192024-03-06T21:00:20.748Z 21 420-439 52 52 52 84 84 84 F F F
58202024-03-06T21:00:20.748Z 22 440-459 45 45 45 71 71 71 F F F
58212024-03-06T21:00:20.749Z 23 460-479 50 50 50 80 80 80 F F F
58222024-03-06T21:00:20.749Z 24 480-499 42 42 42 67 67 67 F F F
58232024-03-06T21:00:20.749Z 25 500-519 42 42 42 67 67 67 F F F
58242024-03-06T21:00:20.749Z 26 520-539 51 51 51 82 82 82 F F F
58252024-03-06T21:00:20.749Z 27 540-559 52 52 52 83 83 83 F F F
58262024-03-06T21:00:20.749Z 28 560-579 45 45 45 71 71 71 F F F
58272024-03-06T21:00:20.749Z 29 580-599 53 53 53 86 86 86 F F F
58282024-03-06T21:00:20.749ZMax gen: 53, Max flush: 86
58292024-03-06T21:00:20.750Z
58302024-03-06T21:00:20.750ZBegin loop 27
58312024-03-06T21:00:20.750ZDownstairs to restart: 1
58322024-03-06T21:00:20.768Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.517919784Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355}
58332024-03-06T21:00:20.769Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.518198085Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355}
58342024-03-06T21:00:20.769Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
58352024-03-06T21:00:20.769Z,"time":"2024-03-06T21:00:19.518230933Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355}
58362024-03-06T21:00:20.769Z{"msg":"Crucible b54895a1-d7fe-4d0b-be25-e346e5853b0e has session id: d60dec59-866c-469d-b397-826a47565553","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.51826659Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"session_id":"d60dec59-866c-469d-b397-826a47565553"}
58372024-03-06T21:00:20.769Z{"msg":"Upstairs opts: Upstairs UUID: b54895a1-d7fe-4d0b-be25-e346e5853b0e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.518323656Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"session_id":"d60dec59-866c-469d-b397-826a47565553"}
58382024-03-06T21:00:20.769Z{"msg":"Crucible stats registered with UUID: b54895a1-d7fe-4d0b-be25-e346e5853b0e","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.518373903Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"session_id":"d60dec59-866c-469d-b397-826a47565553"}
58392024-03-06T21:00:20.769Z{"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.518399941Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355}
58402024-03-06T21:00:20.770Z{"msg":"b54895a1-d7fe-4d0b-be25-e346e5853b0e active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.518520803Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"session_id":"d60dec59-866c-469d-b397-826a47565553"}
58412024-03-06T21:00:20.770Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.51855951Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"client":"0","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58422024-03-06T21:00:20.770Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.518580049Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"client":"0","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58432024-03-06T21:00:20.770Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.518614896Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"client":"1","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58442024-03-06T21:00:20.770Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.518659053Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"client":"1","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58452024-03-06T21:00:20.770Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.51870128Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"client":"2","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58462024-03-06T21:00:20.770Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.518727458Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"client":"2","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58472024-03-06T21:00:20.771Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.518751697Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"":"io task","client":"2","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58482024-03-06T21:00:20.771Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.518798043Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"":"io task","client":"1","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58492024-03-06T21:00:20.771Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.518828341Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"":"io task","client":"0","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58502024-03-06T21:00:20.771Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.518958563Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"":"io task","client":"2","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58512024-03-06T21:00:20.771Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.519020868Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"":"io task","client":"0","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58522024-03-06T21:00:20.771Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.519123881Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"":"io task","client":"1","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58532024-03-06T21:00:20.787Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.519915577Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"client":"2","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58542024-03-06T21:00:20.787Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.519956594Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"client":"2","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58552024-03-06T21:00:20.788Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.519980412Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"client":"0","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58562024-03-06T21:00:20.788Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.520000391Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"client":"0","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58572024-03-06T21:00:20.788Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.520529484Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"client":"0","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58582024-03-06T21:00:20.788Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.520667015Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"client":"2","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58592024-03-06T21:00:20.788Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.52101995Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"client":"1","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58602024-03-06T21:00:20.788Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.521094895Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"client":"1","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58612024-03-06T21:00:20.788Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.521203108Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"client":"0","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58622024-03-06T21:00:20.788Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.521237395Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"client":"0","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58632024-03-06T21:00:20.789Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.521252864Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"session_id":"d60dec59-866c-469d-b397-826a47565553"}
58642024-03-06T21:00:20.789Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.521265933Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"client":"2","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58652024-03-06T21:00:20.789Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.521286292Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"client":"2","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58662024-03-06T21:00:20.789Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.521300201Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"session_id":"d60dec59-866c-469d-b397-826a47565553"}
58672024-03-06T21:00:20.789Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.521658966Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"client":"1","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58682024-03-06T21:00:20.789Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.522096996Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"client":"1","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58692024-03-06T21:00:20.789Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.522145253Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"client":"1","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58702024-03-06T21:00:20.789Z{"msg":"[0]R flush_numbers[0..12]: [84, 85, 84, 83, 81, 82, 78, 61, 71, 82, 82, 78]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.522163332Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"":"downstairs","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58712024-03-06T21:00:20.790Z{"msg":"[0]R generation[0..12]: [52, 52, 52, 52, 50, 51, 49, 38, 45, 51, 51, 49]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.52218651Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"":"downstairs","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58722024-03-06T21:00:20.790Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.522201369Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"":"downstairs","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58732024-03-06T21:00:20.790Z{"msg":"[1]R flush_numbers[0..12]: [84, 85, 84, 83, 81, 82, 78, 61, 71, 82, 82, 78]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.522222698Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"":"downstairs","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58742024-03-06T21:00:20.790Z{"msg":"[1]R generation[0..12]: [52, 52, 52, 52, 50, 51, 49, 38, 45, 51, 51, 49]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.522236637Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"":"downstairs","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58752024-03-06T21:00:20.790Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.522257695Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"":"downstairs","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58762024-03-06T21:00:20.790ZWait for a query_work_queue command to finish before sending IO
58772024-03-06T21:00:20.790Z{"msg":"[2]R flush_numbers[0..12]: [84, 85, 84, 83, 81, 82, 78, 61, 71, 82, 82, 78]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.522277894Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"":"downstairs","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58782024-03-06T21:00:20.791Z{"msg":"[2]R generation[0..12]: [52, 52, 52, 52, 50, 51, 49, 38, 45, 51, 51, 49]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.522292233Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"":"downstairs","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58792024-03-06T21:00:20.791Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.522307562Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"":"downstairs","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58802024-03-06T21:00:20.791Z{"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.522321361Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"":"downstairs","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58812024-03-06T21:00:20.791Z{"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.522342779Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"":"downstairs","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58822024-03-06T21:00:20.791Z{"msg":"Next flush: 87","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.522356288Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"":"downstairs","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58832024-03-06T21:00:20.791Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.522369367Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"":"downstairs","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58842024-03-06T21:00:20.791Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.522389246Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"session_id":"d60dec59-866c-469d-b397-826a47565553"}
58852024-03-06T21:00:20.791Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.522401585Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"client":"0","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58862024-03-06T21:00:20.791Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.522415154Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"client":"1","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58872024-03-06T21:00:20.791Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.522435073Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"client":"2","":"downstairs","session_id":"d60dec59-866c-469d-b397-826a47565553"}
58882024-03-06T21:00:20.791Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.522448572Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"session_id":"d60dec59-866c-469d-b397-826a47565553"}
58892024-03-06T21:00:20.791Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.522461931Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"session_id":"d60dec59-866c-469d-b397-826a47565553"}
58902024-03-06T21:00:20.792Z{"msg":"b54895a1-d7fe-4d0b-be25-e346e5853b0e is now active with session: d60dec59-866c-469d-b397-826a47565553","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.52247491Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"session_id":"d60dec59-866c-469d-b397-826a47565553"}
58912024-03-06T21:00:20.792Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.522486899Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355,"session_id":"d60dec59-866c-469d-b397-826a47565553"}
58922024-03-06T21:00:20.792Z{"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:19.522499548Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355}
58932024-03-06T21:00:20.792ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
58942024-03-06T21:00:20.792ZLoading write count information from file "/tmp/test_repair_verify.data"
58952024-03-06T21:00:20.792ZRun Repair workload
58962024-03-06T21:00:20.793Z01/30 Write block 515 len 24576 data: 5 6 6 5 5 3
58972024-03-06T21:00:20.793Z02/30 Read block 510 len 32768
58982024-03-06T21:00:20.793Z03/30 Read block 513 len 28672
58992024-03-06T21:00:20.793Z04/30 Read block 465 len 8192
59002024-03-06T21:00:20.793Z05/30 Write block 557 len 16384 data: 3 3 3 5
59012024-03-06T21:00:20.793Z06/30 Write block 467 len 8192 data: 7 6
59022024-03-06T21:00:20.793Z07/30 Read block 140 len 36864
59032024-03-06T21:00:20.793Z08/30 Read block 427 len 36864
59042024-03-06T21:00:20.794Z09/30 Write block 444 len 36864 data: 4 3 4 2 6 6 7 8 8
59052024-03-06T21:00:20.794Z10/30 Read block 555 len 8192
59062024-03-06T21:00:20.794Z11/30 Read block 114 len 24576
59072024-03-06T21:00:20.794Z12/30 Read block 279 len 32768
59082024-03-06T21:00:20.794Z13/30 Read block 175 len 12288
59092024-03-06T21:00:20.794Z14/30 Flush
59102024-03-06T21:00:20.794Z15/30 Flush
59112024-03-06T21:00:20.794Z16/30 Write block 57 len 12288 data: 6 8 10
59122024-03-06T21:00:20.807Z17/30 Flush
59132024-03-06T21:00:20.807Z18/30 Flush
59142024-03-06T21:00:20.807Z19/30 Write block 587 len 20480 data: 8 7 7 7 6
59152024-03-06T21:00:20.807Z20/30 Read block 72 len 32768
59162024-03-06T21:00:20.807Z21/30 Write block 389 len 24576 data: 4 5 5 4 4 3
59172024-03-06T21:00:20.807Z22/30 Read block 258 len 8192
59182024-03-06T21:00:20.807Z23/30 Write block 397 len 8192 data: 4 4
59192024-03-06T21:00:20.808Z24/30 Read block 94 len 28672
59202024-03-06T21:00:20.808Z25/30 Flush
59212024-03-06T21:00:20.808Z26/30 Write block 46 len 32768 data: 5 6 5 6 4 4 6 5
59222024-03-06T21:00:20.808Z27/30 Write block 216 len 16384 data: 6 7 6 4
59232024-03-06T21:00:20.808Z28/30 Write block 127 len 12288 data: 5 6 6
59242024-03-06T21:00:20.808Z29/30 Write block 125 len 36864 data: 5 4 6 7 7 6 5 6 4
59252024-03-06T21:00:20.809Z30/30 Write block 360 len 40960 data: 7 8 6 9 8 6 5 6 4 6
59262024-03-06T21:00:20.809Z----------------------------------------------------------------
59272024-03-06T21:00:20.809Z Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:30
59282024-03-06T21:00:20.809ZStates: Active Active Active
59292024-03-06T21:00:20.809ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
59302024-03-06T21:00:20.809Z 1 Acked 1000 Write 6 Done Done Done false
59312024-03-06T21:00:20.809Z 2 Acked 1001 Read 8 Done Done Done false
59322024-03-06T21:00:20.809Z 3 Acked 1002 Read 7 Done Done Done false
59332024-03-06T21:00:20.810Z 4 Acked 1003 Read 2 Done Done Done false
59342024-03-06T21:00:20.810Z 5 Acked 1004 Write 4 Done Done Done false
59352024-03-06T21:00:20.810Z 6 Acked 1005 Write 2 Done Sent Done false
59362024-03-06T21:00:20.810Z 7 Acked 1006 Read 9 Done Sent Done false
59372024-03-06T21:00:20.810Z 8 Acked 1007 Read 9 Done Sent Done false
59382024-03-06T21:00:20.810Z 9 Acked 1008 Write 9 Done Sent Done false
59392024-03-06T21:00:20.811Z 10 Acked 1009 Read 2 Done Sent Done false
59402024-03-06T21:00:20.811Z 11 Acked 1010 Read 6 Done Sent Done false
59412024-03-06T21:00:20.811Z 12 Acked 1011 Read 8 Done Sent Done false
59422024-03-06T21:00:20.811Z 13 Acked 1012 Read 3 Done Sent Done false
59432024-03-06T21:00:20.811Z 14 Acked 1013 Flush 0 Done Sent Done false
59442024-03-06T21:00:20.811Z 15 Acked 1014 Flush 0 Done Sent Done false
59452024-03-06T21:00:20.811Z 16 Acked 1015 Write 3 Done Sent Done false
59462024-03-06T21:00:20.811Z 17 Acked 1016 Flush 0 Done Sent Done false
59472024-03-06T21:00:20.811Z 18 Acked 1017 Flush 0 Done Sent Done false
59482024-03-06T21:00:20.811Z 19 Acked 1018 Write 5 Done Sent Done false
59492024-03-06T21:00:20.811Z 20 Acked 1019 Read 8 Done Sent Done false
59502024-03-06T21:00:20.811Z 21 Acked 1020 Write 6 Done Sent Done false
59512024-03-06T21:00:20.812Z 22 Acked 1021 Read 2 Done Sent Done false
59522024-03-06T21:00:20.812Z 23 Acked 1022 Write 2 Done Sent Done false
59532024-03-06T21:00:20.812Z 24 Acked 1023 Read 7 Done Sent Done false
59542024-03-06T21:00:20.812Z 25 Acked 1024 Flush 0 Done Sent Done false
59552024-03-06T21:00:20.812Z 26 Acked 1025 Write 8 Done Sent Done false
59562024-03-06T21:00:20.812Z 27 Acked 1026 Write 4 Done Sent Done false
59572024-03-06T21:00:20.812Z 28 Acked 1027 Write 3 Done Sent Done false
59582024-03-06T21:00:20.813Z 29 Acked 1028 Write 9 Done Sent Done false
59592024-03-06T21:00:20.813Z 30 Acked 1029 Write 10 Sent Sent Sent false
59602024-03-06T21:00:20.813Z STATES DS:0 DS:1 DS:2 TOTAL
59612024-03-06T21:00:20.813Z New 0 0 0 0
59622024-03-06T21:00:20.813Z Sent 1 25 1 27
59632024-03-06T21:00:20.813Z Done 29 5 29 63
59642024-03-06T21:00:20.813Z Skipped 0 0 0 0
59652024-03-06T21:00:20.813Z Error 0 0 0 0
59662024-03-06T21:00:20.813ZLast Flush: 1024 0 1024
59672024-03-06T21:00:20.813ZDownstairs last five completed:
59682024-03-06T21:00:20.813ZUpstairs last five completed: 30 29 28 27 26
59692024-03-06T21:00:20.813Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:00:19.558379716Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1355}
59702024-03-06T21:00:20.814ZWrote out file "/tmp/test_repair_verify.data"
59712024-03-06T21:00:20.814Z
59722024-03-06T21:00:20.824ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
59732024-03-06T21:00:21.266ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
59742024-03-06T21:00:21.266Z 0 000-019 52 52 52 84 84 84 F F F
59752024-03-06T21:00:21.266Z 1 020-039 52 52 52 85 85 85 F F F
59762024-03-06T21:00:21.266Z 2 040-059 54 52 54 89 84 89 T F T <---
59772024-03-06T21:00:21.267Z 3 060-079 52 52 52 83 83 83 F F F
59782024-03-06T21:00:21.267Z 4 080-099 50 50 50 81 81 81 F F F
59792024-03-06T21:00:21.267Z 5 100-119 51 51 51 82 82 82 F F F
59802024-03-06T21:00:21.267Z 6 120-139 49 49 49 78 78 78 T F T <---
59812024-03-06T21:00:21.267Z 7 140-159 38 38 38 61 61 61 F F F
59822024-03-06T21:00:21.267Z 8 160-179 45 45 45 71 71 71 F F F
59832024-03-06T21:00:21.267Z 9 180-199 51 51 51 82 82 82 F F F
59842024-03-06T21:00:21.267Z 10 200-219 51 51 51 82 82 82 T F T <---
59852024-03-06T21:00:21.268Z 11 220-239 49 49 49 78 78 78 F F F
59862024-03-06T21:00:21.268Z 12 240-259 47 47 47 73 73 73 F F F
59872024-03-06T21:00:21.268Z 13 260-279 53 53 53 86 86 86 F F F
59882024-03-06T21:00:21.268Z 14 280-299 53 53 53 86 86 86 F F F
59892024-03-06T21:00:21.268Z 15 300-319 50 50 50 80 80 80 F F F
59902024-03-06T21:00:21.268Z 16 320-339 32 32 32 47 47 47 F F F
59912024-03-06T21:00:21.269Z 17 340-359 53 53 53 86 86 86 F F F
59922024-03-06T21:00:21.269Z 18 360-379 51 51 51 82 82 82 T F F <---
59932024-03-06T21:00:21.269Z 19 380-399 54 52 54 91 83 91 F F F <---
59942024-03-06T21:00:21.269Z 20 400-419 52 52 52 85 85 85 F F F
59952024-03-06T21:00:21.269Z 21 420-439 52 52 52 84 84 84 F F F
59962024-03-06T21:00:21.269Z 22 440-459 54 45 54 87 71 87 F F F <---
59972024-03-06T21:00:21.269Z 23 460-479 54 50 54 87 80 87 F F F <---
59982024-03-06T21:00:21.270Z 24 480-499 42 42 42 67 67 67 F F F
59992024-03-06T21:00:21.270Z 25 500-519 54 42 54 87 67 87 F T F <---
60002024-03-06T21:00:21.270Z 26 520-539 54 51 54 87 82 87 F T F <---
60012024-03-06T21:00:21.270Z 27 540-559 54 52 54 87 83 87 F T F <---
60022024-03-06T21:00:21.270Z 28 560-579 54 45 54 87 71 87 F T F <---
60032024-03-06T21:00:21.270Z 29 580-599 54 53 54 91 86 91 F F F <---
60042024-03-06T21:00:21.270ZMax gen: 54, Max flush: 91
60052024-03-06T21:00:21.270ZError: Difference in extent metadata found!
60062024-03-06T21:00:21.271ZA Difference in extent metadata is expected here
60072024-03-06T21:00:21.271ZOn loop 27
60082024-03-06T21:00:21.271Z
60092024-03-06T21:00:21.271ZVerifying data now
60102024-03-06T21:00:26.479ZNo change after verify
60112024-03-06T21:00:26.479ZLoop: 27 Downstairs dump after verify (and repair):
60122024-03-06T21:00:26.503ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
60132024-03-06T21:00:26.503Z 0 000-019 52 52 52 84 84 84 F F F
60142024-03-06T21:00:26.504Z 1 020-039 52 52 52 85 85 85 F F F
60152024-03-06T21:00:26.504Z 2 040-059 55 55 55 92 92 92 F F F
60162024-03-06T21:00:26.504Z 3 060-079 52 52 52 83 83 83 F F F
60172024-03-06T21:00:26.504Z 4 080-099 50 50 50 81 81 81 F F F
60182024-03-06T21:00:26.504Z 5 100-119 51 51 51 82 82 82 F F F
60192024-03-06T21:00:26.504Z 6 120-139 55 55 55 92 92 92 F F F
60202024-03-06T21:00:26.504Z 7 140-159 38 38 38 61 61 61 F F F
60212024-03-06T21:00:26.504Z 8 160-179 45 45 45 71 71 71 F F F
60222024-03-06T21:00:26.505Z 9 180-199 51 51 51 82 82 82 F F F
60232024-03-06T21:00:26.505Z 10 200-219 55 55 55 92 92 92 F F F
60242024-03-06T21:00:26.505Z 11 220-239 49 49 49 78 78 78 F F F
60252024-03-06T21:00:26.505Z 12 240-259 47 47 47 73 73 73 F F F
60262024-03-06T21:00:26.505Z 13 260-279 53 53 53 86 86 86 F F F
60272024-03-06T21:00:26.505Z 14 280-299 53 53 53 86 86 86 F F F
60282024-03-06T21:00:26.505Z 15 300-319 50 50 50 80 80 80 F F F
60292024-03-06T21:00:26.505Z 16 320-339 32 32 32 47 47 47 F F F
60302024-03-06T21:00:26.505Z 17 340-359 53 53 53 86 86 86 F F F
60312024-03-06T21:00:26.505Z 18 360-379 55 55 55 92 92 92 F F F
60322024-03-06T21:00:26.505Z 19 380-399 54 54 54 91 91 91 F F F
60332024-03-06T21:00:26.505Z 20 400-419 52 52 52 85 85 85 F F F
60342024-03-06T21:00:26.505Z 21 420-439 52 52 52 84 84 84 F F F
60352024-03-06T21:00:26.505Z 22 440-459 54 54 54 87 87 87 F F F
60362024-03-06T21:00:26.505Z 23 460-479 54 54 54 87 87 87 F F F
60372024-03-06T21:00:26.505Z 24 480-499 42 42 42 67 67 67 F F F
60382024-03-06T21:00:26.505Z 25 500-519 54 54 54 87 87 87 F F F
60392024-03-06T21:00:26.505Z 26 520-539 54 54 54 87 87 87 F F F
60402024-03-06T21:00:26.505Z 27 540-559 54 54 54 87 87 87 F F F
60412024-03-06T21:00:26.505Z 28 560-579 54 54 54 87 87 87 F F F
60422024-03-06T21:00:26.506Z 29 580-599 54 54 54 91 91 91 F F F
60432024-03-06T21:00:26.506ZMax gen: 55, Max flush: 92
60442024-03-06T21:00:26.506Z
60452024-03-06T21:00:26.506ZBegin loop 28
60462024-03-06T21:00:26.506ZDownstairs to restart: 0
60472024-03-06T21:00:26.634Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.269763884Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363}
60482024-03-06T21:00:26.634Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
60492024-03-06T21:00:26.634Z,"time":"2024-03-06T21:00:25.270078962Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363}
60502024-03-06T21:00:26.635Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.270129359Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363}
60512024-03-06T21:00:26.635Z{"msg":"Crucible a98f5d4e-47f1-46f1-b26e-ea546e723dd8 has session id: cb807c6f-f98c-4f8c-b45c-574aa55bda35","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.270158037Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60522024-03-06T21:00:26.635Z{"msg":"Upstairs opts: Upstairs UUID: a98f5d4e-47f1-46f1-b26e-ea546e723dd8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.270213453Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60532024-03-06T21:00:26.635Z{"msg":"Crucible stats registered with UUID: a98f5d4e-47f1-46f1-b26e-ea546e723dd8","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.270255Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60542024-03-06T21:00:26.635Z{"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.270283508Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363}
60552024-03-06T21:00:26.635Z{"msg":"a98f5d4e-47f1-46f1-b26e-ea546e723dd8 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.270315316Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60562024-03-06T21:00:26.635Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.270352784Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"client":"0","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60572024-03-06T21:00:26.635Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.270385401Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"client":"0","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60582024-03-06T21:00:26.635Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.270424609Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"client":"1","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60592024-03-06T21:00:26.635Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.270454267Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"client":"1","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60602024-03-06T21:00:26.636Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.270487924Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"client":"2","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60612024-03-06T21:00:26.636Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.270517212Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"client":"2","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60622024-03-06T21:00:26.636Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.27054665Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"":"io task","client":"1","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60632024-03-06T21:00:26.636Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.270598127Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"":"io task","client":"2","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60642024-03-06T21:00:26.636Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.270636474Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"":"io task","client":"0","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60652024-03-06T21:00:26.636Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.270668502Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"":"io task","client":"1","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60662024-03-06T21:00:26.636Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.27069982Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"":"io task","client":"0","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60672024-03-06T21:00:26.636Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.270721788Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"":"io task","client":"2","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60682024-03-06T21:00:26.637Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.2714212Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"client":"1","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60692024-03-06T21:00:26.637Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.271465897Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"client":"1","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60702024-03-06T21:00:26.637Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.271483946Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"client":"2","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60712024-03-06T21:00:26.637Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.271506424Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"client":"2","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60722024-03-06T21:00:26.637Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.272065656Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"client":"2","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60732024-03-06T21:00:26.637Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.272223635Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"client":"1","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60742024-03-06T21:00:26.637Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.27229688Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"client":"0","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60752024-03-06T21:00:26.637Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.272326758Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"client":"0","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60762024-03-06T21:00:26.637Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.272620867Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"client":"2","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60772024-03-06T21:00:26.638Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.272689793Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"client":"2","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60782024-03-06T21:00:26.638Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.272739299Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60792024-03-06T21:00:26.638Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.272776987Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"client":"1","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60802024-03-06T21:00:26.638Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.272803855Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"client":"1","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60812024-03-06T21:00:26.638Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.272819174Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60822024-03-06T21:00:26.638Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.2730113Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"client":"0","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60832024-03-06T21:00:26.638Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.27345281Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"client":"0","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60842024-03-06T21:00:26.638Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.273508936Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"client":"0","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60852024-03-06T21:00:26.639Z{"msg":"[0]R flush_numbers[0..12]: [84, 85, 92, 83, 81, 82, 92, 61, 71, 82, 92, 78]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.273536304Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"":"downstairs","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60862024-03-06T21:00:26.639Z{"msg":"[0]R generation[0..12]: [52, 52, 55, 52, 50, 51, 55, 38, 45, 51, 55, 49]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.273570052Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"":"downstairs","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60872024-03-06T21:00:26.639Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.27360427Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"":"downstairs","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60882024-03-06T21:00:26.639Z{"msg":"[1]R flush_numbers[0..12]: [84, 85, 92, 83, 81, 82, 92, 61, 71, 82, 92, 78]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
60892024-03-06T21:00:26.639Z,"time":"2024-03-06T21:00:25.273637697Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"":"downstairs","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60902024-03-06T21:00:26.639Z{"msg":"[1]R generation[0..12]: [52, 52, 55, 52, 50, 51, 55, 38, 45, 51, 55, 49]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.273664635Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"":"downstairs","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60912024-03-06T21:00:26.640Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.273689654Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"":"downstairs","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60922024-03-06T21:00:26.640Z{"msg":"[2]R flush_numbers[0..12]: [84, 85, 92, 83, 81, 82, 92, 61, 71, 82, 92, 78]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.273721931Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"":"downstairs","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60932024-03-06T21:00:26.640Z{"msg":"[2]R generation[0..12]: [52, 52, 55, 52, 50, 51, 55, 38, 45, 51, 55, 49]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.2737463Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"":"downstairs","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60942024-03-06T21:00:26.640Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.273774118Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"":"downstairs","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60952024-03-06T21:00:26.640Z{"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.273796776Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"":"downstairs","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60962024-03-06T21:00:26.640Z{"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.273823075Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"":"downstairs","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60972024-03-06T21:00:26.640Z{"msg":"Next flush: 93","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.273845673Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"":"downstairs","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60982024-03-06T21:00:26.640Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.273866392Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"":"downstairs","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
60992024-03-06T21:00:26.641Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.27388856Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
61002024-03-06T21:00:26.641Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.273908579Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"client":"0","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
61012024-03-06T21:00:26.641Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.273930307Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"client":"1","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
61022024-03-06T21:00:26.641Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.273961355Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"client":"2","":"downstairs","session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
61032024-03-06T21:00:26.641Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.273984583Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"session_id":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
61042024-03-06T21:00:26.641Zcb807c6f-f98c-4f8c-b45c-574aa55bda35"}
61052024-03-06T21:00:26.642Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.27403686Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
61062024-03-06T21:00:26.642Z{"msg":"a98f5d4e-47f1-46f1-b26e-ea546e723dd8 is now active with session: cb807c6f-f98c-4f8c-b45c-574aa55bda35","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.274067538Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
61072024-03-06T21:00:26.642Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.274089066Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363,"session_id":"cb807c6f-f98c-4f8c-b45c-574aa55bda35"}
61082024-03-06T21:00:26.642Z{"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:25.274117954Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363}
61092024-03-06T21:00:26.642ZLoading write count information from file "/tmp/test_repair_verify.data"
61102024-03-06T21:00:26.642ZRun Repair workload
61112024-03-06T21:00:26.642Z01/30 Write block 125 len 24576 data: 6 5 7 8 8 7
61122024-03-06T21:00:26.642Z02/30 Read block 224 len 12288
61132024-03-06T21:00:26.642Z03/30 Read block 403 len 8192
61142024-03-06T21:00:26.642Z04/30 Read block 215 len 40960
61152024-03-06T21:00:26.643Z05/30 Write block 353 len 4096 data: 5
61162024-03-06T21:00:26.643Z06/30 Read block 305 len 36864
61172024-03-06T21:00:26.659Z07/30 Write block 338 len 36864 data: 5 5 6 6 6 7 7 6 4
61182024-03-06T21:00:26.659Z08/30 Read block 353 len 28672
61192024-03-06T21:00:26.660Z09/30 Write block 318 len 36864 data: 4 4 3 3 3 2 2 2 2
61202024-03-06T21:00:26.660Z10/30 Write block 263 len 24576 data: 4 4 5 5 5 6
61212024-03-06T21:00:26.660Z11/30 Flush
61222024-03-06T21:00:26.660Z12/30 Write block 457 len 20480 data: 5 5 5 6 9
61232024-03-06T21:00:26.660Z13/30 Read block 455 len 32768
61242024-03-06T21:00:26.660Z14/30 Read block 451 len 24576
61252024-03-06T21:00:26.660Z15/30 Write block 288 len 20480 data: 12 12 12 8 5
61262024-03-06T21:00:26.660Z16/30 Read block 282 len 4096
61272024-03-06T21:00:26.660Z17/30 Flush
61282024-03-06T21:00:26.661Z18/30 Write block 419 len 12288 data: 5 6 9
61292024-03-06T21:00:26.661Z19/30 Flush
61302024-03-06T21:00:26.661Z20/30 Flush
61312024-03-06T21:00:26.661Z21/30 Flush
61322024-03-06T21:00:26.661Z22/30 Write block 258 len 16384 data: 3 3 2 3
61332024-03-06T21:00:26.661Z23/30 Read block 386 len 8192
61342024-03-06T21:00:26.661Z24/30 Read block 77 len 12288
61352024-03-06T21:00:26.661Z25/30 Write block 111 len 28672 data: 6 3 2 2 2 2 3
61362024-03-06T21:00:26.662Z26/30 Write block 155 len 36864 data: 3 4 5 6 5 5 4 4 4
61372024-03-06T21:00:26.662Z27/30 Write block 333 len 20480 data: 5 4 4 3 3
61382024-03-06T21:00:26.662Z28/30 Read block 189 len 8192
61392024-03-06T21:00:26.662Z29/30 Write block 485 len 36864 data: 4 5 6 7 8 6 7 8 9
61402024-03-06T21:00:26.662Z30/30 Write block 370 len 20480 data: 7 6 7 6 6
61412024-03-06T21:00:26.662Z----------------------------------------------------------------
61422024-03-06T21:00:26.662Z Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30
61432024-03-06T21:00:26.662ZStates: Active Active Active
61442024-03-06T21:00:26.663ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
61452024-03-06T21:00:26.663Z 1 Acked 1000 Write 6 Done Done Done false
61462024-03-06T21:00:26.663Z 2 Acked 1001 Read 3 Done Done Done false
61472024-03-06T21:00:26.663Z 3 Acked 1002 Read 2 Sent Done Done false
61482024-03-06T21:00:26.663Z 4 Acked 1003 Read 10 Sent Done Done false
61492024-03-06T21:00:26.663Z 5 Acked 1004 Write 1 Sent Done Done false
61502024-03-06T21:00:26.663Z 6 Acked 1005 Read 9 Sent Done Done false
61512024-03-06T21:00:26.663Z 7 Acked 1006 Write 9 Sent Done Done false
61522024-03-06T21:00:26.664Z 8 Acked 1007 Read 7 Sent Done Done false
61532024-03-06T21:00:26.664Z 9 Acked 1008 Write 9 Sent Done Done false
61542024-03-06T21:00:26.664Z 10 Acked 1009 Write 6 Sent Done Done false
61552024-03-06T21:00:26.664Z 11 Acked 1010 Flush 0 Sent Done Done false
61562024-03-06T21:00:26.664Z 12 Acked 1011 Write 5 Sent Done Done false
61572024-03-06T21:00:26.664Z 13 Acked 1012 Read 8 Sent Done Done false
61582024-03-06T21:00:26.664Z 14 Acked 1013 Read 6 Sent Done Done false
61592024-03-06T21:00:26.665Z 15 Acked 1014 Write 5 Sent Done Done false
61602024-03-06T21:00:26.665Z 16 Acked 1015 Read 1 Sent Done Done false
61612024-03-06T21:00:26.665Z 17 Acked 1016 Flush 0 Sent Done Done false
61622024-03-06T21:00:26.665Z 18 Acked 1017 Write 3 Sent Done Done false
61632024-03-06T21:00:26.665Z 19 Acked 1018 Flush 0 Sent Done Done false
61642024-03-06T21:00:26.665Z 20 Acked 1019 Flush 0 Sent Done Done false
61652024-03-06T21:00:26.665Z 21 Acked 1020 Flush 0 Sent Done Done false
61662024-03-06T21:00:26.665Z 22 Acked 1021 Write 4 Sent Done Done false
61672024-03-06T21:00:26.665Z 23 Acked 1022 Read 2 Sent Done Done false
61682024-03-06T21:00:26.666Z 24 Acked 1023 Read 3 Sent Done Done false
61692024-03-06T21:00:26.666Z 25 Acked 1024 Write 7 Sent Done Done false
61702024-03-06T21:00:26.666Z 26 Acked 1025 Write 9 Sent Done Done false
61712024-03-06T21:00:26.666Z 27 Acked 1026 Write 5 Sent Done Done false
61722024-03-06T21:00:26.666Z 28 Acked 1027 Read 2 Sent Done Done false
61732024-03-06T21:00:26.666Z 29 Acked 1028 Write 9 Sent Sent Sent false
61742024-03-06T21:00:26.666Z 30 Acked 1029 Write 5 Sent Sent Sent false
61752024-03-06T21:00:26.666Z STATES DS:0 DS:1 DS:2 TOTAL
61762024-03-06T21:00:26.666Z New 0 0 0 0
61772024-03-06T21:00:26.667Z Sent 28 2 2 32
61782024-03-06T21:00:26.667Z Done 2 28 28 58
61792024-03-06T21:00:26.667Z Skipped 0 0 0 0
61802024-03-06T21:00:26.667Z Error 0 0 0 0
61812024-03-06T21:00:26.667ZLast Flush: 0 1020 1020
61822024-03-06T21:00:26.667ZDownstairs last five completed:
61832024-03-06T21:00:26.667ZUpstairs last five completed: 30 29 28 27 26
61842024-03-06T21:00:26.667Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:00:25.308930445Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1363}
61852024-03-06T21:00:26.668ZWrote out file "/tmp/test_repair_verify.data"
61862024-03-06T21:00:26.668Z
61872024-03-06T21:00:26.668ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
61882024-03-06T21:00:26.668ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
61892024-03-06T21:00:26.668Z 0 000-019 52 52 52 84 84 84 F F F
61902024-03-06T21:00:26.668Z 1 020-039 52 52 52 85 85 85 F F F
61912024-03-06T21:00:26.668Z 2 040-059 55 55 55 92 92 92 F F F
61922024-03-06T21:00:26.668Z 3 060-079 52 52 52 83 83 83 F F F
61932024-03-06T21:00:26.668Z 4 080-099 50 50 50 81 81 81 F F F
61942024-03-06T21:00:26.669Z 5 100-119 51 51 51 82 82 82 F T T <---
61952024-03-06T21:00:26.669Z 6 120-139 55 56 56 92 93 93 T F F <---
61962024-03-06T21:00:26.669Z 7 140-159 38 38 38 61 61 61 F T T <---
61972024-03-06T21:00:26.669Z 8 160-179 45 45 45 71 71 71 F T T <---
61982024-03-06T21:00:26.669Z 9 180-199 51 51 51 82 82 82 F F F
61992024-03-06T21:00:26.669Z 10 200-219 55 55 55 92 92 92 F F F
62002024-03-06T21:00:26.669Z 11 220-239 49 49 49 78 78 78 F F F
62012024-03-06T21:00:26.669Z 12 240-259 47 47 47 73 73 73 F T T <---
62022024-03-06T21:00:26.669Z 13 260-279 53 56 56 86 93 93 F T T <---
62032024-03-06T21:00:26.670Z 14 280-299 53 56 56 86 94 94 F F F <---
62042024-03-06T21:00:26.670Z 15 300-319 50 56 56 80 93 93 F F F <---
62052024-03-06T21:00:26.670Z 16 320-339 32 56 56 47 93 93 F T T <---
62062024-03-06T21:00:26.670Z 17 340-359 53 56 56 86 93 93 F F F <---
62072024-03-06T21:00:26.670Z 18 360-379 55 55 55 92 92 92 F T T <---
62082024-03-06T21:00:26.670Z 19 380-399 54 54 54 91 91 91 F F F
62092024-03-06T21:00:26.670Z 20 400-419 52 56 56 85 95 95 F F F <---
62102024-03-06T21:00:26.670Z 21 420-439 52 56 56 84 95 95 F F F <---
62112024-03-06T21:00:26.670Z 22 440-459 54 56 56 87 94 94 F F F <---
62122024-03-06T21:00:26.671Z 23 460-479 54 56 56 87 94 94 F F F <---
62132024-03-06T21:00:26.671Z 24 480-499 42 42 42 67 67 67 F T T <---
62142024-03-06T21:00:26.671Z 25 500-519 54 54 54 87 87 87 F F F
62152024-03-06T21:00:26.671Z 26 520-539 54 54 54 87 87 87 F F F
62162024-03-06T21:00:26.671Z 27 540-559 54 54 54 87 87 87 F F F
62172024-03-06T21:00:26.679Z 28 560-579 54 54 54 87 87 87 F F F
62182024-03-06T21:00:26.679Z 29 580-599 54 54 54 91 91 91 F F F
62192024-03-06T21:00:26.679ZMax gen: 56, Max flush: 95
62202024-03-06T21:00:26.680ZError: Difference in extent metadata found!
62212024-03-06T21:00:26.680ZA Difference in extent metadata is expected here
62222024-03-06T21:00:26.680ZOn loop 28
62232024-03-06T21:00:26.680Z
62242024-03-06T21:00:26.680ZVerifying data now
62252024-03-06T21:00:31.808ZNo change after verify
62262024-03-06T21:00:31.808ZLoop: 28 Downstairs dump after verify (and repair):
62272024-03-06T21:00:31.827ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
62282024-03-06T21:00:31.827Z 0 000-019 52 52 52 84 84 84 F F F
62292024-03-06T21:00:31.827Z 1 020-039 52 52 52 85 85 85 F F F
62302024-03-06T21:00:31.828Z 2 040-059 55 55 55 92 92 92 F F F
62312024-03-06T21:00:31.828Z 3 060-079 52 52 52 83 83 83 F F F
62322024-03-06T21:00:31.828Z 4 080-099 50 50 50 81 81 81 F F F
62332024-03-06T21:00:31.828Z 5 100-119 57 57 57 96 96 96 F F F
62342024-03-06T21:00:31.828Z 6 120-139 56 56 56 93 93 93 F F F
62352024-03-06T21:00:31.828Z 7 140-159 57 57 57 96 96 96 F F F
62362024-03-06T21:00:31.828Z 8 160-179 57 57 57 96 96 96 F F F
62372024-03-06T21:00:31.828Z 9 180-199 51 51 51 82 82 82 F F F
62382024-03-06T21:00:31.828Z 10 200-219 55 55 55 92 92 92 F F F
62392024-03-06T21:00:31.828Z 11 220-239 49 49 49 78 78 78 F F F
62402024-03-06T21:00:31.828Z 12 240-259 57 57 57 96 96 96 F F F
62412024-03-06T21:00:31.828Z 13 260-279 57 57 57 96 96 96 F F F
62422024-03-06T21:00:31.828Z 14 280-299 56 56 56 94 94 94 F F F
62432024-03-06T21:00:31.828Z 15 300-319 56 56 56 93 93 93 F F F
62442024-03-06T21:00:31.828Z 16 320-339 57 57 57 96 96 96 F F F
62452024-03-06T21:00:31.828Z 17 340-359 56 56 56 93 93 93 F F F
62462024-03-06T21:00:31.828Z 18 360-379 57 57 57 96 96 96 F F F
62472024-03-06T21:00:31.828Z 19 380-399 54 54 54 91 91 91 F F F
62482024-03-06T21:00:31.828Z 20 400-419 56 56 56 95 95 95 F F F
62492024-03-06T21:00:31.829Z 21 420-439 56 56 56 95 95 95 F F F
62502024-03-06T21:00:31.829Z 22 440-459 56 56 56 94 94 94 F F F
62512024-03-06T21:00:31.829Z 23 460-479 56 56 56 94 94 94 F F F
62522024-03-06T21:00:31.829Z 24 480-499 57 57 57 96 96 96 F F F
62532024-03-06T21:00:31.829Z 25 500-519 54 54 54 87 87 87 F F F
62542024-03-06T21:00:31.829Z 26 520-539 54 54 54 87 87 87 F F F
62552024-03-06T21:00:31.829Z 27 540-559 54 54 54 87 87 87 F F F
62562024-03-06T21:00:31.829Z 28 560-579 54 54 54 87 87 87 F F F
62572024-03-06T21:00:31.829Z 29 580-599 54 54 54 91 91 91 F F F
62582024-03-06T21:00:31.829ZMax gen: 57, Max flush: 96
62592024-03-06T21:00:31.829Z
62602024-03-06T21:00:31.829ZBegin loop 29
62612024-03-06T21:00:31.829ZDownstairs to restart: 1
62622024-03-06T21:00:31.829Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.203987683Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371}
62632024-03-06T21:00:31.829Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.204245885Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371}
62642024-03-06T21:00:31.829Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.204263644Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371}
62652024-03-06T21:00:31.829Z{"msg":"Crucible 392e72b4-ddc0-4f89-b58d-8a60c3150d00 has session id: 4c1a5075-5308-47a2-8ee3-e86014c34c63","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.204278763Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62662024-03-06T21:00:31.829Z{"msg":"Upstairs opts: Upstairs UUID: 392e72b4-ddc0-4f89-b58d-8a60c3150d00, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.204315641Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62672024-03-06T21:00:31.829Z{"msg":"Crucible stats registered with UUID: 392e72b4-ddc0-4f89-b58d-8a60c3150d00","v":0,"name":"crucible","level":30Crucible runtime is spawned
62682024-03-06T21:00:31.829Z,"time":"2024-03-06T21:00:30.204339379Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62692024-03-06T21:00:31.829Z{"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.204454111Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371}
62702024-03-06T21:00:31.830Z{"msg":"392e72b4-ddc0-4f89-b58d-8a60c3150d00 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.20461021Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62712024-03-06T21:00:31.830Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.204677356Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"client":"0","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62722024-03-06T21:00:31.830Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.204729232Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"client":"0","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62732024-03-06T21:00:31.830Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.20475667Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"client":"1","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62742024-03-06T21:00:31.830Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.204792968Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"client":"1","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62752024-03-06T21:00:31.830Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.204817406Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"client":"2","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62762024-03-06T21:00:31.830Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.204840324Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"client":"2","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62772024-03-06T21:00:31.830Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.204870592Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"":"io task","client":"2","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62782024-03-06T21:00:31.830Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.20489854Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"":"io task","client":"0","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62792024-03-06T21:00:31.830Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.204947307Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"":"io task","client":"1","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62802024-03-06T21:00:31.830Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.205086737Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"":"io task","client":"0","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62812024-03-06T21:00:31.830Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.205135694Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"":"io task","client":"2","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62822024-03-06T21:00:31.830Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.205304672Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"":"io task","client":"1","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62832024-03-06T21:00:31.830Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.205829546Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"client":"2","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62842024-03-06T21:00:31.830Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.205869123Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"client":"2","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62852024-03-06T21:00:31.830Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.205895182Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"client":"0","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62862024-03-06T21:00:31.830Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.205926059Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"client":"0","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62872024-03-06T21:00:31.830Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.206539147Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"client":"2","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62882024-03-06T21:00:31.830Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.206583674Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"client":"0","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62892024-03-06T21:00:31.830Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.207058072Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"client":"1","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62902024-03-06T21:00:31.830Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.207097119Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"client":"1","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62912024-03-06T21:00:31.831Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.207292255Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"client":"2","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62922024-03-06T21:00:31.831Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.207331963Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"client":"2","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62932024-03-06T21:00:31.831Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.207349781Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62942024-03-06T21:00:31.831Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.20736282Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"client":"0","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62952024-03-06T21:00:31.831Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.207384859Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"client":"0","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62962024-03-06T21:00:31.831Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.207401688Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62972024-03-06T21:00:31.831Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.207712856Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"client":"1","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62982024-03-06T21:00:31.831Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.208167145Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"client":"1","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
62992024-03-06T21:00:31.831Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.208229701Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"client":"1","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
63002024-03-06T21:00:31.831Z{"msg":"[0]R flush_numbers[0..12]: [84, 85, 92, 83, 81, 96, 93, 96, 96, 82, 92, 78]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.208252449Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"":"downstairs","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
63012024-03-06T21:00:31.831Z{"msg":"[0]R generation[0..12]: [52, 52, 55, 52, 50, 57, 56, 57, 57, 51, 55, 49]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.208276737Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"":"downstairs","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
63022024-03-06T21:00:31.831Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.208291636Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"":"downstairs","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
63032024-03-06T21:00:31.831Z{"msg":"[1]R flush_numbers[0..12]: [84, 85, 92, 83, 81, 96, 93, 96, 96, 82, 92, 78]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.208314315Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"":"downstairs","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
63042024-03-06T21:00:31.831Z{"msg":"[1]R generation[0..12]: [52, 52, 55, 52, 50, 57, 56, 57, 57, 51, 55, 49]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.208328674Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"":"downstairs","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
63052024-03-06T21:00:31.831ZWait for a query_work_queue command to finish before sending IO
63062024-03-06T21:00:31.831Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.208354952Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"":"downstairs","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
63072024-03-06T21:00:31.831Z{"msg":"[2]R flush_numbers[0..12]: [84, 85, 92, 83, 81, 96, 93, 96, 96, 82, 92, 78]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.208370021Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"":"downstairs","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
63082024-03-06T21:00:31.831Z{"msg":"[2]R generation[0..12]: [52, 52, 55, 52, 50, 57, 56, 57, 57, 51, 55, 49]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.20839254Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"":"downstairs","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
63092024-03-06T21:00:31.831Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.208406999Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"":"downstairs","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
63102024-03-06T21:00:31.831Z{"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.208428277Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"":"downstairs","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
63112024-03-06T21:00:31.832Z{"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.208451085Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"":"downstairs","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
63122024-03-06T21:00:31.832Z{"msg":"Next flush: 97","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.208486463Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"":"downstairs","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
63132024-03-06T21:00:31.832Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.208518691Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"":"downstairs","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
63142024-03-06T21:00:31.832Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.208553478Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
63152024-03-06T21:00:31.832Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.208586086Z","hostnameRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
63162024-03-06T21:00:31.832Z":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"client":"0","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
63172024-03-06T21:00:31.832Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.208623304Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"client":"1","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
63182024-03-06T21:00:31.832Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.208638183Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"client":"2","":"downstairs","session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
63192024-03-06T21:00:31.832Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.208651612Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
63202024-03-06T21:00:31.832Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.208663781Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
63212024-03-06T21:00:31.832Z{"msg":"392e72b4-ddc0-4f89-b58d-8a60c3150d00 is now active with session: 4c1a5075-5308-47a2-8ee3-e86014c34c63","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.20867654Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
63222024-03-06T21:00:31.832Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.208688759Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371,"session_id":"4c1a5075-5308-47a2-8ee3-e86014c34c63"}
63232024-03-06T21:00:31.832Z{"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:30.208703878Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371}
63242024-03-06T21:00:31.832ZLoading write count information from file "/tmp/test_repair_verify.data"
63252024-03-06T21:00:31.832ZRun Repair workload
63262024-03-06T21:00:31.832Z01/30 Write block 381 len 28672 data: 2 4 4 7 7 8 6
63272024-03-06T21:00:31.841Z02/30 Write block 184 len 12288 data: 5 5 5
63282024-03-06T21:00:31.841Z03/30 Read block 251 len 12288
63292024-03-06T21:00:31.842Z04/30 Read block 555 len 28672
63302024-03-06T21:00:31.842Z05/30 Read block 528 len 4096
63312024-03-06T21:00:31.842Z06/30 Read block 204 len 36864
63322024-03-06T21:00:31.842Z07/30 Read block 597 len 12288
63332024-03-06T21:00:31.842Z08/30 Write block 60 len 24576 data: 11 13 13 15 13 12
63342024-03-06T21:00:31.842Z09/30 Read block 163 len 12288
63352024-03-06T21:00:31.842Z10/30 Read block 237 len 12288
63362024-03-06T21:00:31.842Z11/30 Write block 499 len 16384 data: 6 5 3 3
63372024-03-06T21:00:31.842Z12/30 Write block 347 len 40960 data: 5 7 6 7 8 6 6 6 5 7
63382024-03-06T21:00:31.842Z13/30 Write block 196 len 36864 data: 3 3 4 3 4 3 3 3 3
63392024-03-06T21:00:31.842Z14/30 Read block 126 len 20480
63402024-03-06T21:00:31.842Z15/30 Write block 495 len 8192 data: 8 8
63412024-03-06T21:00:31.842Z16/30 Write block 91 len 12288 data: 4 4 5
63422024-03-06T21:00:31.842Z17/30 Read block 190 len 8192
63432024-03-06T21:00:31.842Z18/30 Read block 165 len 36864
63442024-03-06T21:00:31.842Z19/30 Read block 288 len 40960
63452024-03-06T21:00:31.843Z20/30 Read block 325 len 32768
63462024-03-06T21:00:31.843Z21/30 Write block 63 len 28672 data: 16 14 13 12 11 9 9
63472024-03-06T21:00:31.843Z22/30 Read block 563 len 24576
63482024-03-06T21:00:31.843Z23/30 Read block 326 len 4096
63492024-03-06T21:00:31.843Z24/30 Flush
63502024-03-06T21:00:31.843Z25/30 Write block 21 len 36864 data: 4 5 6 5 5 5 5 4 4
63512024-03-06T21:00:31.843Z26/30 Write block 152 len 32768 data: 3 3 3 4 5 6 7 6
63522024-03-06T21:00:31.843Z27/30 Write block 94 len 12288 data: 6 5 6
63532024-03-06T21:00:31.843Z28/30 Write block 470 len 8192 data: 4 4
63542024-03-06T21:00:31.844Z29/30 Read block 404 len 8192
63552024-03-06T21:00:31.844Z30/30 Write block 114 len 8192 data: 3 3
63562024-03-06T21:00:31.844Z----------------------------------------------------------------
63572024-03-06T21:00:31.844Z Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:30
63582024-03-06T21:00:31.844ZStates: Active Active Active
63592024-03-06T21:00:31.844ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
63602024-03-06T21:00:31.844Z 1 Acked 1000 Write 7 Done Sent Done false
63612024-03-06T21:00:31.844Z 2 Acked 1001 Write 3 Done Sent Done false
63622024-03-06T21:00:31.844Z 3 Acked 1002 Read 3 Done Sent Done false
63632024-03-06T21:00:31.844Z 4 Acked 1003 Read 7 Done Sent Done false
63642024-03-06T21:00:31.844Z 5 Acked 1004 Read 1 Done Sent Done false
63652024-03-06T21:00:31.844Z 6 Acked 1005 Read 9 Done Sent Done false
63662024-03-06T21:00:31.844Z 7 Acked 1006 Read 3 Done Sent Done false
63672024-03-06T21:00:31.844Z 8 Acked 1007 Write 6 Done Sent Done false
63682024-03-06T21:00:31.844Z 9 Acked 1008 Read 3 Done Sent Done false
63692024-03-06T21:00:31.844Z 10 Acked 1009 Read 3 Done Sent Done false
63702024-03-06T21:00:31.845Z 11 Acked 1010 Write 4 Done Sent Done false
63712024-03-06T21:00:31.845Z 12 Acked 1011 Write 10 Done Sent Done false
63722024-03-06T21:00:31.845Z 13 Acked 1012 Write 9 Done Sent Done false
63732024-03-06T21:00:31.845Z 14 Acked 1013 Read 5 Done Sent Done false
63742024-03-06T21:00:31.845Z 15 Acked 1014 Write 2 Done Sent Done false
63752024-03-06T21:00:31.845Z 16 Acked 1015 Write 3 Done Sent Done false
63762024-03-06T21:00:31.845Z 17 Acked 1016 Read 2 Done Sent Done false
63772024-03-06T21:00:31.845Z 18 Acked 1017 Read 9 Done Sent Done false
63782024-03-06T21:00:31.845Z 19 Acked 1018 Read 10 Done Sent Done false
63792024-03-06T21:00:31.845Z 20 Acked 1019 Read 8 Done Sent Done false
63802024-03-06T21:00:31.845Z 21 Acked 1020 Write 7 Done Sent Done false
63812024-03-06T21:00:31.845Z 22 Acked 1021 Read 6 Done Sent Done false
63822024-03-06T21:00:31.845Z 23 Acked 1022 Read 1 Done Sent Done false
63832024-03-06T21:00:31.846Z 24 Acked 1023 Flush 0 Done Sent Done false
63842024-03-06T21:00:31.846Z 25 Acked 1024 Write 9 Done Sent Done false
63852024-03-06T21:00:31.846Z 26 Acked 1025 Write 8 Done Sent Done false
63862024-03-06T21:00:31.846Z 27 Acked 1026 Write 3 Done Sent Done false
63872024-03-06T21:00:31.846Z 28 Acked 1027 Write 2 Done Sent Done false
63882024-03-06T21:00:31.846Z 29 Acked 1028 Read 2 Done Sent Done false
63892024-03-06T21:00:31.846Z 30 Acked 1029 Write 2 Sent Sent Sent false
63902024-03-06T21:00:31.846Z STATES DS:0 DS:1 DS:2 TOTAL
63912024-03-06T21:00:31.846Z New 0 0 0 0
63922024-03-06T21:00:31.846Z Sent 1 30 1 32
63932024-03-06T21:00:31.846Z Done 29 0 29 58
63942024-03-06T21:00:31.846Z Skipped 0 0 0 0
63952024-03-06T21:00:31.846Z Error 0 0 0 0
63962024-03-06T21:00:31.847ZLast Flush: 1023 0 1023
63972024-03-06T21:00:31.847ZDownstairs last five completed:
63982024-03-06T21:00:31.847ZUpstairs last five completed: 30 29 28 27 26
63992024-03-06T21:00:31.847Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:00:30.240913938Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1371}
64002024-03-06T21:00:31.847ZWrote out file "/tmp/test_repair_verify.data"
64012024-03-06T21:00:31.848Z
64022024-03-06T21:00:31.848ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
64032024-03-06T21:00:31.848ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
64042024-03-06T21:00:31.848Z 0 000-019 52 52 52 84 84 84 F F F
64052024-03-06T21:00:31.848Z 1 020-039 52 52 52 85 85 85 T F T <---
64062024-03-06T21:00:31.848Z 2 040-059 55 55 55 92 92 92 F F F
64072024-03-06T21:00:31.848Z 3 060-079 58 52 58 97 83 97 F F F <---
64082024-03-06T21:00:31.848Z 4 080-099 58 50 58 97 81 97 T F T <---
64092024-03-06T21:00:31.848Z 5 100-119 57 57 57 96 96 96 T F T <---
64102024-03-06T21:00:31.848Z 6 120-139 56 56 56 93 93 93 F F F
64112024-03-06T21:00:31.848Z 7 140-159 57 57 57 96 96 96 T F T <---
64122024-03-06T21:00:31.848Z 8 160-179 57 57 57 96 96 96 F F F
64132024-03-06T21:00:31.848Z 9 180-199 58 51 58 97 82 97 F F F <---
64142024-03-06T21:00:31.848Z 10 200-219 58 55 58 97 92 97 F F F <---
64152024-03-06T21:00:31.848Z 11 220-239 49 49 49 78 78 78 F F F
64162024-03-06T21:00:31.848Z 12 240-259 57 57 57 96 96 96 F F F
64172024-03-06T21:00:31.848Z 13 260-279 57 57 57 96 96 96 F F F
64182024-03-06T21:00:31.848Z 14 280-299 56 56 56 94 94 94 F F F
64192024-03-06T21:00:31.848Z 15 300-319 56 56 56 93 93 93 F F F
64202024-03-06T21:00:31.848Z 16 320-339 57 57 57 96 96 96 F F F
64212024-03-06T21:00:31.849Z 17 340-359 58 56 58 97 93 97 F F F <---
64222024-03-06T21:00:31.849Z 18 360-379 57 57 57 96 96 96 F F F
64232024-03-06T21:00:31.849Z 19 380-399 58 54 58 97 91 97 F F F <---
64242024-03-06T21:00:31.849Z 20 400-419 56 56 56 95 95 95 F F F
64252024-03-06T21:00:31.849Z 21 420-439 56 56 56 95 95 95 F F F
64262024-03-06T21:00:31.849Z 22 440-459 56 56 56 94 94 94 F F F
64272024-03-06T21:00:31.859Z 23 460-479 56 56 56 94 94 94 T F T <---
64282024-03-06T21:00:31.859Z 24 480-499 58 57 58 97 96 97 F F F <---
64292024-03-06T21:00:31.859Z 25 500-519 58 54 58 97 87 97 F F F <---
64302024-03-06T21:00:31.859Z 26 520-539 54 54 54 87 87 87 F F F
64312024-03-06T21:00:31.859Z 27 540-559 54 54 54 87 87 87 F F F
64322024-03-06T21:00:31.859Z 28 560-579 54 54 54 87 87 87 F F F
64332024-03-06T21:00:31.859Z 29 580-599 54 54 54 91 91 91 F F F
64342024-03-06T21:00:31.859ZMax gen: 58, Max flush: 97
64352024-03-06T21:00:31.859ZError: Difference in extent metadata found!
64362024-03-06T21:00:31.859ZA Difference in extent metadata is expected here
64372024-03-06T21:00:31.859ZOn loop 29
64382024-03-06T21:00:31.859Z
64392024-03-06T21:00:31.859ZVerifying data now
64402024-03-06T21:00:36.165ZNo change after verify
64412024-03-06T21:00:36.165ZLoop: 29 Downstairs dump after verify (and repair):
64422024-03-06T21:00:36.180ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
64432024-03-06T21:00:36.180Z 0 000-019 52 52 52 84 84 84 F F F
64442024-03-06T21:00:36.180Z 1 020-039 59 59 59 98 98 98 F F F
64452024-03-06T21:00:36.180Z 2 040-059 55 55 55 92 92 92 F F F
64462024-03-06T21:00:36.181Z 3 060-079 58 58 58 97 97 97 F F F
64472024-03-06T21:00:36.181Z 4 080-099 59 59 59 98 98 98 F F F
64482024-03-06T21:00:36.181Z 5 100-119 59 59 59 98 98 98 F F F
64492024-03-06T21:00:36.181Z 6 120-139 56 56 56 93 93 93 F F F
64502024-03-06T21:00:36.181Z 7 140-159 59 59 59 98 98 98 F F F
64512024-03-06T21:00:36.181Z 8 160-179 57 57 57 96 96 96 F F F
64522024-03-06T21:00:36.181Z 9 180-199 58 58 58 97 97 97 F F F
64532024-03-06T21:00:36.181Z 10 200-219 58 58 58 97 97 97 F F F
64542024-03-06T21:00:36.181Z 11 220-239 49 49 49 78 78 78 F F F
64552024-03-06T21:00:36.181Z 12 240-259 57 57 57 96 96 96 F F F
64562024-03-06T21:00:36.182Z 13 260-279 57 57 57 96 96 96 F F F
64572024-03-06T21:00:36.182Z 14 280-299 56 56 56 94 94 94 F F F
64582024-03-06T21:00:36.182Z 15 300-319 56 56 56 93 93 93 F F F
64592024-03-06T21:00:36.182Z 16 320-339 57 57 57 96 96 96 F F F
64602024-03-06T21:00:36.182Z 17 340-359 58 58 58 97 97 97 F F F
64612024-03-06T21:00:36.182Z 18 360-379 57 57 57 96 96 96 F F F
64622024-03-06T21:00:36.182Z 19 380-399 58 58 58 97 97 97 F F F
64632024-03-06T21:00:36.190Z 20 400-419 56 56 56 95 95 95 F F F
64642024-03-06T21:00:36.190Z 21 420-439 56 56 56 95 95 95 F F F
64652024-03-06T21:00:36.190Z 22 440-459 56 56 56 94 94 94 F F F
64662024-03-06T21:00:36.191Z 23 460-479 59 59 59 98 98 98 F F F
64672024-03-06T21:00:36.191Z 24 480-499 58 58 58 97 97 97 F F F
64682024-03-06T21:00:36.191Z 25 500-519 58 58 58 97 97 97 F F F
64692024-03-06T21:00:36.191Z 26 520-539 54 54 54 87 87 87 F F F
64702024-03-06T21:00:36.191Z 27 540-559 54 54 54 87 87 87 F F F
64712024-03-06T21:00:36.191Z 28 560-579 54 54 54 87 87 87 F F F
64722024-03-06T21:00:36.191Z 29 580-599 54 54 54 91 91 91 F F F
64732024-03-06T21:00:36.191ZMax gen: 59, Max flush: 98
64742024-03-06T21:00:36.191Z
64752024-03-06T21:00:36.191ZBegin loop 30
64762024-03-06T21:00:36.191ZDownstairs to restart: 1
64772024-03-06T21:00:36.205Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.965923395Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379}
64782024-03-06T21:00:36.205Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.966206045Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379}
64792024-03-06T21:00:36.205Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.966233213Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379}
64802024-03-06T21:00:36.225Z{"msg":"Crucible 9dd6ae04-88b9-470c-b4fd-5c5324987b73 has session id: a7c44e3e-ce96-44d8-9472-cdbebdd99f9a","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.966275741Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
64812024-03-06T21:00:36.225ZCrucible runtime is spawned
64822024-03-06T21:00:36.226Z{"msg":"Upstairs opts: Upstairs UUID: 9dd6ae04-88b9-470c-b4fd-5c5324987b73, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.966334446Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
64832024-03-06T21:00:36.226Z{"msg":"Crucible stats registered with UUID: 9dd6ae04-88b9-470c-b4fd-5c5324987b73","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.966372014Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
64842024-03-06T21:00:36.226Z{"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.966395572Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379}
64852024-03-06T21:00:36.226Z{"msg":"9dd6ae04-88b9-470c-b4fd-5c5324987b73 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.96643607Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
64862024-03-06T21:00:36.226Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.966462138Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"client":"0","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
64872024-03-06T21:00:36.226Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.966506945Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"client":"0","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
64882024-03-06T21:00:36.226Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.966531753Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"client":"1","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
64892024-03-06T21:00:36.226Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.966565081Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"client":"1","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
64902024-03-06T21:00:36.226Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.966589919Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"client":"2","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
64912024-03-06T21:00:36.226Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.966638025Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"client":"2","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
64922024-03-06T21:00:36.227Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.966668734Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"":"io task","client":"0","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
64932024-03-06T21:00:36.227Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.966700041Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"":"io task","client":"1","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
64942024-03-06T21:00:36.227Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.966736529Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"":"io task","client":"2","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
64952024-03-06T21:00:36.227Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.966955994Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"":"io task","client":"0","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
64962024-03-06T21:00:36.227Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.967034368Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"":"io task","client":"2","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
64972024-03-06T21:00:36.227Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.967073066Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"":"io task","client":"1","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
64982024-03-06T21:00:36.227Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.967778927Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"client":"2","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
64992024-03-06T21:00:36.227Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.967823014Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"client":"2","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65002024-03-06T21:00:36.228Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.967849612Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"client":"0","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65012024-03-06T21:00:36.228Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.96787477Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"client":"0","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65022024-03-06T21:00:36.228Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.968443391Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"client":"0","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65032024-03-06T21:00:36.228Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.96860569Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"client":"2","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65042024-03-06T21:00:36.228Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.968959165Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"client":"0","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65052024-03-06T21:00:36.228Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.969007142Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"client":"0","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65062024-03-06T21:00:36.228Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.969050079Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65072024-03-06T21:00:36.228Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.969164511Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"client":"2","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65082024-03-06T21:00:36.228Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.969207299Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"client":"2","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65092024-03-06T21:00:36.228Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.969232477Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65102024-03-06T21:00:36.228Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.969260735Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"client":"1","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65112024-03-06T21:00:36.229Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.969282183Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"client":"1","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65122024-03-06T21:00:36.229Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.969806137Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"client":"1","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65132024-03-06T21:00:36.229Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.97020171Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"client":"1","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65142024-03-06T21:00:36.229Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.970241947Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"client":"1","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65152024-03-06T21:00:36.229Z{"msg":"[0]R flush_numbers[0..12]: [84, 98, 92, 97, 98, 98, 93, 98, 96, 97, 97, 78]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.970260946Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"":"downstairs","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65162024-03-06T21:00:36.229Z{"msg":"[0]R generation[0..12]: [52, 59, 55, 58, 59, 59, 56, 59, 57, 58, 58, 49]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.970276755Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"":"downstairs","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65172024-03-06T21:00:36.229Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.970298563Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"":"downstairs","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65182024-03-06T21:00:36.229Z{"msg":"[1]R flush_numbers[0..12]: [84, 98, 92, 97, 98, 98, 93, 98, 96, 97, 97, 78]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.970312802Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"":"downstairs","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65192024-03-06T21:00:36.230Z{"msg":"[1]R generation[0..12]: [52, 59, 55, 58, 59, 59, 56, 59, 57, 58, 58, 49]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.970333261Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"":"downstairs","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65202024-03-06T21:00:36.230Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.97034783Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"":"downstairs","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65212024-03-06T21:00:36.230Z{"msg":"[2]R flush_numbers[0..12]: [84, 98, 92, 97, 98, 98, 93, 98, 96, 97, 97, 78]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.970368319Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"":"downstairs","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65222024-03-06T21:00:36.230Z{"msg":"[2]R generation[0..12]: [52, 59, 55, 58, 59, 59, 56, 59, 57, 58, 58, 49]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.970382378Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB"Wait for a query_work_queue command to finish before sending IO
65232024-03-06T21:00:36.230Z,"pid":1379,"":"downstairs","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65242024-03-06T21:00:36.231Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.970410405Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"":"downstairs","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65252024-03-06T21:00:36.231Z{"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.970423595Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"":"downstairs","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65262024-03-06T21:00:36.231Z{"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.970437304Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"":"downstairs","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65272024-03-06T21:00:36.231Z{"msg":"Next flush: 99","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.970450453Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"":"downstairs","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65282024-03-06T21:00:36.231Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.970464242Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"":"downstairs","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65292024-03-06T21:00:36.231Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.970477661Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65302024-03-06T21:00:36.231Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.97049683Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"client":"0","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65312024-03-06T21:00:36.231Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.970511629Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"client":"1","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65322024-03-06T21:00:36.231Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.970525518Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"client":"2","":"downstairs","session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65332024-03-06T21:00:36.231Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.970539487Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65342024-03-06T21:00:36.232Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.970558515Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65352024-03-06T21:00:36.232Z{"msg":"9dd6ae04-88b9-470c-b4fd-5c5324987b73 is now active with session: a7c44e3e-ce96-44d8-9472-cdbebdd99f9a","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.970571465Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65362024-03-06T21:00:36.232Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.970589723Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379,"session_id":"a7c44e3e-ce96-44d8-9472-cdbebdd99f9a"}
65372024-03-06T21:00:36.232Z{"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:34.970602172Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379}
65382024-03-06T21:00:36.232ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
65392024-03-06T21:00:36.232ZLoading write count information from file "/tmp/test_repair_verify.data"
65402024-03-06T21:00:36.232ZRun Repair workload
65412024-03-06T21:00:36.232Z01/30 Write block 365 len 28672 data: 7 6 7 5 7 8 7
65422024-03-06T21:00:36.232Z02/30 Read block 58 len 16384
65432024-03-06T21:00:36.244Z03/30 Write block 153 len 40960 data: 4 4 5 6 7 8 7 6 5 5
65442024-03-06T21:00:36.244Z04/30 Write block 288 len 8192 data: 13 13
65452024-03-06T21:00:36.244Z05/30 Read block 505 len 40960
65462024-03-06T21:00:36.244Z06/30 Flush
65472024-03-06T21:00:36.244Z07/30 Write block 507 len 36864 data: 4 3 2 4 4 5 5 5 6
65482024-03-06T21:00:36.244Z08/30 Read block 234 len 4096
65492024-03-06T21:00:36.244Z09/30 Write block 521 len 40960 data: 2 2 2 4 4 4 5 6 7 8
65502024-03-06T21:00:36.244Z10/30 Read block 436 len 36864
65512024-03-06T21:00:36.245Z11/30 Read block 372 len 32768
65522024-03-06T21:00:36.245Z12/30 Read block 547 len 40960
65532024-03-06T21:00:36.245Z13/30 Flush
65542024-03-06T21:00:36.245Z14/30 Write block 223 len 8192 data: 3 3
65552024-03-06T21:00:36.245Z15/30 Write block 592 len 12288 data: 7 4 5
65562024-03-06T21:00:36.245Z16/30 Flush
65572024-03-06T21:00:36.245Z17/30 Flush
65582024-03-06T21:00:36.246Z18/30 Flush
65592024-03-06T21:00:36.246Z19/30 Write block 4 len 28672 data: 3 4 4 5 4 4 3
65602024-03-06T21:00:36.246Z20/30 Write block 58 len 12288 data: 9 11 12
65612024-03-06T21:00:36.246Z21/30 Write block 188 len 4096 data: 5
65622024-03-06T21:00:36.246Z22/30 Read block 305 len 4096
65632024-03-06T21:00:36.246Z23/30 Write block 78 len 20480 data: 5 4 4 4 4
65642024-03-06T21:00:36.246Z24/30 Read block 280 len 32768
65652024-03-06T21:00:36.246Z25/30 Write block 535 len 16384 data: 7 6 8 6
65662024-03-06T21:00:36.252Z26/30 Write block 457 len 12288 data: 6 6 6
65672024-03-06T21:00:36.252Z27/30 Read block 432 len 32768
65682024-03-06T21:00:36.252Z28/30 Write block 223 len 40960 data: 4 4 4 5 5 4 4 4 5 5
65692024-03-06T21:00:36.252Z29/30 Read block 25 len 20480
65702024-03-06T21:00:36.252Z30/30 Write block 519 len 36864 data: 6 4 3 3 3 5 5 5 6
65712024-03-06T21:00:36.252Z----------------------------------------------------------------
65722024-03-06T21:00:36.253Z Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:30
65732024-03-06T21:00:36.253ZStates: Active Active Active
65742024-03-06T21:00:36.253ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
65752024-03-06T21:00:36.253Z 1 Acked 1000 Write 7 Done Sent Done false
65762024-03-06T21:00:36.253Z 2 Acked 1001 Read 4 Done Sent Done false
65772024-03-06T21:00:36.253Z 3 Acked 1002 Write 10 Done Sent Done false
65782024-03-06T21:00:36.253Z 4 Acked 1003 Write 2 Done Sent Done false
65792024-03-06T21:00:36.253Z 5 Acked 1004 Read 10 Done Sent Done false
65802024-03-06T21:00:36.253Z 6 Acked 1005 Flush 0 Done Sent Done false
65812024-03-06T21:00:36.253Z 7 Acked 1006 Write 9 Done Sent Done false
65822024-03-06T21:00:36.253Z 8 Acked 1007 Read 1 Done Sent Done false
65832024-03-06T21:00:36.253Z 9 Acked 1008 Write 10 Done Sent Done false
65842024-03-06T21:00:36.253Z 10 Acked 1009 Read 9 Done Sent Done false
65852024-03-06T21:00:36.253Z 11 Acked 1010 Read 8 Done Sent Done false
65862024-03-06T21:00:36.254Z 12 Acked 1011 Read 10 Done Sent Done false
65872024-03-06T21:00:36.254Z 13 Acked 1012 Flush 0 Done Sent Done false
65882024-03-06T21:00:36.254Z 14 Acked 1013 Write 2 Done Sent Done false
65892024-03-06T21:00:36.254Z 15 Acked 1014 Write 3 Done Sent Done false
65902024-03-06T21:00:36.254Z 16 Acked 1015 Flush 0 Done Sent Done false
65912024-03-06T21:00:36.254Z 17 Acked 1016 Flush 0 Done Sent Done false
65922024-03-06T21:00:36.254Z 18 Acked 1017 Flush 0 Done Sent Done false
65932024-03-06T21:00:36.254Z 19 Acked 1018 Write 7 Done Sent Done false
65942024-03-06T21:00:36.254Z 20 Acked 1019 Write 3 Done Sent Done false
65952024-03-06T21:00:36.254Z 21 Acked 1020 Write 1 Done Sent Done false
65962024-03-06T21:00:36.254Z 22 Acked 1021 Read 1 Done Sent Done false
65972024-03-06T21:00:36.254Z 23 Acked 1022 Write 5 Done Sent Done false
65982024-03-06T21:00:36.254Z 24 Acked 1023 Read 8 Done Sent Done false
65992024-03-06T21:00:36.254Z 25 Acked 1024 Write 4 Done Sent Done false
66002024-03-06T21:00:36.255Z 26 Acked 1025 Write 3 Done Sent Done false
66012024-03-06T21:00:36.255Z 27 Acked 1026 Read 8 Done Sent Done false
66022024-03-06T21:00:36.255Z 28 Acked 1027 Write 10 Done Sent Done false
66032024-03-06T21:00:36.255Z 29 Acked 1028 Read 5 Done Sent Done false
66042024-03-06T21:00:36.255Z 30 Acked 1029 Write 9 Sent Sent Sent false
66052024-03-06T21:00:36.255Z STATES DS:0 DS:1 DS:2 TOTAL
66062024-03-06T21:00:36.255Z New 0 0 0 0
66072024-03-06T21:00:36.255Z Sent 1 30 1 32
66082024-03-06T21:00:36.255Z Done 29 0 29 58
66092024-03-06T21:00:36.255Z Skipped 0 0 0 0
66102024-03-06T21:00:36.255Z Error 0 0 0 0
66112024-03-06T21:00:36.255ZLast Flush: 1017 0 1017
66122024-03-06T21:00:36.255ZDownstairs last five completed:
66132024-03-06T21:00:36.255ZUpstairs last five completed: 30 29 28 27 26
66142024-03-06T21:00:36.255Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:00:35.00778176Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1379}
66152024-03-06T21:00:36.255ZWrote out file "/tmp/test_repair_verify.data"
66162024-03-06T21:00:36.256Z
66172024-03-06T21:00:36.263ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
66182024-03-06T21:00:36.280ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
66192024-03-06T21:00:36.280Z 0 000-019 52 52 52 84 84 84 T F T <---
66202024-03-06T21:00:36.280Z 1 020-039 59 59 59 98 98 98 F F F
66212024-03-06T21:00:36.280Z 2 040-059 55 55 55 92 92 92 T F T <---
66222024-03-06T21:00:36.280Z 3 060-079 58 58 58 97 97 97 T F T <---
66232024-03-06T21:00:36.280Z 4 080-099 59 59 59 98 98 98 T F T <---
66242024-03-06T21:00:36.281Z 5 100-119 59 59 59 98 98 98 F F F
66252024-03-06T21:00:36.281Z 6 120-139 56 56 56 93 93 93 F F F
66262024-03-06T21:00:36.281Z 7 140-159 60 59 60 99 98 99 F F F <---
66272024-03-06T21:00:36.281Z 8 160-179 60 57 60 99 96 99 F F F <---
66282024-03-06T21:00:36.281Z 9 180-199 58 58 58 97 97 97 T F T <---
66292024-03-06T21:00:36.281Z 10 200-219 58 58 58 97 97 97 F F F
66302024-03-06T21:00:36.281Z 11 220-239 60 49 60 101 78 101 T F T <---
66312024-03-06T21:00:36.281Z 12 240-259 57 57 57 96 96 96 F F F
66322024-03-06T21:00:36.281Z 13 260-279 57 57 57 96 96 96 F F F
66332024-03-06T21:00:36.281Z 14 280-299 60 56 60 99 94 99 F F F <---
66342024-03-06T21:00:36.281Z 15 300-319 56 56 56 93 93 93 F F F
66352024-03-06T21:00:36.281Z 16 320-339 57 57 57 96 96 96 F F F
66362024-03-06T21:00:36.291Z 17 340-359 58 58 58 97 97 97 F F F
66372024-03-06T21:00:36.291Z 18 360-379 60 57 60 99 96 99 F F F <---
66382024-03-06T21:00:36.291Z 19 380-399 58 58 58 97 97 97 F F F
66392024-03-06T21:00:36.291Z 20 400-419 56 56 56 95 95 95 F F F
66402024-03-06T21:00:36.291Z 21 420-439 56 56 56 95 95 95 F F F
66412024-03-06T21:00:36.291Z 22 440-459 56 56 56 94 94 94 T F T <---
66422024-03-06T21:00:36.291Z 23 460-479 59 59 59 98 98 98 F F F
66432024-03-06T21:00:36.291Z 24 480-499 58 58 58 97 97 97 F F F
66442024-03-06T21:00:36.291Z 25 500-519 60 58 60 100 97 100 T F F <---
66452024-03-06T21:00:36.291Z 26 520-539 60 54 60 100 87 100 T F T <---
66462024-03-06T21:00:36.292Z 27 540-559 54 54 54 87 87 87 F F F
66472024-03-06T21:00:36.292Z 28 560-579 54 54 54 87 87 87 F F F
66482024-03-06T21:00:36.292Z 29 580-599 60 54 60 101 91 101 F F F <---
66492024-03-06T21:00:36.292ZMax gen: 60, Max flush: 101
66502024-03-06T21:00:36.292ZError: Difference in extent metadata found!
66512024-03-06T21:00:36.292ZA Difference in extent metadata is expected here
66522024-03-06T21:00:36.292ZOn loop 30
66532024-03-06T21:00:36.292Z
66542024-03-06T21:00:36.292ZVerifying data now
66552024-03-06T21:00:41.017ZNo change after verify
66562024-03-06T21:00:41.017ZLoop: 30 Downstairs dump after verify (and repair):
66572024-03-06T21:00:41.031ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
66582024-03-06T21:00:41.032Z 0 000-019 61 61 61 102 102 102 F F F
66592024-03-06T21:00:41.032Z 1 020-039 59 59 59 98 98 98 F F F
66602024-03-06T21:00:41.032Z 2 040-059 61 61 61 102 102 102 F F F
66612024-03-06T21:00:41.032Z 3 060-079 61 61 61 102 102 102 F F F
66622024-03-06T21:00:41.032Z 4 080-099 61 61 61 102 102 102 F F F
66632024-03-06T21:00:41.033Z 5 100-119 59 59 59 98 98 98 F F F
66642024-03-06T21:00:41.033Z 6 120-139 56 56 56 93 93 93 F F F
66652024-03-06T21:00:41.033Z 7 140-159 60 60 60 99 99 99 F F F
66662024-03-06T21:00:41.033Z 8 160-179 60 60 60 99 99 99 F F F
66672024-03-06T21:00:41.033Z 9 180-199 61 61 61 102 102 102 F F F
66682024-03-06T21:00:41.033Z 10 200-219 58 58 58 97 97 97 F F F
66692024-03-06T21:00:41.033Z 11 220-239 61 61 61 102 102 102 F F F
66702024-03-06T21:00:41.034Z 12 240-259 57 57 57 96 96 96 F F F
66712024-03-06T21:00:41.057Z 13 260-279 57 57 57 96 96 96 F F F
66722024-03-06T21:00:41.057Z 14 280-299 60 60 60 99 99 99 F F F
66732024-03-06T21:00:41.057Z 15 300-319 56 56 56 93 93 93 F F F
66742024-03-06T21:00:41.057Z 16 320-339 57 57 57 96 96 96 F F F
66752024-03-06T21:00:41.058Z 17 340-359 58 58 58 97 97 97 F F F
66762024-03-06T21:00:41.058Z 18 360-379 60 60 60 99 99 99 F F F
66772024-03-06T21:00:41.058Z 19 380-399 58 58 58 97 97 97 F F F
66782024-03-06T21:00:41.058Z 20 400-419 56 56 56 95 95 95 F F F
66792024-03-06T21:00:41.058Z 21 420-439 56 56 56 95 95 95 F F F
66802024-03-06T21:00:41.058Z 22 440-459 61 61 61 102 102 102 F F F
66812024-03-06T21:00:41.059Z 23 460-479 59 59 59 98 98 98 F F F
66822024-03-06T21:00:41.059Z 24 480-499 58 58 58 97 97 97 F F F
66832024-03-06T21:00:41.059Z 25 500-519 61 61 61 102 102 102 F F F
66842024-03-06T21:00:41.059Z 26 520-539 61 61 61 102 102 102 F F F
66852024-03-06T21:00:41.059Z 27 540-559 54 54 54 87 87 87 F F F
66862024-03-06T21:00:41.059Z 28 560-579 54 54 54 87 87 87 F F F
66872024-03-06T21:00:41.059Z 29 580-599 60 60 60 101 101 101 F F F
66882024-03-06T21:00:41.060ZMax gen: 61, Max flush: 102
66892024-03-06T21:00:41.060Z
66902024-03-06T21:00:41.060ZBegin loop 31
66912024-03-06T21:00:41.060ZDownstairs to restart: 2
66922024-03-06T21:00:41.095Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.815077696Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387}
66932024-03-06T21:00:41.095Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.815396454Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387}
66942024-03-06T21:00:41.095Z{Crucible runtime is spawned
66952024-03-06T21:00:41.096Z"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.815430962Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387}
66962024-03-06T21:00:41.096Z{"msg":"Crucible 2fb747b3-77bc-46e4-8e37-91b15c4c0c28 has session id: 8a2c5439-ffe4-4af0-a528-b37bae05013e","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.815470219Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
66972024-03-06T21:00:41.096Z{"msg":"Upstairs opts: Upstairs UUID: 2fb747b3-77bc-46e4-8e37-91b15c4c0c28, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.815515946Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
66982024-03-06T21:00:41.096Z{"msg":"Crucible stats registered with UUID: 2fb747b3-77bc-46e4-8e37-91b15c4c0c28","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.815554084Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
66992024-03-06T21:00:41.097Z{"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.815582211Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387}
67002024-03-06T21:00:41.097Z{"msg":"2fb747b3-77bc-46e4-8e37-91b15c4c0c28 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.815698514Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67012024-03-06T21:00:41.097Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.815734651Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"client":"0","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67022024-03-06T21:00:41.097Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.81575351Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"client":"0","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67032024-03-06T21:00:41.097Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.815787207Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"client":"1","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67042024-03-06T21:00:41.098Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.815815355Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"client":"1","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67052024-03-06T21:00:41.098Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.815857103Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"client":"2","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67062024-03-06T21:00:41.098Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.815918058Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"client":"2","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67072024-03-06T21:00:41.099Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.815970225Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"":"io task","client":"0","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67082024-03-06T21:00:41.099Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.816002003Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"":"io task","client":"2","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67092024-03-06T21:00:41.099Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.816026311Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"":"io task","client":"1","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67102024-03-06T21:00:41.099Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.816061409Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"":"io task","client":"0","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67112024-03-06T21:00:41.099Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.816305032Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"":"io task","client":"1","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67122024-03-06T21:00:41.100Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.816359118Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"":"io task","client":"2","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67132024-03-06T21:00:41.100Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.816733852Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"client":"0","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67142024-03-06T21:00:41.100Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.816788838Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"client":"0","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67152024-03-06T21:00:41.100Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.816966136Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"client":"1","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67162024-03-06T21:00:41.100Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.81706111Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"client":"1","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67172024-03-06T21:00:41.100Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.817430304Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"client":"2","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67182024-03-06T21:00:41.101Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.817474981Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"client":"2","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67192024-03-06T21:00:41.101Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.8174926Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"client":"0","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67202024-03-06T21:00:41.101Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.817615701Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"client":"1","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67212024-03-06T21:00:41.101Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.818109847Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"client":"1","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67222024-03-06T21:00:41.102Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.818149595Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"client":"1","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67232024-03-06T21:00:41.102Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.818166173Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67242024-03-06T21:00:41.102Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.818199381Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"client":"2","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67252024-03-06T21:00:41.102Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.818236499Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"client":"0","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67262024-03-06T21:00:41.102Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.818262067Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"client":"0","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67272024-03-06T21:00:41.103Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.818277956Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67282024-03-06T21:00:41.103Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.81864876Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"client":"2","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67292024-03-06T21:00:41.103Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.818690607Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"client":"2","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67302024-03-06T21:00:41.103Z{"msg":"[0]R flush_numbers[0..12]: [102, 98, 102, 102, 102, 98, 93, 99, 99, 102, 97, 102]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.818707806Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"":"downstairs","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67312024-03-06T21:00:41.103Z{"msg":"[0]R generation[0..12]: [61, 59, 61, 61, 61, 59, 56, 60, 60, 61, 58, 61]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.818727165Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"":"downstairs","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67322024-03-06T21:00:41.103Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.818744394Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"":"downstairs","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67332024-03-06T21:00:41.103Z{"msg":"[1]R flush_numbers[0..12]: [102, 98, 102, 102, 102, 98, 93, 99, 99, 102, 97, 102]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.818759792Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"":"downstairs","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67342024-03-06T21:00:41.104Z{"msg":"[1]R generation[0..12]: [61, 59, 61, 61, 61, 59, 56, 60, 60, 61, 58, 61]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.818774071Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"":"downstairs","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67352024-03-06T21:00:41.104Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.81879647Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"":"downstairs","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67362024-03-06T21:00:41.104Z{"msg":"[2]R flush_numbers[0..12]: [102, 98, 102, 102, 102, 98, 93, 99, 99, 102, 97, 102]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.818811269Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"":"downstairs","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67372024-03-06T21:00:41.104Z{"msg":"[2]R generation[0..12]: [61, 59, 61, 61, 61, 59, 56, 60, 60, 61, 58, 61]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
67382024-03-06T21:00:41.105Z2024-03-06T21:00:39.818825908Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"":"downstairs","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67392024-03-06T21:00:41.105Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.818843307Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"":"downstairs","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67402024-03-06T21:00:41.105Z{"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.818863375Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"":"downstairs","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67412024-03-06T21:00:41.105Z{"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.818876824Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"":"downstairs","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67422024-03-06T21:00:41.105Z{"msg":"Next flush: 103","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.818889634Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"":"downstairs","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67432024-03-06T21:00:41.105Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.818908882Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"":"downstairs","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67442024-03-06T21:00:41.106Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.818923401Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67452024-03-06T21:00:41.106Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.81893542Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"client":"0","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67462024-03-06T21:00:41.106Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.81894941Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"client":"1","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67472024-03-06T21:00:41.106Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.818963358Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"client":"2","":"downstairs","session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67482024-03-06T21:00:41.106Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.818979817Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67492024-03-06T21:00:41.107Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.819010995Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67502024-03-06T21:00:41.107Z{"msg":"2fb747b3-77bc-46e4-8e37-91b15c4c0c28 is now active with session: 8a2c5439-ffe4-4af0-a528-b37bae05013e","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.819034784Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67512024-03-06T21:00:41.107Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.819056532Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387,"session_id":"8a2c5439-ffe4-4af0-a528-b37bae05013e"}
67522024-03-06T21:00:41.107Z{"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:39.819069251Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387}
67532024-03-06T21:00:41.107ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
67542024-03-06T21:00:41.108ZLoading write count information from file "/tmp/test_repair_verify.data"
67552024-03-06T21:00:41.108ZRun Repair workload
67562024-03-06T21:00:41.108Z01/30 Write block 280 len 12288 data: 6 7 7
67572024-03-06T21:00:41.108Z02/30 Read block 437 len 16384
67582024-03-06T21:00:41.108Z03/30 Write block 460 len 28672 data: 7 10 9 11 10 10 8
67592024-03-06T21:00:41.109Z04/30 Read block 89 len 20480
67602024-03-06T21:00:41.109Z05/30 Write block 485 len 4096 data: 5
67612024-03-06T21:00:41.109Z06/30 Read block 101 len 36864
67622024-03-06T21:00:41.109Z07/30 Write block 575 len 8192 data: 5 6
67632024-03-06T21:00:41.109Z08/30 Write block 397 len 8192 data: 5 5
67642024-03-06T21:00:41.109Z09/30 Read block 108 len 24576
67652024-03-06T21:00:41.109Z10/30 Flush
67662024-03-06T21:00:41.146Z11/30 Write block 358 len 8192 data: 6 6
67672024-03-06T21:00:41.147Z12/30 Write block 215 len 12288 data: 6 7 8
67682024-03-06T21:00:41.147Z13/30 Read block 29 len 20480
67692024-03-06T21:00:41.148Z14/30 Read block 162 len 28672
67702024-03-06T21:00:41.148Z15/30 Write block 182 len 12288 data: 8 6 6
67712024-03-06T21:00:41.148Z16/30 Write block 30 len 24576 data: 5 5 5 6 6 6
67722024-03-06T21:00:41.149Z17/30 Write block 149 len 36864 data: 6 7 6 4 5 5 6 7 8
67732024-03-06T21:00:41.149Z18/30 Read block 498 len 8192
67742024-03-06T21:00:41.149Z19/30 Read block 129 len 20480
67752024-03-06T21:00:41.149Z20/30 Read block 24 len 24576
67762024-03-06T21:00:41.149Z21/30 Write block 555 len 8192 data: 4 3
67772024-03-06T21:00:41.149Z22/30 Read block 558 len 12288
67782024-03-06T21:00:41.149Z23/30 Write block 270 len 28672 data: 5 5 5 5 8 6 7
67792024-03-06T21:00:41.150Z24/30 Write block 362 len 16384 data: 7 10 9 8
67802024-03-06T21:00:41.150Z25/30 Flush
67812024-03-06T21:00:41.150Z26/30 Write block 478 len 24576 data: 5 3 3 4 4 4
67822024-03-06T21:00:41.150Z27/30 Write block 196 len 16384 data: 4 4 5 4
67832024-03-06T21:00:41.150Z28/30 Read block 117 len 24576
67842024-03-06T21:00:41.150Z29/30 Write block 32 len 12288 data: 6 7 7
67852024-03-06T21:00:41.150Z30/30 Read block 71 len 8192
67862024-03-06T21:00:41.151Z----------------------------------------------------------------
67872024-03-06T21:00:41.151Z Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:30
67882024-03-06T21:00:41.151ZStates: Active Active Active
67892024-03-06T21:00:41.151ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
67902024-03-06T21:00:41.151Z 1 Acked 1000 Write 3 Done Done Done false
67912024-03-06T21:00:41.151Z 2 Acked 1001 Read 4 Done Done Done false
67922024-03-06T21:00:41.152Z 3 Acked 1002 Write 7 Done Done Sent false
67932024-03-06T21:00:41.152Z 4 Acked 1003 Read 5 Done Done Sent false
67942024-03-06T21:00:41.152Z 5 Acked 1004 Write 1 Done Done Sent false
67952024-03-06T21:00:41.152Z 6 Acked 1005 Read 9 Done Done Sent false
67962024-03-06T21:00:41.153Z 7 Acked 1006 Write 2 Done Done Sent false
67972024-03-06T21:00:41.153Z 8 Acked 1007 Write 2 Done Done Sent false
67982024-03-06T21:00:41.153Z 9 Acked 1008 Read 6 Done Done Sent false
67992024-03-06T21:00:41.153Z 10 Acked 1009 Flush 0 Done Done Sent false
68002024-03-06T21:00:41.154Z 11 Acked 1010 Write 2 Done Done Sent false
68012024-03-06T21:00:41.154Z 12 Acked 1011 Write 3 Done Done Sent false
68022024-03-06T21:00:41.155Z 13 Acked 1012 Read 5 Done Done Sent false
68032024-03-06T21:00:41.155Z 14 Acked 1013 Read 7 Done Done Sent false
68042024-03-06T21:00:41.155Z 15 Acked 1014 Write 3 Done Done Sent false
68052024-03-06T21:00:41.156Z 16 Acked 1015 Write 6 Done Done Sent false
68062024-03-06T21:00:41.156Z 17 Acked 1016 Write 9 Done Done Sent false
68072024-03-06T21:00:41.156Z 18 Acked 1017 Read 2 Done Done Sent false
68082024-03-06T21:00:41.156Z 19 Acked 1018 Read 5 Done Done Sent false
68092024-03-06T21:00:41.156Z 20 Acked 1019 Read 6 Done Done Sent false
68102024-03-06T21:00:41.156Z 21 Acked 1020 Write 2 Done Done Sent false
68112024-03-06T21:00:41.156Z 22 Acked 1021 Read 3 Done Done Sent false
68122024-03-06T21:00:41.157Z 23 Acked 1022 Write 7 Done Done Sent false
68132024-03-06T21:00:41.157Z 24 Acked 1023 Write 4 Done Done Sent false
68142024-03-06T21:00:41.157Z 25 Acked 1024 Flush 0 Done Done Sent false
68152024-03-06T21:00:41.157Z 26 Acked 1025 Write 6 Done Done Sent false
68162024-03-06T21:00:41.157Z 27 Acked 1026 Write 4 Done Done Sent false
68172024-03-06T21:00:41.157Z 28 Acked 1027 Read 6 Done Done Sent false
68182024-03-06T21:00:41.158Z 29 Acked 1028 Write 3 Sent Done Sent false
68192024-03-06T21:00:41.158Z 30 Acked 1029 Read 2 Sent Done Sent false
68202024-03-06T21:00:41.158Z STATES DS:0 DS:1 DS:2 TOTAL
68212024-03-06T21:00:41.159Z New 0 0 0 0
68222024-03-06T21:00:41.159Z Sent 2 0 28 30
68232024-03-06T21:00:41.159Z Done 28 30 2 60
68242024-03-06T21:00:41.159Z Skipped 0 0 0 0
68252024-03-06T21:00:41.160Z Error 0 0 0 0
68262024-03-06T21:00:41.160ZLast Flush: 1024 1024 0
68272024-03-06T21:00:41.160ZDownstairs last five completed:
68282024-03-06T21:00:41.160ZUpstairs last five completed: 30 29 28 27 26
68292024-03-06T21:00:41.160Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:00:39.853085227Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1387}
68302024-03-06T21:00:41.161ZWrote out file "/tmp/test_repair_verify.data"
68312024-03-06T21:00:41.161Z
68322024-03-06T21:00:41.161ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
68332024-03-06T21:00:41.162ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
68342024-03-06T21:00:41.162Z 0 000-019 61 61 61 102 102 102 F F F
68352024-03-06T21:00:41.162Z 1 020-039 62 62 59 104 104 98 T T F <---
68362024-03-06T21:00:41.162Z 2 040-059 61 61 61 102 102 102 F F F
68372024-03-06T21:00:41.162Z 3 060-079 61 61 61 102 102 102 F F F
68382024-03-06T21:00:41.163Z 4 080-099 61 61 61 102 102 102 F F F
68392024-03-06T21:00:41.163Z 5 100-119 59 59 59 98 98 98 F F F
68402024-03-06T21:00:41.163Z 6 120-139 56 56 56 93 93 93 F F F
68412024-03-06T21:00:41.164Z 7 140-159 62 62 60 104 104 99 F F F <---
68422024-03-06T21:00:41.164Z 8 160-179 60 60 60 99 99 99 F F F
68432024-03-06T21:00:41.164Z 9 180-199 62 62 61 104 104 102 T T F <---
68442024-03-06T21:00:41.165Z 10 200-219 62 62 58 104 104 97 F F F <---
68452024-03-06T21:00:41.165Z 11 220-239 61 61 61 102 102 102 F F F
68462024-03-06T21:00:41.165Z 12 240-259 57 57 57 96 96 96 F F F
68472024-03-06T21:00:41.165Z 13 260-279 62 62 57 104 104 96 F F F <---
68482024-03-06T21:00:41.166Z 14 280-299 62 62 60 103 103 99 F F T <---
68492024-03-06T21:00:41.166Z 15 300-319 56 56 56 93 93 93 F F F
68502024-03-06T21:00:41.166Z 16 320-339 57 57 57 96 96 96 F F F
68512024-03-06T21:00:41.166Z 17 340-359 62 62 58 104 104 97 F F F <---
68522024-03-06T21:00:41.166Z 18 360-379 62 62 60 104 104 99 F F F <---
68532024-03-06T21:00:41.167Z 19 380-399 62 62 58 103 103 97 F F F <---
68542024-03-06T21:00:41.167Z 20 400-419 56 56 56 95 95 95 F F F
68552024-03-06T21:00:41.167Z 21 420-439 56 56 56 95 95 95 F F F
68562024-03-06T21:00:41.167Z 22 440-459 61 61 61 102 102 102 F F F
68572024-03-06T21:00:41.167Z 23 460-479 62 62 59 103 103 98 T T F <---
68582024-03-06T21:00:41.167Z 24 480-499 62 62 58 103 103 97 T T F <---
68592024-03-06T21:00:41.168Z 25 500-519 61 61 61 102 102 102 F F F
68602024-03-06T21:00:41.168Z 26 520-539 61 61 61 102 102 102 F F F
68612024-03-06T21:00:41.168Z 27 540-559 62 62 54 104 104 87 F F F <---
68622024-03-06T21:00:41.168Z 28 560-579 62 62 54 103 103 87 F F F <---
68632024-03-06T21:00:41.168Z 29 580-599 60 60 60 101 101 101 F F F
68642024-03-06T21:00:41.169ZMax gen: 62, Max flush: 104
68652024-03-06T21:00:41.169ZError: Difference in extent metadata found!
68662024-03-06T21:00:41.187ZA Difference in extent metadata is expected here
68672024-03-06T21:00:41.187ZOn loop 31
68682024-03-06T21:00:41.187Z
68692024-03-06T21:00:41.187ZVerifying data now
68702024-03-06T21:00:45.810ZNo change after verify
68712024-03-06T21:00:45.810ZLoop: 31 Downstairs dump after verify (and repair):
68722024-03-06T21:00:45.827ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
68732024-03-06T21:00:45.827Z 0 000-019 61 61 61 102 102 102 F F F
68742024-03-06T21:00:45.827Z 1 020-039 63 63 63 105 105 105 F F F
68752024-03-06T21:00:45.827Z 2 040-059 61 61 61 102 102 102 F F F
68762024-03-06T21:00:45.827Z 3 060-079 61 61 61 102 102 102 F F F
68772024-03-06T21:00:45.827Z 4 080-099 61 61 61 102 102 102 F F F
68782024-03-06T21:00:45.827Z 5 100-119 59 59 59 98 98 98 F F F
68792024-03-06T21:00:45.827Z 6 120-139 56 56 56 93 93 93 F F F
68802024-03-06T21:00:45.827Z 7 140-159 62 62 62 104 104 104 F F F
68812024-03-06T21:00:45.827Z 8 160-179 60 60 60 99 99 99 F F F
68822024-03-06T21:00:45.827Z 9 180-199 63 63 63 105 105 105 F F F
68832024-03-06T21:00:45.827Z 10 200-219 62 62 62 104 104 104 F F F
68842024-03-06T21:00:45.827Z 11 220-239 61 61 61 102 102 102 F F F
68852024-03-06T21:00:45.827Z 12 240-259 57 57 57 96 96 96 F F F
68862024-03-06T21:00:45.827Z 13 260-279 62 62 62 104 104 104 F F F
68872024-03-06T21:00:45.827Z 14 280-299 62 62 62 103 103 103 F F F
68882024-03-06T21:00:45.827Z 15 300-319 56 56 56 93 93 93 F F F
68892024-03-06T21:00:45.827Z 16 320-339 57 57 57 96 96 96 F F F
68902024-03-06T21:00:45.827Z 17 340-359 62 62 62 104 104 104 F F F
68912024-03-06T21:00:45.827Z 18 360-379 62 62 62 104 104 104 F F F
68922024-03-06T21:00:45.827Z 19 380-399 62 62 62 103 103 103 F F F
68932024-03-06T21:00:45.828Z 20 400-419 56 56 56 95 95 95 F F F
68942024-03-06T21:00:45.828Z 21 420-439 56 56 56 95 95 95 F F F
68952024-03-06T21:00:45.828Z 22 440-459 61 61 61 102 102 102 F F F
68962024-03-06T21:00:45.828Z 23 460-479 63 63 63 105 105 105 F F F
68972024-03-06T21:00:45.833Z 24 480-499 63 63 63 105 105 105 F F F
68982024-03-06T21:00:45.833Z 25 500-519 61 61 61 102 102 102 F F F
68992024-03-06T21:00:45.833Z 26 520-539 61 61 61 102 102 102 F F F
69002024-03-06T21:00:45.833Z 27 540-559 62 62 62 104 104 104 F F F
69012024-03-06T21:00:45.833Z 28 560-579 62 62 62 103 103 103 F F F
69022024-03-06T21:00:45.833Z 29 580-599 60 60 60 101 101 101 F F F
69032024-03-06T21:00:45.833ZMax gen: 63, Max flush: 105
69042024-03-06T21:00:45.833Z
69052024-03-06T21:00:45.833ZBegin loop 32
69062024-03-06T21:00:45.833ZDownstairs to restart: 2
69072024-03-06T21:00:45.860Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.615347976Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395}
69082024-03-06T21:00:45.860Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.615597579Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395}
69092024-03-06T21:00:45.861Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.615615238Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395}
69102024-03-06T21:00:45.861Z{"msg":"Crucible 0274cb0f-7f7f-4b58-9583-cde8db94d8f0 has session id: 7cc07446-366a-4e7a-b9dd-00263a34f958","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.615635796Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395Crucible runtime is spawned
69112024-03-06T21:00:45.861Z,"session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69122024-03-06T21:00:45.861Z{"msg":"Upstairs opts: Upstairs UUID: 0274cb0f-7f7f-4b58-9583-cde8db94d8f0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.615690432Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69132024-03-06T21:00:45.871Z{"msg":"Crucible stats registered with UUID: 0274cb0f-7f7f-4b58-9583-cde8db94d8f0","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.615716751Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69142024-03-06T21:00:45.871Z{"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.615757588Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395}
69152024-03-06T21:00:45.871Z{"msg":"0274cb0f-7f7f-4b58-9583-cde8db94d8f0 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.615886769Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69162024-03-06T21:00:45.871Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.615923436Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"client":"0","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69172024-03-06T21:00:45.871Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.615950264Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"client":"0","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69182024-03-06T21:00:45.871Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.615981142Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"client":"1","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69192024-03-06T21:00:45.871Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.616006191Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"client":"1","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69202024-03-06T21:00:45.871Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.616028429Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"client":"2","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69212024-03-06T21:00:45.871Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.616049728Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"client":"2","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69222024-03-06T21:00:45.871Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.616086695Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"":"io task","client":"2","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69232024-03-06T21:00:45.871Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.616118623Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"":"io task","client":"1","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69242024-03-06T21:00:45.871Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.61615854Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"":"io task","client":"0","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69252024-03-06T21:00:45.871Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.616195687Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"":"io task","client":"2","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69262024-03-06T21:00:45.872Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.616362136Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"":"io task","client":"1","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69272024-03-06T21:00:45.872Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.616423742Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"":"io task","client":"0","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69282024-03-06T21:00:45.872Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.617004942Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"client":"1","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69292024-03-06T21:00:45.872Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.617050539Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"client":"1","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69302024-03-06T21:00:45.872Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.617105575Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"client":"0","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69312024-03-06T21:00:45.872Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.617158031Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"client":"0","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69322024-03-06T21:00:45.872Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.617739991Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"client":"0","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69332024-03-06T21:00:45.872Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.617772689Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"client":"1","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69342024-03-06T21:00:45.872Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.61833829Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"client":"1","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69352024-03-06T21:00:45.872Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.618373617Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"client":"1","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69362024-03-06T21:00:45.872Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.618423924Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69372024-03-06T21:00:45.872Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.618438203Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"client":"0","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69382024-03-06T21:00:45.872Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.618459422Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"client":"0","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69392024-03-06T21:00:45.872Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.61847343Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69402024-03-06T21:00:45.872Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.618737032Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"client":"2","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69412024-03-06T21:00:45.872Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.61876889Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"client":"2","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69422024-03-06T21:00:45.872Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.619272915Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"client":"2","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69432024-03-06T21:00:45.872Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.619664179Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"client":"2","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69442024-03-06T21:00:45.872Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.619695416Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"client":"2","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69452024-03-06T21:00:45.872Z{"msg":"[0]R flush_numbers[0..12]: [102, 105, 102, 102, 102, 98, 93, 104, 99, 105, 104, 102]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.619710895Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"":"downstairs","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69462024-03-06T21:00:45.872Z{"msg":"[0]R generation[0..12]: [61, 63, 61, 61, 61, 59, 56, 62, 60, 63, 62, 61]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.619726354Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"":"downstairs","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69472024-03-06T21:00:45.872Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.619760192Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"":"downstairs","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69482024-03-06T21:00:45.872Z{"msg":"[1]R flush_numbers[0..12]: [102, 105, 102, 102, 102, 98, 93, 104, 99, 105, 104, 102]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.61977878Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"":"downstairs","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69492024-03-06T21:00:45.872Z{"msg":"[1]R generation[0..12]: [61, 63, 61, 61, 61, 59, 56, 62, 60, 63, 62, 61]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.61979298Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"":"downstairs","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69502024-03-06T21:00:45.873Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.619817398Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"":"downstairs","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69512024-03-06T21:00:45.873ZWait for a query_work_queue command to finish before sending IO
69522024-03-06T21:00:45.873Z{"msg":"[2]R flush_numbers[0..12]: [102, 105, 102, 102, 102, 98, 93, 104, 99, 105, 104, 102]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.619840656Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"":"downstairs","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69532024-03-06T21:00:45.873Z{"msg":"[2]R generation[0..12]: [61, 63, 61, 61, 61, 59, 56, 62, 60, 63, 62, 61]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.619863295Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"":"downstairs","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69542024-03-06T21:00:45.873Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.619877804Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"":"downstairs","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69552024-03-06T21:00:45.873Z{"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.619898142Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"":"downstairs","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69562024-03-06T21:00:45.873Z{"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.619912181Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"":"downstairs","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69572024-03-06T21:00:45.873Z{"msg":"Next flush: 106","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.61993253Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"":"downstairs","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69582024-03-06T21:00:45.873Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.619948619Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"":"downstairs","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69592024-03-06T21:00:45.873Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.619973037Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69602024-03-06T21:00:45.873Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.619986916Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"client":"0","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69612024-03-06T21:00:45.873Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.620006895Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"client":"1","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69622024-03-06T21:00:45.873Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.620020614Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"client":"2","":"downstairs","session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69632024-03-06T21:00:45.873Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.620041033Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69642024-03-06T21:00:45.873Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.620053582Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69652024-03-06T21:00:45.873Z{"msg":"0274cb0f-7f7f-4b58-9583-cde8db94d8f0 is now active with session: 7cc07446-366a-4e7a-b9dd-00263a34f958","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.62007287Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69662024-03-06T21:00:45.873Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.62008529Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395,"session_id":"7cc07446-366a-4e7a-b9dd-00263a34f958"}
69672024-03-06T21:00:45.873Z{"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:44.620104958Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395}
69682024-03-06T21:00:45.873ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
69692024-03-06T21:00:45.873ZLoading write count information from file "/tmp/test_repair_verify.data"
69702024-03-06T21:00:45.873ZRun Repair workload
69712024-03-06T21:00:45.873Z01/30 Flush
69722024-03-06T21:00:45.874Z02/30 Write block 18 len 28672 data: 4 5 6 5 6 7 6
69732024-03-06T21:00:45.874Z03/30 Read block 238 len 8192
69742024-03-06T21:00:45.874Z04/30 Flush
69752024-03-06T21:00:45.874Z05/30 Write block 300 len 32768 data: 4 3 3 3 4 6 8 8
69762024-03-06T21:00:45.877Z06/30 Read block 8 len 32768
69772024-03-06T21:00:45.877Z07/30 Write block 164 len 4096 data: 3
69782024-03-06T21:00:45.877Z08/30 Write block 228 len 16384 data: 5 5 5 6
69792024-03-06T21:00:45.878Z09/30 Write block 554 len 20480 data: 4 5 4 4 4
69802024-03-06T21:00:45.878Z10/30 Write block 310 len 40960 data: 7 7 7 7 6 6 5 4 5 5
69812024-03-06T21:00:45.878Z11/30 Read block 288 len 32768
69822024-03-06T21:00:45.878Z12/30 Flush
69832024-03-06T21:00:45.878Z13/30 Write block 67 len 40960 data: 12 10 10 7 6 7 6 6 6 5
69842024-03-06T21:00:45.878Z14/30 Read block 342 len 8192
69852024-03-06T21:00:45.878Z15/30 Write block 198 len 16384 data: 6 5 5 4
69862024-03-06T21:00:45.881Z16/30 Write block 433 len 32768 data: 7 8 6 5 5 5 5 5
69872024-03-06T21:00:45.881Z17/30 Read block 560 len 4096
69882024-03-06T21:00:45.881Z18/30 Flush
69892024-03-06T21:00:45.884Z19/30 Write block 371 len 28672 data: 8 8 7 7 7 4 4
69902024-03-06T21:00:45.884Z20/30 Write block 380 len 40960 data: 4 3 5 5 8 8 9 7 5 5
69912024-03-06T21:00:45.885Z21/30 Read block 518 len 16384
69922024-03-06T21:00:45.891Z22/30 Write block 337 len 8192 data: 4 6
69932024-03-06T21:00:45.891Z23/30 Write block 234 len 40960 data: 5 7 8 9 9 9 8 9 7 5
69942024-03-06T21:00:45.891Z24/30 Read block 372 len 40960
69952024-03-06T21:00:45.894Z25/30 Read block 455 len 12288
69962024-03-06T21:00:45.894Z26/30 Read block 206 len 16384
69972024-03-06T21:00:45.894Z27/30 Read block 496 len 28672
69982024-03-06T21:00:45.895Z28/30 Read block 448 len 32768
69992024-03-06T21:00:45.895Z29/30 Write block 373 len 32768 data: 8 8 8 5 5 4 5 5
70002024-03-06T21:00:45.895Z30/30 Read block 505 len 8192
70012024-03-06T21:00:45.898Z----------------------------------------------------------------
70022024-03-06T21:00:45.898Z Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30
70032024-03-06T21:00:45.898ZStates: Active Active Active
70042024-03-06T21:00:45.898ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
70052024-03-06T21:00:45.899Z 1 Acked 1000 Flush 0 Done Done Sent false
70062024-03-06T21:00:45.899Z 2 Acked 1001 Write 7 Done Done Sent false
70072024-03-06T21:00:45.899Z 3 Acked 1002 Read 2 Done Done Sent false
70082024-03-06T21:00:45.899Z 4 Acked 1003 Flush 0 Done Done Sent false
70092024-03-06T21:00:45.899Z 5 Acked 1004 Write 8 Done Done Sent false
70102024-03-06T21:00:45.899Z 6 Acked 1005 Read 8 Done Done Sent false
70112024-03-06T21:00:45.899Z 7 Acked 1006 Write 1 Done Done Sent false
70122024-03-06T21:00:45.899Z 8 Acked 1007 Write 4 Done Done Sent false
70132024-03-06T21:00:45.899Z 9 Acked 1008 Write 5 Done Done Sent false
70142024-03-06T21:00:45.899Z 10 Acked 1009 Write 10 Done Done Sent false
70152024-03-06T21:00:45.899Z 11 Acked 1010 Read 8 Done Done Sent false
70162024-03-06T21:00:45.899Z 12 Acked 1011 Flush 0 Done Done Sent false
70172024-03-06T21:00:45.899Z 13 Acked 1012 Write 10 Done Done Sent false
70182024-03-06T21:00:45.899Z 14 Acked 1013 Read 2 Done Done Sent false
70192024-03-06T21:00:45.899Z 15 Acked 1014 Write 4 Done Done Sent false
70202024-03-06T21:00:45.899Z 16 Acked 1015 Write 8 Done Done Sent false
70212024-03-06T21:00:45.899Z 17 Acked 1016 Read 1 Done Done Sent false
70222024-03-06T21:00:45.899Z 18 Acked 1017 Flush 0 Done Done Sent false
70232024-03-06T21:00:45.899Z 19 Acked 1018 Write 7 Done Done Sent false
70242024-03-06T21:00:45.899Z 20 Acked 1019 Write 10 Done Done Sent false
70252024-03-06T21:00:45.899Z 21 Acked 1020 Read 4 Done Done Sent false
70262024-03-06T21:00:45.899Z 22 Acked 1021 Write 2 Done Done Sent false
70272024-03-06T21:00:45.899Z 23 Acked 1022 Write 10 Done Done Sent false
70282024-03-06T21:00:45.899Z 24 Acked 1023 Read 10 Done Done Sent false
70292024-03-06T21:00:45.900Z 25 Acked 1024 Read 3 Done Done Sent false
70302024-03-06T21:00:45.900Z 26 Acked 1025 Read 4 Done Done Sent false
70312024-03-06T21:00:45.900Z 27 Acked 1026 Read 7 Done Done Sent false
70322024-03-06T21:00:45.900Z 28 Acked 1027 Read 8 Done Done Sent false
70332024-03-06T21:00:45.900Z 29 Acked 1028 Write 8 Done Sent Sent false
70342024-03-06T21:00:45.900Z 30 Acked 1029 Read 2 Done Sent Sent false
70352024-03-06T21:00:45.900Z STATES DS:0 DS:1 DS:2 TOTAL
70362024-03-06T21:00:45.900Z New 0 0 0 0
70372024-03-06T21:00:45.900Z Sent 0 2 30 32
70382024-03-06T21:00:45.900Z Done 30 28 0 58
70392024-03-06T21:00:45.900Z Skipped 0 0 0 0
70402024-03-06T21:00:45.900Z Error 0 0 0 0
70412024-03-06T21:00:45.900ZLast Flush: 1017 1017 0
70422024-03-06T21:00:45.900ZDownstairs last five completed:
70432024-03-06T21:00:45.900ZUpstairs last five completed: 30 29 28 27 26
70442024-03-06T21:00:45.900Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:00:44.6583822Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1395}
70452024-03-06T21:00:45.900ZWrote out file "/tmp/test_repair_verify.data"
70462024-03-06T21:00:45.904Z
70472024-03-06T21:00:45.907ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
70482024-03-06T21:00:45.922ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70492024-03-06T21:00:45.922Z 0 000-019 64 64 61 107 107 102 F F F <---
70502024-03-06T21:00:45.923Z 1 020-039 64 64 63 107 107 105 F F F <---
70512024-03-06T21:00:45.923Z 2 040-059 61 61 61 102 102 102 F F F
70522024-03-06T21:00:45.923Z 3 060-079 64 64 61 109 109 102 F F F <---
70532024-03-06T21:00:45.923Z 4 080-099 61 61 61 102 102 102 F F F
70542024-03-06T21:00:45.923Z 5 100-119 59 59 59 98 98 98 F F F
70552024-03-06T21:00:45.923Z 6 120-139 56 56 56 93 93 93 F F F
70562024-03-06T21:00:45.923Z 7 140-159 62 62 62 104 104 104 F F F
70572024-03-06T21:00:45.923Z 8 160-179 64 64 60 108 108 99 F F F <---
70582024-03-06T21:00:45.923Z 9 180-199 64 64 63 109 109 105 F F F <---
70592024-03-06T21:00:45.923Z 10 200-219 64 64 62 109 109 104 F F F <---
70602024-03-06T21:00:45.923Z 11 220-239 64 64 61 108 108 102 T T F <---
70612024-03-06T21:00:45.923Z 12 240-259 57 57 57 96 96 96 T T F <---
70622024-03-06T21:00:45.923Z 13 260-279 62 62 62 104 104 104 F F F
70632024-03-06T21:00:45.923Z 14 280-299 62 62 62 103 103 103 F F F
70642024-03-06T21:00:45.923Z 15 300-319 64 64 56 108 108 93 F F F <---
70652024-03-06T21:00:45.923Z 16 320-339 57 57 57 96 96 96 T T F <---
70662024-03-06T21:00:45.923Z 17 340-359 62 62 62 104 104 104 F F F
70672024-03-06T21:00:45.923Z 18 360-379 62 62 62 104 104 104 T T F <---
70682024-03-06T21:00:45.923Z 19 380-399 62 62 62 103 103 103 T T F <---
70692024-03-06T21:00:45.923Z 20 400-419 56 56 56 95 95 95 F F F
70702024-03-06T21:00:45.923Z 21 420-439 64 64 56 109 109 95 F F F <---
70712024-03-06T21:00:45.927Z 22 440-459 64 64 61 109 109 102 F F F <---
70722024-03-06T21:00:45.927Z 23 460-479 63 63 63 105 105 105 F F F
70732024-03-06T21:00:45.927Z 24 480-499 63 63 63 105 105 105 F F F
70742024-03-06T21:00:45.927Z 25 500-519 61 61 61 102 102 102 F F F
70752024-03-06T21:00:45.927Z 26 520-539 61 61 61 102 102 102 F F F
70762024-03-06T21:00:45.927Z 27 540-559 64 64 62 108 108 104 F F F <---
70772024-03-06T21:00:45.927Z 28 560-579 62 62 62 103 103 103 F F F
70782024-03-06T21:00:45.927Z 29 580-599 60 60 60 101 101 101 F F F
70792024-03-06T21:00:45.927ZMax gen: 64, Max flush: 109
70802024-03-06T21:00:45.927ZError: Difference in extent metadata found!
70812024-03-06T21:00:45.927ZA Difference in extent metadata is expected here
70822024-03-06T21:00:45.927ZOn loop 32
70832024-03-06T21:00:45.927Z
70842024-03-06T21:00:45.931ZVerifying data now
70852024-03-06T21:00:50.767ZNo change after verify
70862024-03-06T21:00:50.767ZLoop: 32 Downstairs dump after verify (and repair):
70872024-03-06T21:00:52.129ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70882024-03-06T21:00:52.129Z 0 000-019 64 64 64 107 107 107 F F F
70892024-03-06T21:00:52.129Z 1 020-039 64 64 64 107 107 107 F F F
70902024-03-06T21:00:52.130Z 2 040-059 61 61 61 102 102 102 F F F
70912024-03-06T21:00:52.130Z 3 060-079 64 64 64 109 109 109 F F F
70922024-03-06T21:00:52.130Z 4 080-099 61 61 61 102 102 102 F F F
70932024-03-06T21:00:52.130Z 5 100-119 59 59 59 98 98 98 F F F
70942024-03-06T21:00:52.130Z 6 120-139 56 56 56 93 93 93 F F F
70952024-03-06T21:00:52.130Z 7 140-159 62 62 62 104 104 104 F F F
70962024-03-06T21:00:52.130Z 8 160-179 64 64 64 108 108 108 F F F
70972024-03-06T21:00:52.131Z 9 180-199 64 64 64 109 109 109 F F F
70982024-03-06T21:00:52.131Z 10 200-219 64 64 64 109 109 109 F F F
70992024-03-06T21:00:52.131Z 11 220-239 65 65 65 110 110 110 F F F
71002024-03-06T21:00:52.131Z 12 240-259 65 65 65 110 110 110 F F F
71012024-03-06T21:00:52.131Z 13 260-279 62 62 62 104 104 104 F F F
71022024-03-06T21:00:52.131Z 14 280-299 62 62 62 103 103 103 F F F
71032024-03-06T21:00:52.131Z 15 300-319 64 64 64 108 108 108 F F F
71042024-03-06T21:00:52.131Z 16 320-339 65 65 65 110 110 110 F F F
71052024-03-06T21:00:52.131Z 17 340-359 62 62 62 104 104 104 F F F
71062024-03-06T21:00:52.131Z 18 360-379 65 65 65 110 110 110 F F F
71072024-03-06T21:00:52.131Z 19 380-399 65 65 65 110 110 110 F F F
71082024-03-06T21:00:52.131Z 20 400-419 56 56 56 95 95 95 F F F
71092024-03-06T21:00:52.131Z 21 420-439 64 64 64 109 109 109 F F F
71102024-03-06T21:00:52.131Z 22 440-459 64 64 64 109 109 109 F F F
71112024-03-06T21:00:52.132Z 23 460-479 63 63 63 105 105 105 F F F
71122024-03-06T21:00:52.132Z 24 480-499 63 63 63 105 105 105 F F F
71132024-03-06T21:00:52.132Z 25 500-519 61 61 61 102 102 102 F F F
71142024-03-06T21:00:52.132Z 26 520-539 61 61 61 102 102 102 F F F
71152024-03-06T21:00:52.132Z 27 540-559 64 64 64 108 108 108 F F F
71162024-03-06T21:00:52.132Z 28 560-579 62 62 62 103 103 103 F F F
71172024-03-06T21:00:52.132Z 29 580-599 60 60 60 101 101 101 F F F
71182024-03-06T21:00:52.132ZMax gen: 65, Max flush: 110
71192024-03-06T21:00:52.132Z
71202024-03-06T21:00:52.132ZBegin loop 33
71212024-03-06T21:00:52.132ZDownstairs to restart: 1
71222024-03-06T21:00:52.132Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.514472534Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403}
71232024-03-06T21:00:52.132Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.514732906Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403}
71242024-03-06T21:00:52.132Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.514750775Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403}
71252024-03-06T21:00:52.132Z{"msg":"Crucible f4141ed5-a1cb-4676-af7c-ac2c162299cc has session id: 1217f304-7c86-4f54-ba9b-5a093485de9c","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.514765314Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71262024-03-06T21:00:52.132Z{"msg":"Upstairs opts: Upstairs UUID: f4141ed5-a1cb-4676-af7c-ac2c162299cc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.51481213Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71272024-03-06T21:00:52.133Z{"msg":"Crucible stats registered with UUID: f4141ed5-a1cb-4676-af7c-ac2c162299cc","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.514835289Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71282024-03-06T21:00:52.133ZCrucible runtime is spawned
71292024-03-06T21:00:52.133Z{"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.51496338Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403}
71302024-03-06T21:00:52.133Z{"msg":"f4141ed5-a1cb-4676-af7c-ac2c162299cc active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.51510745Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71312024-03-06T21:00:52.133Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.515135928Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"client":"0","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71322024-03-06T21:00:52.133Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.515180805Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"client":"0","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71332024-03-06T21:00:52.133Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.515204063Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"client":"1","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71342024-03-06T21:00:52.133Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.515220042Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"client":"1","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71352024-03-06T21:00:52.133Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.515238771Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"client":"2","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71362024-03-06T21:00:52.133Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.515271219Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"client":"2","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71372024-03-06T21:00:52.133Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.515290897Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"":"io task","client":"2","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71382024-03-06T21:00:52.133Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.515318955Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"":"io task","client":"0","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71392024-03-06T21:00:52.134Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.515345674Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"":"io task","client":"1","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71402024-03-06T21:00:52.134Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.515488594Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"":"io task","client":"0","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71412024-03-06T21:00:52.134Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.51554512Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"":"io task","client":"2","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71422024-03-06T21:00:52.134Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.515657302Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"":"io task","client":"1","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71432024-03-06T21:00:52.134Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.51627554Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"client":"2","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71442024-03-06T21:00:52.134Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.516321636Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"client":"2","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71452024-03-06T21:00:52.134Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.516347455Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"client":"0","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71462024-03-06T21:00:52.134Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.51641483Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"client":"0","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71472024-03-06T21:00:52.134Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.516879328Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"client":"1","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71482024-03-06T21:00:52.134Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.516939904Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"client":"1","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71492024-03-06T21:00:52.134Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.517088524Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"client":"2","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71502024-03-06T21:00:52.134Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.517122481Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"client":"0","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71512024-03-06T21:00:52.135Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.517546412Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"client":"1","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71522024-03-06T21:00:52.135Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.517756367Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"client":"2","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71532024-03-06T21:00:52.135Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.517797105Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"client":"2","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71542024-03-06T21:00:52.135Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.517823813Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71552024-03-06T21:00:52.135Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.517853891Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"client":"0","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71562024-03-06T21:00:52.135Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.517877659Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"client":"0","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71572024-03-06T21:00:52.135Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.517905917Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71582024-03-06T21:00:52.135Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.51801581Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"client":"1","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71592024-03-06T21:00:52.135Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.518034538Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"client":"1","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71602024-03-06T21:00:52.135Z{"msg":"[0]R flush_numbers[0..12]: [107, 107, 102, 109, 102, 98, 93, 104, 108, 109, 109, 110]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.518085275Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"":"downstairs","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71612024-03-06T21:00:52.135Z{"msg":"[0]R generation[0..12]: [64, 64, 61, 64, 61, 59, 56, 62, 64, 64, 64, 65]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.518137671Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"":"downstairs","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71622024-03-06T21:00:52.135Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.518166929Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"":"downstairs","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71632024-03-06T21:00:52.136Z{"msg":"[1]R flush_numbers[0..12]: [107, 107, 102, 109, 102, 98, 93, 104, 108, 109, 109, 110]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.518181908Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"":"downstairs","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71642024-03-06T21:00:52.136Z{"msg":"[1]R generation[0..12]: [64, 64, 61, 64, 61, 59, 56, 62, 64, 64, 64, 65]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.518202687Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"":"downstairs","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71652024-03-06T21:00:52.136Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.518217056Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB"Wait for a query_work_queue command to finish before sending IO
71662024-03-06T21:00:52.136Z,"pid":1403,"":"downstairs","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71672024-03-06T21:00:52.136Z{"msg":"[2]R flush_numbers[0..12]: [107, 107, 102, 109, 102, 98, 93, 104, 108, 109, 109, 110]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.518235005Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"":"downstairs","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71682024-03-06T21:00:52.136Z{"msg":"[2]R generation[0..12]: [64, 64, 61, 64, 61, 59, 56, 62, 64, 64, 64, 65]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.518249044Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"":"downstairs","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71692024-03-06T21:00:52.136Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.518270602Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"":"downstairs","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71702024-03-06T21:00:52.136Z{"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.518284191Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"":"downstairs","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71712024-03-06T21:00:52.136Z{"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.51830485Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"":"downstairs","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71722024-03-06T21:00:52.136Z{"msg":"Next flush: 111","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.518318199Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"":"downstairs","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71732024-03-06T21:00:52.136Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.518331268Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"":"downstairs","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71742024-03-06T21:00:52.137Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.518346177Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71752024-03-06T21:00:52.137Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.518358536Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"client":"0","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71762024-03-06T21:00:52.137Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.518400133Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"client":"1","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71772024-03-06T21:00:52.137Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.518424741Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"client":"2","":"downstairs","session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71782024-03-06T21:00:52.137Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.51845087Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71792024-03-06T21:00:52.137Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.518485307Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71802024-03-06T21:00:52.137Z{"msg":"f4141ed5-a1cb-4676-af7c-ac2c162299cc is now active with session: 1217f304-7c86-4f54-ba9b-5a093485de9c","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.518511735Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71812024-03-06T21:00:52.138Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.518534044Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403,"session_id":"1217f304-7c86-4f54-ba9b-5a093485de9c"}
71822024-03-06T21:00:52.138Z{"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:49.518567002Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403}
71832024-03-06T21:00:52.138ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
71842024-03-06T21:00:52.138ZLoading write count information from file "/tmp/test_repair_verify.data"
71852024-03-06T21:00:52.138ZRun Repair workload
71862024-03-06T21:00:52.138Z01/30 Write block 585 len 12288 data: 7 7 9
71872024-03-06T21:00:52.187Z02/30 Read block 455 len 32768
71882024-03-06T21:00:52.187Z03/30 Read block 484 len 8192
71892024-03-06T21:00:52.187Z04/30 Read block 154 len 16384
71902024-03-06T21:00:52.187Z05/30 Write block 380 len 16384 data: 6 4 6 6
71912024-03-06T21:00:52.187Z06/30 Read block 558 len 36864
71922024-03-06T21:00:52.187Z07/30 Write block 408 len 4096 data: 6
71932024-03-06T21:00:52.187Z08/30 Write block 3 len 36864 data: 3 4 5 5 6 5 5 4 7
71942024-03-06T21:00:52.188Z09/30 Read block 413 len 4096
71952024-03-06T21:00:52.188Z10/30 Write block 336 len 24576 data: 4 5 7 6 7 7
71962024-03-06T21:00:52.188Z11/30 Write block 193 len 28672 data: 3 3 3 5 5 7 6
71972024-03-06T21:00:52.188Z12/30 Write block 381 len 8192 data: 5 7
71982024-03-06T21:00:52.188Z13/30 Read block 185 len 24576
71992024-03-06T21:00:52.188Z14/30 Write block 392 len 12288 data: 5 5 4
72002024-03-06T21:00:52.189Z15/30 Write block 39 len 40960 data: 4 4 5 6 6 5 5 6 7 6
72012024-03-06T21:00:52.189Z16/30 Read block 181 len 8192
72022024-03-06T21:00:52.189Z17/30 Read block 450 len 16384
72032024-03-06T21:00:52.189Z18/30 Write block 437 len 28672 data: 6 6 6 6 3 2 2
72042024-03-06T21:00:52.189Z19/30 Read block 574 len 40960
72052024-03-06T21:00:52.189Z20/30 Write block 222 len 36864 data: 3 5 5 5 6 6 6 6 6
72062024-03-06T21:00:52.189Z21/30 Write block 322 len 20480 data: 4 3 3 3 3
72072024-03-06T21:00:52.189Z22/30 Flush
72082024-03-06T21:00:52.189Z23/30 Write block 255 len 28672 data: 4 3 3 4 4 3 4
72092024-03-06T21:00:52.189Z24/30 Read block 451 len 32768
72102024-03-06T21:00:52.189Z25/30 Write block 552 len 8192 data: 4 3
72112024-03-06T21:00:52.189Z26/30 Read block 410 len 20480
72122024-03-06T21:00:52.189Z27/30 Read block 432 len 16384
72132024-03-06T21:00:52.189Z28/30 Write block 361 len 28672 data: 9 8 11 10 9 7 8
72142024-03-06T21:00:52.189Z29/30 Read block 382 len 40960
72152024-03-06T21:00:52.189Z30/30 Read block 209 len 32768
72162024-03-06T21:00:52.189Z----------------------------------------------------------------
72172024-03-06T21:00:52.189Z Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:30
72182024-03-06T21:00:52.189ZStates: Active Active Active
72192024-03-06T21:00:52.190ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
72202024-03-06T21:00:52.190Z 1 Acked 1000 Write 3 Done Sent Done false
72212024-03-06T21:00:52.190Z 2 Acked 1001 Read 8 Done Sent Done false
72222024-03-06T21:00:52.190Z 3 Acked 1002 Read 2 Done Sent Done false
72232024-03-06T21:00:52.190Z 4 Acked 1003 Read 4 Done Sent Done false
72242024-03-06T21:00:52.190Z 5 Acked 1004 Write 4 Done Sent Done false
72252024-03-06T21:00:52.190Z 6 Acked 1005 Read 9 Done Sent Done false
72262024-03-06T21:00:52.190Z 7 Acked 1006 Write 1 Done Sent Done false
72272024-03-06T21:00:52.190Z 8 Acked 1007 Write 9 Done Sent Done false
72282024-03-06T21:00:52.190Z 9 Acked 1008 Read 1 Done Sent Done false
72292024-03-06T21:00:52.190Z 10 Acked 1009 Write 6 Done Sent Done false
72302024-03-06T21:00:52.190Z 11 Acked 1010 Write 7 Done Sent Done false
72312024-03-06T21:00:52.190Z 12 Acked 1011 Write 2 Done Sent Done false
72322024-03-06T21:00:52.190Z 13 Acked 1012 Read 6 Done Sent Done false
72332024-03-06T21:00:52.190Z 14 Acked 1013 Write 3 Done Sent Done false
72342024-03-06T21:00:52.190Z 15 Acked 1014 Write 10 Done Sent Done false
72352024-03-06T21:00:52.190Z 16 Acked 1015 Read 2 Done Sent Done false
72362024-03-06T21:00:52.190Z 17 Acked 1016 Read 4 Done Sent Done false
72372024-03-06T21:00:52.190Z 18 Acked 1017 Write 7 Done Sent Done false
72382024-03-06T21:00:52.190Z 19 Acked 1018 Read 10 Done Sent Done false
72392024-03-06T21:00:52.190Z 20 Acked 1019 Write 9 Done Sent Done false
72402024-03-06T21:00:52.190Z 21 Acked 1020 Write 5 Done Sent Done false
72412024-03-06T21:00:52.190Z 22 Acked 1021 Flush 0 Done Sent Done false
72422024-03-06T21:00:52.190Z 23 Acked 1022 Write 7 Done Sent Done false
72432024-03-06T21:00:52.191Z 24 Acked 1023 Read 8 Done Sent Done false
72442024-03-06T21:00:52.191Z 25 Acked 1024 Write 2 Done Sent Done false
72452024-03-06T21:00:52.191Z 26 Acked 1025 Read 5 Done Sent Done false
72462024-03-06T21:00:52.191Z 27 Acked 1026 Read 4 Done Sent Done false
72472024-03-06T21:00:52.191Z 28 Acked 1027 Write 7 Done Sent Done false
72482024-03-06T21:00:52.191Z 29 Acked 1028 Read 10 Done Sent Done false
72492024-03-06T21:00:52.191Z 30 Acked 1029 Read 8 Done Sent Done false
72502024-03-06T21:00:52.191Z STATES DS:0 DS:1 DS:2 TOTAL
72512024-03-06T21:00:52.191Z New 0 0 0 0
72522024-03-06T21:00:52.191Z Sent 0 30 0 30
72532024-03-06T21:00:52.191Z Done 30 0 30 60
72542024-03-06T21:00:52.191Z Skipped 0 0 0 0
72552024-03-06T21:00:52.191Z Error 0 0 0 0
72562024-03-06T21:00:52.191ZLast Flush: 1021 0 1021
72572024-03-06T21:00:52.191ZDownstairs last five completed:
72582024-03-06T21:00:52.191ZUpstairs last five completed: 30 29 28 27 26
72592024-03-06T21:00:52.191Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:00:49.553940884Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1403}
72602024-03-06T21:00:52.192ZWrote out file "/tmp/test_repair_verify.data"
72612024-03-06T21:00:52.192Z
72622024-03-06T21:00:52.192ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
72632024-03-06T21:00:52.192ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
72642024-03-06T21:00:52.192Z 0 000-019 66 64 66 111 107 111 F F F <---
72652024-03-06T21:00:52.192Z 1 020-039 66 64 66 111 107 111 F F F <---
72662024-03-06T21:00:52.192Z 2 040-059 66 61 66 111 102 111 F F F <---
72672024-03-06T21:00:52.192Z 3 060-079 64 64 64 109 109 109 F F F
72682024-03-06T21:00:52.192Z 4 080-099 61 61 61 102 102 102 F F F
72692024-03-06T21:00:52.192Z 5 100-119 59 59 59 98 98 98 F F F
72702024-03-06T21:00:52.192Z 6 120-139 56 56 56 93 93 93 F F F
72712024-03-06T21:00:52.192Z 7 140-159 62 62 62 104 104 104 F F F
72722024-03-06T21:00:52.192Z 8 160-179 64 64 64 108 108 108 F F F
72732024-03-06T21:00:52.192Z 9 180-199 66 64 66 111 109 111 F F F <---
72742024-03-06T21:00:52.192Z 10 200-219 64 64 64 109 109 109 F F F
72752024-03-06T21:00:52.192Z 11 220-239 66 65 66 111 110 111 F F F <---
72762024-03-06T21:00:52.193Z 12 240-259 65 65 65 110 110 110 T F T <---
72772024-03-06T21:00:52.193Z 13 260-279 62 62 62 104 104 104 T F T <---
72782024-03-06T21:00:52.193Z 14 280-299 62 62 62 103 103 103 F F F
72792024-03-06T21:00:52.193Z 15 300-319 64 64 64 108 108 108 F F F
72802024-03-06T21:00:52.193Z 16 320-339 66 65 66 111 110 111 F F F <---
72812024-03-06T21:00:52.193Z 17 340-359 66 62 66 111 104 111 F F F <---
72822024-03-06T21:00:52.193Z 18 360-379 65 65 65 110 110 110 T F T <---
72832024-03-06T21:00:52.193Z 19 380-399 66 65 66 111 110 111 F F F <---
72842024-03-06T21:00:52.193Z 20 400-419 66 56 66 111 95 111 F F F <---
72852024-03-06T21:00:52.193Z 21 420-439 66 64 66 111 109 111 F F F <---
72862024-03-06T21:00:52.193Z 22 440-459 66 64 66 111 109 111 F F F <---
72872024-03-06T21:00:52.215Z 23 460-479 63 63 63 105 105 105 F F F
72882024-03-06T21:00:52.215Z 24 480-499 63 63 63 105 105 105 F F F
72892024-03-06T21:00:52.216Z 25 500-519 61 61 61 102 102 102 F F F
72902024-03-06T21:00:52.216Z 26 520-539 61 61 61 102 102 102 F F F
72912024-03-06T21:00:52.216Z 27 540-559 64 64 64 108 108 108 T F T <---
72922024-03-06T21:00:52.216Z 28 560-579 62 62 62 103 103 103 F F F
72932024-03-06T21:00:52.216Z 29 580-599 66 60 66 111 101 111 F F F <---
72942024-03-06T21:00:52.216ZMax gen: 66, Max flush: 111
72952024-03-06T21:00:52.216ZError: Difference in extent metadata found!
72962024-03-06T21:00:52.217ZA Difference in extent metadata is expected here
72972024-03-06T21:00:52.217ZOn loop 33
72982024-03-06T21:00:52.217Z
72992024-03-06T21:00:52.217ZVerifying data now
73002024-03-06T21:00:55.642ZNo change after verify
73012024-03-06T21:00:55.642ZLoop: 33 Downstairs dump after verify (and repair):
73022024-03-06T21:00:55.670ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
73032024-03-06T21:00:55.670Z 0 000-019 66 66 66 111 111 111 F F F
73042024-03-06T21:00:55.670Z 1 020-039 66 66 66 111 111 111 F F F
73052024-03-06T21:00:55.670Z 2 040-059 66 66 66 111 111 111 F F F
73062024-03-06T21:00:55.670Z 3 060-079 64 64 64 109 109 109 F F F
73072024-03-06T21:00:55.671Z 4 080-099 61 61 61 102 102 102 F F F
73082024-03-06T21:00:55.671Z 5 100-119 59 59 59 98 98 98 F F F
73092024-03-06T21:00:55.671Z 6 120-139 56 56 56 93 93 93 F F F
73102024-03-06T21:00:55.671Z 7 140-159 62 62 62 104 104 104 F F F
73112024-03-06T21:00:55.671Z 8 160-179 64 64 64 108 108 108 F F F
73122024-03-06T21:00:55.671Z 9 180-199 66 66 66 111 111 111 F F F
73132024-03-06T21:00:55.671Z 10 200-219 64 64 64 109 109 109 F F F
73142024-03-06T21:00:55.671Z 11 220-239 66 66 66 111 111 111 F F F
73152024-03-06T21:00:55.671Z 12 240-259 67 67 67 112 112 112 F F F
73162024-03-06T21:00:55.672Z 13 260-279 67 67 67 112 112 112 F F F
73172024-03-06T21:00:55.672Z 14 280-299 62 62 62 103 103 103 F F F
73182024-03-06T21:00:55.696Z 15 300-319 64 64 64 108 108 108 F F F
73192024-03-06T21:00:55.697Z 16 320-339 66 66 66 111 111 111 F F F
73202024-03-06T21:00:55.697Z 17 340-359 66 66 66 111 111 111 F F F
73212024-03-06T21:00:55.697Z 18 360-379 67 67 67 112 112 112 F F F
73222024-03-06T21:00:55.697Z 19 380-399 66 66 66 111 111 111 F F F
73232024-03-06T21:00:55.697Z 20 400-419 66 66 66 111 111 111 F F F
73242024-03-06T21:00:55.697Z 21 420-439 66 66 66 111 111 111 F F F
73252024-03-06T21:00:55.697Z 22 440-459 66 66 66 111 111 111 F F F
73262024-03-06T21:00:55.697Z 23 460-479 63 63 63 105 105 105 F F F
73272024-03-06T21:00:55.698Z 24 480-499 63 63 63 105 105 105 F F F
73282024-03-06T21:00:55.698Z 25 500-519 61 61 61 102 102 102 F F F
73292024-03-06T21:00:55.698Z 26 520-539 61 61 61 102 102 102 F F F
73302024-03-06T21:00:55.698Z 27 540-559 67 67 67 112 112 112 F F F
73312024-03-06T21:00:55.698Z 28 560-579 62 62 62 103 103 103 F F F
73322024-03-06T21:00:55.698Z 29 580-599 66 66 66 111 111 111 F F F
73332024-03-06T21:00:55.699ZMax gen: 67, Max flush: 112
73342024-03-06T21:00:55.699Z
73352024-03-06T21:00:55.699ZBegin loop 34
73362024-03-06T21:00:55.699ZDownstairs to restart: 2
73372024-03-06T21:00:55.699Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.448840482Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411}
73382024-03-06T21:00:55.699Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.449137302Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411}
73392024-03-06T21:00:55.699Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.44915965Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411}
73402024-03-06T21:00:55.699Z{"msg":"Crucible 8de56f4f-bd25-45d6-ac41-6e0c85e7fd35 has session id: 327631a0-c78a-4dfa-94bb-33241ca00e08","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.449196307Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73412024-03-06T21:00:55.699Z{"Crucible runtime is spawned
73422024-03-06T21:00:55.699Zmsg":"Upstairs opts: Upstairs UUID: 8de56f4f-bd25-45d6-ac41-6e0c85e7fd35, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.449251794Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73432024-03-06T21:00:55.699Z{"msg":"Crucible stats registered with UUID: 8de56f4f-bd25-45d6-ac41-6e0c85e7fd35","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.44929612Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73442024-03-06T21:00:55.699Z{"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.449338348Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411}
73452024-03-06T21:00:55.700Z{"msg":"8de56f4f-bd25-45d6-ac41-6e0c85e7fd35 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.449555653Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73462024-03-06T21:00:55.700Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.44959895Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"client":"0","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73472024-03-06T21:00:55.700Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.449629728Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"client":"0","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73482024-03-06T21:00:55.700Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.449661865Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"client":"1","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73492024-03-06T21:00:55.700Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.449681164Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"client":"1","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73502024-03-06T21:00:55.700Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.449716642Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"client":"2","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73512024-03-06T21:00:55.700Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.449749139Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"client":"2","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73522024-03-06T21:00:55.700Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.449787857Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"":"io task","client":"0","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73532024-03-06T21:00:55.700Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.449849382Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"":"io task","client":"1","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73542024-03-06T21:00:55.700Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.449896639Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"":"io task","client":"2","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73552024-03-06T21:00:55.700Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.449926617Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"":"io task","client":"1","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73562024-03-06T21:00:55.700Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.449954865Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"":"io task","client":"0","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73572024-03-06T21:00:55.701Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.450136443Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"":"io task","client":"2","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73582024-03-06T21:00:55.701Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.450655467Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"client":"0","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73592024-03-06T21:00:55.701Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.450707753Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"client":"0","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73602024-03-06T21:00:55.701Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.450731392Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"client":"1","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73612024-03-06T21:00:55.701Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.450768519Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"client":"1","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73622024-03-06T21:00:55.701Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.451248106Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"client":"2","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73632024-03-06T21:00:55.701Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.451296563Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"client":"2","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73642024-03-06T21:00:55.701Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.4513308Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"client":"1","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73652024-03-06T21:00:55.701Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.451433963Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"client":"0","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73662024-03-06T21:00:55.725Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.451959827Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"client":"2","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73672024-03-06T21:00:55.726Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.452044371Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"client":"1","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73682024-03-06T21:00:55.726Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.452081539Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"client":"1","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73692024-03-06T21:00:55.726Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.452097207Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73702024-03-06T21:00:55.726Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.452109707Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"client":"0","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73712024-03-06T21:00:55.726Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.452131295Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"client":"0","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73722024-03-06T21:00:55.726Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.452145134Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73732024-03-06T21:00:55.726Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.452398367Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"client":"2","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73742024-03-06T21:00:55.726Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.452444994Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"client":"2","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73752024-03-06T21:00:55.726Z{"msg":"[0]R flush_numbers[0..12]: [111, 111, 111, 109, 102, 98, 93, 104, 108, 111, 109, 111]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.452471592Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"":"downstairs","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73762024-03-06T21:00:55.727Z{"msg":"[0]R generation[0..12]: [66, 66, 66, 64, 61, 59, 56, 62, 64, 66, 64, 66]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.452505749Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"":"downstairs","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73772024-03-06T21:00:55.727Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.452541367Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"":"downstairs","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73782024-03-06T21:00:55.727Z{"msg":"[1]R flush_numbers[0..12]: [111, 111, 111, 109, 102, 98, 93, 104, 108, 111, 109, 111]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.452577544Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"":"downstairs","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73792024-03-06T21:00:55.727Z{"msg":"[1]R generation[0..12]: [66, 66, 66, 64, 61, 59, 56, 62, 64, 66, 64, 66]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
73802024-03-06T21:00:55.728Z,"time":"2024-03-06T21:00:54.452613502Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"":"downstairs","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73812024-03-06T21:00:55.728Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.452650669Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"":"downstairs","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73822024-03-06T21:00:55.728Z{"msg":"[2]R flush_numbers[0..12]: [111, 111, 111, 109, 102, 98, 93, 104, 108, 111, 109, 111]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.452689727Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"":"downstairs","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73832024-03-06T21:00:55.728Z{"msg":"[2]R generation[0..12]: [66, 66, 66, 64, 61, 59, 56, 62, 64, 66, 64, 66]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.452728044Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"":"downstairs","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73842024-03-06T21:00:55.728Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.452774791Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"":"downstairs","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73852024-03-06T21:00:55.728Z{"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.452813378Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"":"downstairs","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73862024-03-06T21:00:55.728Z{"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.452843586Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"":"downstairs","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73872024-03-06T21:00:55.729Z{"msg":"Next flush: 113","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.452857955Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"":"downstairs","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73882024-03-06T21:00:55.729Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.452878194Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"":"downstairs","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73892024-03-06T21:00:55.729Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.452891863Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73902024-03-06T21:00:55.729Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.452903852Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"client":"0","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73912024-03-06T21:00:55.729Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.4529346Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"client":"1","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73922024-03-06T21:00:55.729Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.452952759Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"client":"2","":"downstairs","session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73932024-03-06T21:00:55.729Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.452973357Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73942024-03-06T21:00:55.730Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.452985956Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73952024-03-06T21:00:55.730Z{"msg":"8de56f4f-bd25-45d6-ac41-6e0c85e7fd35 is now active with session: 327631a0-c78a-4dfa-94bb-33241ca00e08","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
73962024-03-06T21:00:55.730Z,"time":"2024-03-06T21:00:54.453005285Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73972024-03-06T21:00:55.730Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.453020804Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411,"session_id":"327631a0-c78a-4dfa-94bb-33241ca00e08"}
73982024-03-06T21:00:55.730Z{"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:54.453033093Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411}
73992024-03-06T21:00:55.730ZLoading write count information from file "/tmp/test_repair_verify.data"
74002024-03-06T21:00:55.730ZRun Repair workload
74012024-03-06T21:00:55.730Z01/30 Write block 295 len 20480 data: 7 6 5 4 4
74022024-03-06T21:00:55.730Z02/30 Write block 417 len 20480 data: 5 5 6 7 10
74032024-03-06T21:00:55.731Z03/30 Read block 311 len 28672
74042024-03-06T21:00:55.731Z04/30 Read block 32 len 24576
74052024-03-06T21:00:55.731Z05/30 Write block 584 len 4096 data: 6
74062024-03-06T21:00:55.731Z06/30 Read block 167 len 40960
74072024-03-06T21:00:55.731Z07/30 Read block 511 len 12288
74082024-03-06T21:00:55.731Z08/30 Read block 376 len 16384
74092024-03-06T21:00:55.731Z09/30 Write block 192 len 28672 data: 5 4 4 4 6 6 8
74102024-03-06T21:00:55.731Z10/30 Read block 40 len 12288
74112024-03-06T21:00:55.731Z11/30 Flush
74122024-03-06T21:00:55.732Z12/30 Write block 160 len 8192 data: 7 6
74132024-03-06T21:00:55.732Z13/30 Read block 306 len 24576
74142024-03-06T21:00:55.732Z14/30 Write block 106 len 24576 data: 8 9 6 6 6 7
74152024-03-06T21:00:55.732Z15/30 Read block 263 len 24576
74162024-03-06T21:00:55.758Z16/30 Flush
74172024-03-06T21:00:55.758Z17/30 Write block 10 len 40960 data: 5 8 6 6 5 6 5 4 5 6
74182024-03-06T21:00:55.758Z18/30 Read block 528 len 20480
74192024-03-06T21:00:55.758Z19/30 Read block 362 len 40960
74202024-03-06T21:00:55.758Z20/30 Write block 215 len 20480 data: 7 8 9 7 5
74212024-03-06T21:00:55.758Z21/30 Write block 561 len 24576 data: 6 7 6 6 5 6
74222024-03-06T21:00:55.758Z22/30 Write block 348 len 36864 data: 8 7 8 9 7 7 7 6 8
74232024-03-06T21:00:55.758Z23/30 Write block 460 len 40960 data: 8 11 10 12 11 11 9 8 7 7
74242024-03-06T21:00:55.758Z24/30 Write block 148 len 36864 data: 6 7 8 7 5 6 6 7 8
74252024-03-06T21:00:55.758Z25/30 Read block 250 len 40960
74262024-03-06T21:00:55.758Z26/30 Write block 34 len 12288 data: 8 7 3
74272024-03-06T21:00:55.758Z27/30 Write block 570 len 28672 data: 4 4 4 5 5 6 7
74282024-03-06T21:00:55.759Z28/30 Read block 577 len 8192
74292024-03-06T21:00:55.759Z29/30 Read block 386 len 24576
74302024-03-06T21:00:55.759Z30/30 Write block 515 len 12288 data: 7 7 7
74312024-03-06T21:00:55.759Z----------------------------------------------------------------
74322024-03-06T21:00:55.759Z Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:30
74332024-03-06T21:00:55.759ZStates: Active Active Active
74342024-03-06T21:00:55.759ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
74352024-03-06T21:00:55.759Z 1 Acked 1000 Write 5 Done Done Done false
74362024-03-06T21:00:55.759Z 2 Acked 1001 Write 5 Done Done Sent false
74372024-03-06T21:00:55.759Z 3 Acked 1002 Read 7 Done Done Sent false
74382024-03-06T21:00:55.759Z 4 Acked 1003 Read 6 Done Done Sent false
74392024-03-06T21:00:55.759Z 5 Acked 1004 Write 1 Done Done Sent false
74402024-03-06T21:00:55.759Z 6 Acked 1005 Read 10 Done Done Sent false
74412024-03-06T21:00:55.759Z 7 Acked 1006 Read 3 Done Done Sent false
74422024-03-06T21:00:55.759Z 8 Acked 1007 Read 4 Done Done Sent false
74432024-03-06T21:00:55.759Z 9 Acked 1008 Write 7 Done Done Sent false
74442024-03-06T21:00:55.759Z 10 Acked 1009 Read 3 Done Done Sent false
74452024-03-06T21:00:55.759Z 11 Acked 1010 Flush 0 Done Done Sent false
74462024-03-06T21:00:55.760Z 12 Acked 1011 Write 2 Done Done Sent false
74472024-03-06T21:00:55.760Z 13 Acked 1012 Read 6 Done Done Sent false
74482024-03-06T21:00:55.760Z 14 Acked 1013 Write 6 Done Done Sent false
74492024-03-06T21:00:55.760Z 15 Acked 1014 Read 6 Done Done Sent false
74502024-03-06T21:00:55.760Z 16 Acked 1015 Flush 0 Done Done Sent false
74512024-03-06T21:00:55.760Z 17 Acked 1016 Write 10 Done Done Sent false
74522024-03-06T21:00:55.760Z 18 Acked 1017 Read 5 Done Done Sent false
74532024-03-06T21:00:55.760Z 19 Acked 1018 Read 10 Done Done Sent false
74542024-03-06T21:00:55.760Z 20 Acked 1019 Write 5 Done Done Sent false
74552024-03-06T21:00:55.760Z 21 Acked 1020 Write 6 Done Done Sent false
74562024-03-06T21:00:55.760Z 22 Acked 1021 Write 9 Done Done Sent false
74572024-03-06T21:00:55.760Z 23 Acked 1022 Write 10 Done Done Sent false
74582024-03-06T21:00:55.760Z 24 Acked 1023 Write 9 Done Done Sent false
74592024-03-06T21:00:55.760Z 25 Acked 1024 Read 10 Done Done Sent false
74602024-03-06T21:00:55.760Z 26 Acked 1025 Write 3 Done Done Sent false
74612024-03-06T21:00:55.760Z 27 Acked 1026 Write 7 Done Done Sent false
74622024-03-06T21:00:55.760Z 28 Acked 1027 Read 2 Done Done Sent false
74632024-03-06T21:00:55.760Z 29 Acked 1028 Read 6 Done Done Sent false
74642024-03-06T21:00:55.760Z 30 Acked 1029 Write 3 Sent Sent Sent false
74652024-03-06T21:00:55.760Z STATES DS:0 DS:1 DS:2 TOTAL
74662024-03-06T21:00:55.760Z New 0 0 0 0
74672024-03-06T21:00:55.760Z Sent 1 1 29 31
74682024-03-06T21:00:55.760Z Done 29 29 1 59
74692024-03-06T21:00:55.761Z Skipped 0 0 0 0
74702024-03-06T21:00:55.761Z Error 0 0 0 0
74712024-03-06T21:00:55.761ZLast Flush: 1015 1015 0
74722024-03-06T21:00:55.761ZDownstairs last five completed:
74732024-03-06T21:00:55.761ZUpstairs last five completed: 30 29 28 27 26
74742024-03-06T21:00:55.761Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:00:54.487851203Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1411}
74752024-03-06T21:00:55.761ZWrote out file "/tmp/test_repair_verify.data"
74762024-03-06T21:00:55.761Z
74772024-03-06T21:00:55.761ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
74782024-03-06T21:00:55.785ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
74792024-03-06T21:00:55.785Z 0 000-019 66 66 66 111 111 111 T T F <---
74802024-03-06T21:00:55.786Z 1 020-039 66 66 66 111 111 111 T T F <---
74812024-03-06T21:00:55.786Z 2 040-059 66 66 66 111 111 111 F F F
74822024-03-06T21:00:55.786Z 3 060-079 64 64 64 109 109 109 F F F
74832024-03-06T21:00:55.786Z 4 080-099 61 61 61 102 102 102 F F F
74842024-03-06T21:00:55.786Z 5 100-119 68 68 59 114 114 98 F F F <---
74852024-03-06T21:00:55.786Z 6 120-139 56 56 56 93 93 93 F F F
74862024-03-06T21:00:55.786Z 7 140-159 62 62 62 104 104 104 T T F <---
74872024-03-06T21:00:55.786Z 8 160-179 68 68 64 114 114 108 F F F <---
74882024-03-06T21:00:55.786Z 9 180-199 68 68 66 113 113 111 F F F <---
74892024-03-06T21:00:55.786Z 10 200-219 64 64 64 109 109 109 T T F <---
74902024-03-06T21:00:55.787Z 11 220-239 66 66 66 111 111 111 F F F
74912024-03-06T21:00:55.787Z 12 240-259 67 67 67 112 112 112 F F F
74922024-03-06T21:00:55.787Z 13 260-279 67 67 67 112 112 112 F F F
74932024-03-06T21:00:55.787Z 14 280-299 68 68 62 113 113 103 F F T <---
74942024-03-06T21:00:55.787Z 15 300-319 64 64 64 108 108 108 F F F
74952024-03-06T21:00:55.788Z 16 320-339 66 66 66 111 111 111 F F F
74962024-03-06T21:00:55.788Z 17 340-359 66 66 66 111 111 111 T T F <---
74972024-03-06T21:00:55.788Z 18 360-379 67 67 67 112 112 112 F F F
74982024-03-06T21:00:55.788Z 19 380-399 66 66 66 111 111 111 F F F
74992024-03-06T21:00:55.788Z 20 400-419 68 68 66 113 113 111 F F F <---
75002024-03-06T21:00:55.789Z 21 420-439 68 68 66 113 113 111 F F F <---
75012024-03-06T21:00:55.789Z 22 440-459 66 66 66 111 111 111 F F F
75022024-03-06T21:00:55.789Z 23 460-479 63 63 63 105 105 105 T T F <---
75032024-03-06T21:00:55.789Z 24 480-499 63 63 63 105 105 105 F F F
75042024-03-06T21:00:55.789Z 25 500-519 61 61 61 102 102 102 T T F <---
75052024-03-06T21:00:55.789Z 26 520-539 61 61 61 102 102 102 F F F
75062024-03-06T21:00:55.789Z 27 540-559 67 67 67 112 112 112 F F F
75072024-03-06T21:00:55.789Z 28 560-579 62 62 62 103 103 103 T T F <---
75082024-03-06T21:00:55.790Z 29 580-599 68 68 66 113 113 111 F F F <---
75092024-03-06T21:00:55.790ZMax gen: 68, Max flush: 114
75102024-03-06T21:00:55.790ZError: Difference in extent metadata found!
75112024-03-06T21:00:55.790ZA Difference in extent metadata is expected here
75122024-03-06T21:00:55.790ZOn loop 34
75132024-03-06T21:00:55.790Z
75142024-03-06T21:00:55.790ZVerifying data now
75152024-03-06T21:01:00.549ZNo change after verify
75162024-03-06T21:01:00.549ZLoop: 34 Downstairs dump after verify (and repair):
75172024-03-06T21:01:00.560ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
75182024-03-06T21:01:00.560Z 0 000-019 69 69 69 115 115 115 F F F
75192024-03-06T21:01:00.560Z 1 020-039 69 69 69 115 115 115 F F F
75202024-03-06T21:01:00.560Z 2 040-059 66 66 66 111 111 111 F F F
75212024-03-06T21:01:00.561Z 3 060-079 64 64 64 109 109 109 F F F
75222024-03-06T21:01:00.561Z 4 080-099 61 61 61 102 102 102 F F F
75232024-03-06T21:01:00.561Z 5 100-119 68 68 68 114 114 114 F F F
75242024-03-06T21:01:00.561Z 6 120-139 56 56 56 93 93 93 F F F
75252024-03-06T21:01:00.561Z 7 140-159 69 69 69 115 115 115 F F F
75262024-03-06T21:01:00.561Z 8 160-179 68 68 68 114 114 114 F F F
75272024-03-06T21:01:00.561Z 9 180-199 68 68 68 113 113 113 F F F
75282024-03-06T21:01:00.562Z 10 200-219 69 69 69 115 115 115 F F F
75292024-03-06T21:01:00.562Z 11 220-239 66 66 66 111 111 111 F F F
75302024-03-06T21:01:00.562Z 12 240-259 67 67 67 112 112 112 F F F
75312024-03-06T21:01:00.562Z 13 260-279 67 67 67 112 112 112 F F F
75322024-03-06T21:01:00.562Z 14 280-299 68 68 68 113 113 113 F F F
75332024-03-06T21:01:00.562Z 15 300-319 64 64 64 108 108 108 F F F
75342024-03-06T21:01:00.562Z 16 320-339 66 66 66 111 111 111 F F F
75352024-03-06T21:01:00.563Z 17 340-359 69 69 69 115 115 115 F F F
75362024-03-06T21:01:00.563Z 18 360-379 67 67 67 112 112 112 F F F
75372024-03-06T21:01:00.563Z 19 380-399 66 66 66 111 111 111 F F F
75382024-03-06T21:01:00.563Z 20 400-419 68 68 68 113 113 113 F F F
75392024-03-06T21:01:00.563Z 21 420-439 68 68 68 113 113 113 F F F
75402024-03-06T21:01:00.563Z 22 440-459 66 66 66 111 111 111 F F F
75412024-03-06T21:01:00.564Z 23 460-479 69 69 69 115 115 115 F F F
75422024-03-06T21:01:00.564Z 24 480-499 63 63 63 105 105 105 F F F
75432024-03-06T21:01:00.564Z 25 500-519 69 69 69 115 115 115 F F F
75442024-03-06T21:01:00.564Z 26 520-539 61 61 61 102 102 102 F F F
75452024-03-06T21:01:00.564Z 27 540-559 67 67 67 112 112 112 F F F
75462024-03-06T21:01:00.565Z 28 560-579 69 69 69 115 115 115 F F F
75472024-03-06T21:01:00.565Z 29 580-599 68 68 68 113 113 113 F F F
75482024-03-06T21:01:00.565ZMax gen: 69, Max flush: 115
75492024-03-06T21:01:00.566Z
75502024-03-06T21:01:00.566ZBegin loop 35
75512024-03-06T21:01:00.566ZDownstairs to restart: 2
75522024-03-06T21:01:00.582Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.340366302Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419}
75532024-03-06T21:01:00.582Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.340621765Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419}
75542024-03-06T21:01:00.583Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.340642203Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419}
75552024-03-06T21:01:00.583Z{"msg":"Crucible 3fb10a8e-b5e7-4b5e-a866-fc64d5f66b62 has session id: 8bb5da90-1bdf-4f6b-8020-475b48f68e7a","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.340656772Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75562024-03-06T21:01:00.583Z{"msg":"Upstairs opts: Upstairs UUID: 3fb10a8e-b5e7-4b5e-a866-fc64d5f66b62, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.340711858Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75572024-03-06T21:01:00.583ZCrucible runtime is spawned
75582024-03-06T21:01:00.596Z{"msg":"Crucible stats registered with UUID: 3fb10a8e-b5e7-4b5e-a866-fc64d5f66b62","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.340756035Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75592024-03-06T21:01:00.597Z{"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.340865678Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419}
75602024-03-06T21:01:00.597Z{"msg":"3fb10a8e-b5e7-4b5e-a866-fc64d5f66b62 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.340998849Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75612024-03-06T21:01:00.597Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.341030257Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"client":"0","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75622024-03-06T21:01:00.597Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.341049215Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"client":"0","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75632024-03-06T21:01:00.597Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.341087703Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"client":"1","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75642024-03-06T21:01:00.597Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.34112469Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"client":"1","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75652024-03-06T21:01:00.597Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.341165377Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"client":"2","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75662024-03-06T21:01:00.597Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.341191965Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"client":"2","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75672024-03-06T21:01:00.598Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.341221143Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"":"io task","client":"0","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75682024-03-06T21:01:00.598Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.341251921Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"":"io task","client":"2","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75692024-03-06T21:01:00.598Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.341276999Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"":"io task","client":"1","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75702024-03-06T21:01:00.598Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.341492565Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"":"io task","client":"2","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75712024-03-06T21:01:00.598Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.3415583Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"":"io task","client":"1","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75722024-03-06T21:01:00.598Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.341625386Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"":"io task","client":"0","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75732024-03-06T21:01:00.598Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.342347996Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"client":"0","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75742024-03-06T21:01:00.598Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.342406092Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"client":"0","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75752024-03-06T21:01:00.599Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.342443859Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"client":"1","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75762024-03-06T21:01:00.599Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.342504045Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"client":"1","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75772024-03-06T21:01:00.599Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.343171689Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"client":"1","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75782024-03-06T21:01:00.599Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.343214746Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"client":"0","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75792024-03-06T21:01:00.599Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.343696133Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"client":"2","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75802024-03-06T21:01:00.599Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.343749319Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"client":"2","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75812024-03-06T21:01:00.599Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.343801616Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"client":"1","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75822024-03-06T21:01:00.599Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.343861481Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"client":"1","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75832024-03-06T21:01:00.600Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.343898179Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75842024-03-06T21:01:00.600Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.343912388Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"client":"0","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75852024-03-06T21:01:00.600Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.343934636Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"client":"0","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75862024-03-06T21:01:00.600Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.343948895Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75872024-03-06T21:01:00.600Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.344212437Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"client":"2","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75882024-03-06T21:01:00.600Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.344626829Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"client":"2","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75892024-03-06T21:01:00.600Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.344655177Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"client":"2","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75902024-03-06T21:01:00.600Z{"msg":"[0]R flush_numbers[0..12]: [115, 115, 111, 109, 102, 114, 93, 115, 114, 113, 115, 111]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.344674596Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"":"downstairs","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75912024-03-06T21:01:00.601Z{"msg":"[0]R generation[0..12]: [69, 69, 66, 64, 61, 68, 56, 69, 68, 68, 69, 66]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.344695944Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"":"downstairs","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75922024-03-06T21:01:00.601Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.344710743Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"":"downstairs","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75932024-03-06T21:01:00.601Z{"msg":"[1]R flush_numbers[0..12]: [115, 115, 111, 109, 102, 114, 93, 115, 114, 113, 115, 111]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.344725622Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"":"downstairs","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75942024-03-06T21:01:00.601Z{"msg":"[1]R generation[0..12]: [69, 69, 66, 64, 61, 68, 56, 69, 68, 68, 69, 66]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.3447462Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"":"downstairs","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75952024-03-06T21:01:00.602Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.344761159Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"":"downstairs","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75962024-03-06T21:01:00.602Z{"msg":"[2]R flush_numbers[0..12]: [115, 115, 111, 109, 102, 114, 93, 115, 114, 113, 115, 111]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.344781418Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"":"downstairs","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75972024-03-06T21:01:00.602Z{"msg":"[2]R generation[0..12]: [69, 69, 66, 64, 61, 68, 56, 69, 68, 68, 69, 66]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.344795167Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"":"downstairs","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75982024-03-06T21:01:00.603Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.344809286Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"":"downstairs","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
75992024-03-06T21:01:00.603Z{"msg":"Max found gen is 70","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO
76002024-03-06T21:01:00.603Z30,"time":"2024-03-06T21:00:59.344835494Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"":"downstairs","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
76012024-03-06T21:01:00.603Z{"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.344852733Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"":"downstairs","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
76022024-03-06T21:01:00.603Z{"msg":"Next flush: 116","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.344875122Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"":"downstairs","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
76032024-03-06T21:01:00.604Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.344888721Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"":"downstairs","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
76042024-03-06T21:01:00.604Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.344908719Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
76052024-03-06T21:01:00.604Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.344921168Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"client":"0","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
76062024-03-06T21:01:00.604Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.344941397Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"client":"1","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
76072024-03-06T21:01:00.605Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.344954646Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"client":"2","":"downstairs","session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
76082024-03-06T21:01:00.605Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.344974115Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
76092024-03-06T21:01:00.605Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.344986664Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
76102024-03-06T21:01:00.605Z{"msg":"3fb10a8e-b5e7-4b5e-a866-fc64d5f66b62 is now active with session: 8bb5da90-1bdf-4f6b-8020-475b48f68e7a","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.345020382Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
76112024-03-06T21:01:00.605Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.345033571Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419,"session_id":"8bb5da90-1bdf-4f6b-8020-475b48f68e7a"}
76122024-03-06T21:01:00.606Z{"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:00:59.34504575Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419}
76132024-03-06T21:01:00.606ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
76142024-03-06T21:01:00.606ZLoading write count information from file "/tmp/test_repair_verify.data"
76152024-03-06T21:01:00.607ZRun Repair workload
76162024-03-06T21:01:00.607Z01/30 Flush
76172024-03-06T21:01:00.607Z02/30 Write block 302 len 28672 data: 4 4 5 7 9 9 8
76182024-03-06T21:01:00.607Z03/30 Read block 312 len 20480
76192024-03-06T21:01:00.607Z04/30 Flush
76202024-03-06T21:01:00.617Z05/30 Write block 2 len 16384 data: 3 4 5 6
76212024-03-06T21:01:00.617Z06/30 Write block 238 len 8192 data: 10 10
76222024-03-06T21:01:00.618Z07/30 Read block 512 len 36864
76232024-03-06T21:01:00.618Z08/30 Write block 242 len 16384 data: 8 6 5 5
76242024-03-06T21:01:00.618Z09/30 Write block 191 len 12288 data: 4 6 5
76252024-03-06T21:01:00.618Z10/30 Write block 388 len 36864 data: 6 6 6 6 6 6 5 3 3
76262024-03-06T21:01:00.618Z11/30 Write block 274 len 36864 data: 9 7 8 6 6 7 7 8 8
76272024-03-06T21:01:00.618Z12/30 Flush
76282024-03-06T21:01:00.618Z13/30 Flush
76292024-03-06T21:01:00.618Z14/30 Write block 583 len 40960 data: 4 7 8 8 10 8 8 8 7 8
76302024-03-06T21:01:00.619Z15/30 Write block 422 len 4096 data: 8
76312024-03-06T21:01:00.619Z16/30 Write block 360 len 16384 data: 8 10 9 12
76322024-03-06T21:01:00.619Z17/30 Write block 418 len 36864 data: 6 7 8 11 9 7 6 6 6
76332024-03-06T21:01:00.619Z18/30 Read block 368 len 32768
76342024-03-06T21:01:00.619Z19/30 Read block 260 len 4096
76352024-03-06T21:01:00.620Z20/30 Write block 106 len 16384 data: 9 10 7 7
76362024-03-06T21:01:00.620Z21/30 Write block 530 len 8192 data: 9 8
76372024-03-06T21:01:00.620Z22/30 Write block 540 len 28672 data: 7 5 7 7 7 7 7
76382024-03-06T21:01:00.620Z23/30 Write block 509 len 20480 data: 3 5 5 6 6
76392024-03-06T21:01:00.620Z24/30 Write block 191 len 24576 data: 5 7 6 5 5 7
76402024-03-06T21:01:00.620Z25/30 Flush
76412024-03-06T21:01:00.621Z26/30 Write block 437 len 24576 data: 7 7 7 7 4 3
76422024-03-06T21:01:00.641Z27/30 Read block 287 len 28672
76432024-03-06T21:01:00.641Z28/30 Write block 326 len 4096 data: 4
76442024-03-06T21:01:00.641Z29/30 Read block 7 len 24576
76452024-03-06T21:01:00.641Z30/30 Write block 41 len 16384 data: 6 7 7 6
76462024-03-06T21:01:00.642Z----------------------------------------------------------------
76472024-03-06T21:01:00.642Z Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:29
76482024-03-06T21:01:00.642ZStates: Active Active Active
76492024-03-06T21:01:00.642ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
76502024-03-06T21:01:00.642Z 2 Acked 1001 Write 7 Done Done Sent false
76512024-03-06T21:01:00.642Z 3 Acked 1002 Read 5 Done Done Sent false
76522024-03-06T21:01:00.642Z 4 Acked 1003 Flush 0 Done Done Sent false
76532024-03-06T21:01:00.642Z 5 Acked 1004 Write 4 Done Done Sent false
76542024-03-06T21:01:00.643Z 6 Acked 1005 Write 2 Done Done Sent false
76552024-03-06T21:01:00.643Z 7 Acked 1006 Read 9 Done Done Sent false
76562024-03-06T21:01:00.643Z 8 Acked 1007 Write 4 Done Done Sent false
76572024-03-06T21:01:00.643Z 9 Acked 1008 Write 3 Done Done Sent false
76582024-03-06T21:01:00.643Z 10 Acked 1009 Write 9 Done Done Sent false
76592024-03-06T21:01:00.643Z 11 Acked 1010 Write 9 Done Done Sent false
76602024-03-06T21:01:00.643Z 12 Acked 1011 Flush 0 Done Done Sent false
76612024-03-06T21:01:00.643Z 13 Acked 1012 Flush 0 Done Done Sent false
76622024-03-06T21:01:00.644Z 14 Acked 1013 Write 10 Done Done Sent false
76632024-03-06T21:01:00.644Z 15 Acked 1014 Write 1 Done Done Sent false
76642024-03-06T21:01:00.644Z 16 Acked 1015 Write 4 Done Done Sent false
76652024-03-06T21:01:00.644Z 17 Acked 1016 Write 9 Done Done Sent false
76662024-03-06T21:01:00.644Z 18 Acked 1017 Read 8 Done Done Sent false
76672024-03-06T21:01:00.644Z 19 Acked 1018 Read 1 Done Done Sent false
76682024-03-06T21:01:00.645Z 20 Acked 1019 Write 4 Done Done Sent false
76692024-03-06T21:01:00.645Z 21 Acked 1020 Write 2 Done Done Sent false
76702024-03-06T21:01:00.645Z 22 Acked 1021 Write 7 Done Done Sent false
76712024-03-06T21:01:00.645Z 23 Acked 1022 Write 5 Done Done Sent false
76722024-03-06T21:01:00.645Z 24 Acked 1023 Write 6 Done Done Sent false
76732024-03-06T21:01:00.645Z 25 Acked 1024 Flush 0 Done Done Sent false
76742024-03-06T21:01:00.645Z 26 Acked 1025 Write 6 Done Done Sent false
76752024-03-06T21:01:00.645Z 27 Acked 1026 Read 7 Done Done Sent false
76762024-03-06T21:01:00.646Z 28 Acked 1027 Write 1 Done Done Sent false
76772024-03-06T21:01:00.646Z 29 Acked 1028 Read 6 Done Done Sent false
76782024-03-06T21:01:00.646Z 30 Acked 1029 Write 4 Sent Sent Sent false
76792024-03-06T21:01:00.646Z STATES DS:0 DS:1 DS:2 TOTAL
76802024-03-06T21:01:00.646Z New 0 0 0 0
76812024-03-06T21:01:00.646Z Sent 1 1 29 31
76822024-03-06T21:01:00.646Z Done 28 28 0 56
76832024-03-06T21:01:00.646Z Skipped 0 0 0 0
76842024-03-06T21:01:00.647Z Error 0 0 0 0
76852024-03-06T21:01:00.647ZLast Flush: 1024 1024 1000
76862024-03-06T21:01:00.647ZDownstairs last five completed: 1000
76872024-03-06T21:01:00.647ZUpstairs last five completed: 30 29 28 27 26
76882024-03-06T21:01:00.647Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:00:59.381122054Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1419}
76892024-03-06T21:01:00.647ZWrote out file "/tmp/test_repair_verify.data"
76902024-03-06T21:01:00.647Z
76912024-03-06T21:01:00.648ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
76922024-03-06T21:01:00.662ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
76932024-03-06T21:01:00.663Z 0 000-019 70 70 69 118 118 115 F F F <---
76942024-03-06T21:01:00.663Z 1 020-039 69 69 69 115 115 115 F F F
76952024-03-06T21:01:00.663Z 2 040-059 66 66 66 111 111 111 T T F <---
76962024-03-06T21:01:00.663Z 3 060-079 64 64 64 109 109 109 F F F
76972024-03-06T21:01:00.663Z 4 080-099 61 61 61 102 102 102 F F F
76982024-03-06T21:01:00.663Z 5 100-119 70 70 68 120 120 114 F F F <---
76992024-03-06T21:01:00.663Z 6 120-139 56 56 56 93 93 93 F F F
77002024-03-06T21:01:00.663Z 7 140-159 69 69 69 115 115 115 F F F
77012024-03-06T21:01:00.664Z 8 160-179 68 68 68 114 114 114 F F F
77022024-03-06T21:01:00.664Z 9 180-199 70 70 68 120 120 113 F F F <---
77032024-03-06T21:01:00.664Z 10 200-219 69 69 69 115 115 115 F F F
77042024-03-06T21:01:00.664Z 11 220-239 70 70 66 118 118 111 F F F <---
77052024-03-06T21:01:00.664Z 12 240-259 70 70 67 118 118 112 F F F <---
77062024-03-06T21:01:00.665Z 13 260-279 70 70 67 118 118 112 F F F <---
77072024-03-06T21:01:00.665Z 14 280-299 70 70 68 118 118 113 F F F <---
77082024-03-06T21:01:00.665Z 15 300-319 70 70 64 117 117 108 F F F <---
77092024-03-06T21:01:00.665Z 16 320-339 66 66 66 111 111 111 T T F <---
77102024-03-06T21:01:00.665Z 17 340-359 69 69 69 115 115 115 F F F
77112024-03-06T21:01:00.665Z 18 360-379 70 70 67 120 120 112 F F F <---
77122024-03-06T21:01:00.665Z 19 380-399 70 70 66 118 118 111 F F F <---
77132024-03-06T21:01:00.665Z 20 400-419 70 70 68 120 120 113 F F F <---
77142024-03-06T21:01:00.665Z 21 420-439 70 70 68 120 120 113 T T F <---
77152024-03-06T21:01:00.665Z 22 440-459 66 66 66 111 111 111 T T F <---
77162024-03-06T21:01:00.666Z 23 460-479 69 69 69 115 115 115 F F F
77172024-03-06T21:01:00.666Z 24 480-499 63 63 63 105 105 105 F F F
77182024-03-06T21:01:00.666Z 25 500-519 70 70 69 120 120 115 F F F <---
77192024-03-06T21:01:00.666Z 26 520-539 70 70 61 120 120 102 F F F <---
77202024-03-06T21:01:00.666Z 27 540-559 70 70 67 120 120 112 F F F <---
77212024-03-06T21:01:00.666Z 28 560-579 69 69 69 115 115 115 F F F
77222024-03-06T21:01:00.666Z 29 580-599 70 70 68 120 120 113 F F F <---
77232024-03-06T21:01:00.667ZMax gen: 70, Max flush: 120
77242024-03-06T21:01:00.667ZError: Difference in extent metadata found!
77252024-03-06T21:01:00.667ZA Difference in extent metadata is expected here
77262024-03-06T21:01:00.667ZOn loop 35
77272024-03-06T21:01:00.667Z
77282024-03-06T21:01:00.684ZVerifying data now
77292024-03-06T21:01:05.591ZNo change after verify
77302024-03-06T21:01:05.591ZLoop: 35 Downstairs dump after verify (and repair):
77312024-03-06T21:01:05.606ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
77322024-03-06T21:01:05.606Z 0 000-019 70 70 70 118 118 118 F F F
77332024-03-06T21:01:05.606Z 1 020-039 69 69 69 115 115 115 F F F
77342024-03-06T21:01:05.606Z 2 040-059 71 71 71 121 121 121 F F F
77352024-03-06T21:01:05.607Z 3 060-079 64 64 64 109 109 109 F F F
77362024-03-06T21:01:05.607Z 4 080-099 61 61 61 102 102 102 F F F
77372024-03-06T21:01:05.607Z 5 100-119 70 70 70 120 120 120 F F F
77382024-03-06T21:01:05.607Z 6 120-139 56 56 56 93 93 93 F F F
77392024-03-06T21:01:05.607Z 7 140-159 69 69 69 115 115 115 F F F
77402024-03-06T21:01:05.607Z 8 160-179 68 68 68 114 114 114 F F F
77412024-03-06T21:01:05.607Z 9 180-199 70 70 70 120 120 120 F F F
77422024-03-06T21:01:05.620Z 10 200-219 69 69 69 115 115 115 F F F
77432024-03-06T21:01:05.620Z 11 220-239 70 70 70 118 118 118 F F F
77442024-03-06T21:01:05.620Z 12 240-259 70 70 70 118 118 118 F F F
77452024-03-06T21:01:05.620Z 13 260-279 70 70 70 118 118 118 F F F
77462024-03-06T21:01:05.620Z 14 280-299 70 70 70 118 118 118 F F F
77472024-03-06T21:01:05.620Z 15 300-319 70 70 70 117 117 117 F F F
77482024-03-06T21:01:05.620Z 16 320-339 71 71 71 121 121 121 F F F
77492024-03-06T21:01:05.620Z 17 340-359 69 69 69 115 115 115 F F F
77502024-03-06T21:01:05.620Z 18 360-379 70 70 70 120 120 120 F F F
77512024-03-06T21:01:05.621Z 19 380-399 70 70 70 118 118 118 F F F
77522024-03-06T21:01:05.621Z 20 400-419 70 70 70 120 120 120 F F F
77532024-03-06T21:01:05.621Z 21 420-439 71 71 71 121 121 121 F F F
77542024-03-06T21:01:05.621Z 22 440-459 71 71 71 121 121 121 F F F
77552024-03-06T21:01:05.621Z 23 460-479 69 69 69 115 115 115 F F F
77562024-03-06T21:01:05.621Z 24 480-499 63 63 63 105 105 105 F F F
77572024-03-06T21:01:05.621Z 25 500-519 70 70 70 120 120 120 F F F
77582024-03-06T21:01:05.621Z 26 520-539 70 70 70 120 120 120 F F F
77592024-03-06T21:01:05.621Z 27 540-559 70 70 70 120 120 120 F F F
77602024-03-06T21:01:05.621Z 28 560-579 69 69 69 115 115 115 F F F
77612024-03-06T21:01:05.621Z 29 580-599 70 70 70 120 120 120 F F F
77622024-03-06T21:01:05.621ZMax gen: 71, Max flush: 121
77632024-03-06T21:01:05.621Z
77642024-03-06T21:01:05.622ZBegin loop 36
77652024-03-06T21:01:05.622ZDownstairs to restart: 1
77662024-03-06T21:01:05.634Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.391808672Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427}
77672024-03-06T21:01:05.634Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.392115501Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427}
77682024-03-06T21:01:05.634Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
77692024-03-06T21:01:05.634Z,"time":"2024-03-06T21:01:04.392148609Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427}
77702024-03-06T21:01:05.666Z{"msg":"Crucible ab05c484-f5be-4f9f-9f76-8ea421800cae has session id: 42669c7b-5459-4ee1-af9a-ab7041a967d0","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.392207385Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
77712024-03-06T21:01:05.666Z{"msg":"Upstairs opts: Upstairs UUID: ab05c484-f5be-4f9f-9f76-8ea421800cae, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.392255001Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
77722024-03-06T21:01:05.666Z{"msg":"Crucible stats registered with UUID: ab05c484-f5be-4f9f-9f76-8ea421800cae","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.392298028Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
77732024-03-06T21:01:05.666Z{"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.392313717Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427}
77742024-03-06T21:01:05.666Z{"msg":"ab05c484-f5be-4f9f-9f76-8ea421800cae active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.392330306Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
77752024-03-06T21:01:05.666Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.392366354Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"client":"0","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
77762024-03-06T21:01:05.666Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.392390712Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"client":"0","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
77772024-03-06T21:01:05.666Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.3924265Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"client":"1","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
77782024-03-06T21:01:05.666Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.392474986Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"client":"1","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
77792024-03-06T21:01:05.666Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.392505824Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"client":"2","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
77802024-03-06T21:01:05.666Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.392526733Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"client":"2","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
77812024-03-06T21:01:05.667Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.392556451Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"":"io task","client":"1","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
77822024-03-06T21:01:05.667Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.392586909Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"":"io task","client":"0","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
77832024-03-06T21:01:05.667Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.392620186Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"":"io task","client":"2","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
77842024-03-06T21:01:05.667Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.392643165Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"":"io task","client":"0","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
77852024-03-06T21:01:05.667Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.392821832Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"":"io task","client":"2","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
77862024-03-06T21:01:05.667Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.392875559Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"":"io task","client":"1","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
77872024-03-06T21:01:05.667Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.393465708Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"client":"2","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
77882024-03-06T21:01:05.667Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.393554082Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"client":"2","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
77892024-03-06T21:01:05.667Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.393594399Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"client":"0","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
77902024-03-06T21:01:05.667Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.393659344Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"client":"0","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
77912024-03-06T21:01:05.667Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.394191398Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"client":"2","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
77922024-03-06T21:01:05.667Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.394229845Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"client":"0","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
77932024-03-06T21:01:05.667Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.394571542Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"client":"1","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
77942024-03-06T21:01:05.667Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.394619318Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"client":"1","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
77952024-03-06T21:01:05.667Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.394794876Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"client":"2","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
77962024-03-06T21:01:05.667Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.394836534Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"client":"2","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
77972024-03-06T21:01:05.667Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.394863692Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
77982024-03-06T21:01:05.667Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.394902869Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"client":"0","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
77992024-03-06T21:01:05.667Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.394923948Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"client":"0","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
78002024-03-06T21:01:05.667Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.394940266Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
78012024-03-06T21:01:05.668Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.395207028Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"client":"1","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
78022024-03-06T21:01:05.668Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.395622089Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"client":"1","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
78032024-03-06T21:01:05.668Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.395662297Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"client":"1","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
78042024-03-06T21:01:05.668Z{"msg":"[0]R flush_numbers[0..12]: [118, 115, 121, 109, 102, 120, 93, 115, 114, 120, 115, 118]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.395727922Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"":"downstairs","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
78052024-03-06T21:01:05.668Z{"msg":"[0]R generation[0..12]: [70, 69, 71, 64, 61, 70, 56, 69, 68, 70, 69, 70]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.395775119Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"":"downstairs","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
78062024-03-06T21:01:05.668Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
78072024-03-06T21:01:05.668Z,"time":"2024-03-06T21:01:04.395814116Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"":"downstairs","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
78082024-03-06T21:01:05.668Z{"msg":"[1]R flush_numbers[0..12]: [118, 115, 121, 109, 102, 120, 93, 115, 114, 120, 115, 118]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.395842034Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"":"downstairs","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
78092024-03-06T21:01:05.668Z{"msg":"[1]R generation[0..12]: [70, 69, 71, 64, 61, 70, 56, 69, 68, 70, 69, 70]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.395879582Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"":"downstairs","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
78102024-03-06T21:01:05.668Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.395918419Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"":"downstairs","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
78112024-03-06T21:01:05.668Z{"msg":"[2]R flush_numbers[0..12]: [118, 115, 121, 109, 102, 120, 93, 115, 114, 120, 115, 118]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.395967485Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"":"downstairs","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
78122024-03-06T21:01:05.668Z{"msg":"[2]R generation[0..12]: [70, 69, 71, 64, 61, 70, 56, 69, 68, 70, 69, 70]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.396026061Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"":"downstairs","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
78132024-03-06T21:01:05.668Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.396054699Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"":"downstairs","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
78142024-03-06T21:01:05.668Z{"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.396069278Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"":"downstairs","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
78152024-03-06T21:01:05.668Z{"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.396090187Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"":"downstairs","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
78162024-03-06T21:01:05.668Z{"msg":"Next flush: 122","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.396103856Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"":"downstairs","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
78172024-03-06T21:01:05.669Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.396123805Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"":"downstairs","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
78182024-03-06T21:01:05.669Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.396137314Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
78192024-03-06T21:01:05.669Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.396149923Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"client":"0","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
78202024-03-06T21:01:05.669ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
78212024-03-06T21:01:05.669Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.396174191Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"client":"1","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
78222024-03-06T21:01:05.669Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.39618832Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"client":"2","":"downstairs","session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
78232024-03-06T21:01:05.669Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.396208519Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
78242024-03-06T21:01:05.669Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.396221338Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
78252024-03-06T21:01:05.669Z{"msg":"ab05c484-f5be-4f9f-9f76-8ea421800cae is now active with session: 42669c7b-5459-4ee1-af9a-ab7041a967d0","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.396240567Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
78262024-03-06T21:01:05.669Z{"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.396253126Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427}
78272024-03-06T21:01:05.669Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:04.396264405Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427,"session_id":"42669c7b-5459-4ee1-af9a-ab7041a967d0"}
78282024-03-06T21:01:05.669ZLoading write count information from file "/tmp/test_repair_verify.data"
78292024-03-06T21:01:05.669ZRun Repair workload
78302024-03-06T21:01:05.669Z01/30 Write block 510 len 4096 data: 6
78312024-03-06T21:01:05.669Z02/30 Read block 233 len 40960
78322024-03-06T21:01:05.669Z03/30 Write block 247 len 20480 data: 3 3 3 3 3
78332024-03-06T21:01:05.670Z04/30 Write block 318 len 40960 data: 6 6 4 4 5 4 4 4 5 3
78342024-03-06T21:01:05.670Z05/30 Write block 468 len 4096 data: 8
78352024-03-06T21:01:05.670Z06/30 Read block 292 len 28672
78362024-03-06T21:01:06.265Z07/30 Read block 44 len 20480
78372024-03-06T21:01:06.265Z08/30 Read block 158 len 4096
78382024-03-06T21:01:06.266Z09/30 Write block 46 len 12288 data: 7 8 7
78392024-03-06T21:01:06.266Z10/30 Read block 15 len 36864
78402024-03-06T21:01:06.266Z11/30 Read block 379 len 20480
78412024-03-06T21:01:06.266Z12/30 Read block 66 len 24576
78422024-03-06T21:01:06.266Z13/30 Read block 252 len 16384
78432024-03-06T21:01:06.266Z14/30 Read block 490 len 16384
78442024-03-06T21:01:06.266Z15/30 Flush
78452024-03-06T21:01:06.266Z16/30 Write block 26 len 8192 data: 6 6
78462024-03-06T21:01:06.266Z17/30 Read block 59 len 8192
78472024-03-06T21:01:06.266Z18/30 Flush
78482024-03-06T21:01:06.267Z19/30 Write block 576 len 16384 data: 8 5 3 2
78492024-03-06T21:01:06.267Z20/30 Read block 75 len 20480
78502024-03-06T21:01:06.267Z21/30 Read block 153 len 4096
78512024-03-06T21:01:06.267Z22/30 Write block 498 len 28672 data: 7 7 6 4 4 4 4
78522024-03-06T21:01:06.267Z23/30 Write block 577 len 4096 data: 6
78532024-03-06T21:01:06.267Z24/30 Write block 564 len 20480 data: 7 6 7 5 3
78542024-03-06T21:01:06.267Z25/30 Read block 425 len 8192
78552024-03-06T21:01:06.267Z26/30 Read block 36 len 32768
78562024-03-06T21:01:06.267Z27/30 Read block 14 len 16384
78572024-03-06T21:01:06.267Z28/30 Write block 216 len 24576 data: 9 10 8 6 4 4
78582024-03-06T21:01:06.267Z29/30 Write block 206 len 4096 data: 4
78592024-03-06T21:01:06.267Z30/30 Read block 351 len 12288
78602024-03-06T21:01:06.267Z----------------------------------------------------------------
78612024-03-06T21:01:06.267Z Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30
78622024-03-06T21:01:06.267ZStates: Active Active Active
78632024-03-06T21:01:06.267ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
78642024-03-06T21:01:06.267Z 1 Acked 1000 Write 1 Done Done Done false
78652024-03-06T21:01:06.267Z 2 Acked 1001 Read 10 Done Done Done false
78662024-03-06T21:01:06.267Z 3 Acked 1002 Write 5 Done Done Done false
78672024-03-06T21:01:06.268Z 4 Acked 1003 Write 10 Done Done Done false
78682024-03-06T21:01:06.268Z 5 Acked 1004 Write 1 Done Done Done false
78692024-03-06T21:01:06.268Z 6 Acked 1005 Read 7 Done Done Done false
78702024-03-06T21:01:06.268Z 7 Acked 1006 Read 5 Done Done Done false
78712024-03-06T21:01:06.268Z 8 Acked 1007 Read 1 Done Done Done false
78722024-03-06T21:01:06.268Z 9 Acked 1008 Write 3 Done Done Done false
78732024-03-06T21:01:06.268Z 10 Acked 1009 Read 9 Done Done Done false
78742024-03-06T21:01:06.268Z 11 Acked 1010 Read 5 Done Sent Done false
78752024-03-06T21:01:06.268Z 12 Acked 1011 Read 6 Done Sent Done false
78762024-03-06T21:01:06.268Z 13 Acked 1012 Read 4 Done Sent Done false
78772024-03-06T21:01:06.268Z 14 Acked 1013 Read 4 Done Sent Done false
78782024-03-06T21:01:06.268Z 15 Acked 1014 Flush 0 Done Sent Done false
78792024-03-06T21:01:06.269Z 16 Acked 1015 Write 2 Done Sent Done false
78802024-03-06T21:01:06.269Z 17 Acked 1016 Read 2 Done Sent Done false
78812024-03-06T21:01:06.269Z 18 Acked 1017 Flush 0 Done Sent Done false
78822024-03-06T21:01:06.269Z 19 Acked 1018 Write 4 Done Sent Done false
78832024-03-06T21:01:06.269Z 20 Acked 1019 Read 5 Done Sent Done false
78842024-03-06T21:01:06.269Z 21 Acked 1020 Read 1 Done Sent Done false
78852024-03-06T21:01:06.269Z 22 Acked 1021 Write 7 Done Sent Done false
78862024-03-06T21:01:06.269Z 23 Acked 1022 Write 1 Done Sent Done false
78872024-03-06T21:01:06.269Z 24 Acked 1023 Write 5 Done Sent Done false
78882024-03-06T21:01:06.269Z 25 Acked 1024 Read 2 Done Sent Done false
78892024-03-06T21:01:06.269Z 26 Acked 1025 Read 8 Done Sent Done false
78902024-03-06T21:01:06.269Z 27 Acked 1026 Read 4 Done Sent Done false
78912024-03-06T21:01:06.269Z 28 Acked 1027 Write 6 Done Sent Done false
78922024-03-06T21:01:06.269Z 29 Acked 1028 Write 1 Done Sent Done false
78932024-03-06T21:01:06.270Z 30 Acked 1029 Read 3 Done Sent Sent false
78942024-03-06T21:01:06.270Z STATES DS:0 DS:1 DS:2 TOTAL
78952024-03-06T21:01:06.270Z New 0 0 0 0
78962024-03-06T21:01:06.270Z Sent 0 20 1 21
78972024-03-06T21:01:06.270Z Done 30 10 29 69
78982024-03-06T21:01:06.270Z Skipped 0 0 0 0
78992024-03-06T21:01:06.270Z Error 0 0 0 0
79002024-03-06T21:01:06.270ZLast Flush: 1017 0 1017
79012024-03-06T21:01:06.270ZDownstairs last five completed:
79022024-03-06T21:01:06.270ZUpstairs last five completed: 30 29 28 27 26
79032024-03-06T21:01:06.270Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:01:04.427640303Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1427}
79042024-03-06T21:01:06.270ZWrote out file "/tmp/test_repair_verify.data"
79052024-03-06T21:01:06.270Z
79062024-03-06T21:01:06.270ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
79072024-03-06T21:01:06.270ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
79082024-03-06T21:01:06.271Z 0 000-019 70 70 70 118 118 118 F F F
79092024-03-06T21:01:06.271Z 1 020-039 72 69 72 123 115 123 F F F <---
79102024-03-06T21:01:06.271Z 2 040-059 72 71 72 122 121 122 F T F <---
79112024-03-06T21:01:06.271Z 3 060-079 64 64 64 109 109 109 F F F
79122024-03-06T21:01:06.271Z 4 080-099 61 61 61 102 102 102 F F F
79132024-03-06T21:01:06.271Z 5 100-119 70 70 70 120 120 120 F F F
79142024-03-06T21:01:06.271Z 6 120-139 56 56 56 93 93 93 F F F
79152024-03-06T21:01:06.271Z 7 140-159 69 69 69 115 115 115 F F F
79162024-03-06T21:01:06.271Z 8 160-179 68 68 68 114 114 114 F F F
79172024-03-06T21:01:06.271Z 9 180-199 70 70 70 120 120 120 F F F
79182024-03-06T21:01:06.271Z 10 200-219 69 69 69 115 115 115 T F T <---
79192024-03-06T21:01:06.271Z 11 220-239 70 70 70 118 118 118 T F T <---
79202024-03-06T21:01:06.271Z 12 240-259 72 70 72 122 118 122 F T F <---
79212024-03-06T21:01:06.271Z 13 260-279 70 70 70 118 118 118 F F F
79222024-03-06T21:01:06.271Z 14 280-299 70 70 70 118 118 118 F F F
79232024-03-06T21:01:06.272Z 15 300-319 72 70 72 122 117 122 F T F <---
79242024-03-06T21:01:06.272Z 16 320-339 72 71 72 122 121 122 F T F <---
79252024-03-06T21:01:06.272Z 17 340-359 69 69 69 115 115 115 F F F
79262024-03-06T21:01:06.272Z 18 360-379 70 70 70 120 120 120 F F F
79272024-03-06T21:01:06.272Z 19 380-399 70 70 70 118 118 118 F F F
79282024-03-06T21:01:06.272Z 20 400-419 70 70 70 120 120 120 F F F
79292024-03-06T21:01:06.272Z 21 420-439 71 71 71 121 121 121 F F F
79302024-03-06T21:01:06.272Z 22 440-459 71 71 71 121 121 121 F F F
79312024-03-06T21:01:06.272Z 23 460-479 72 69 72 122 115 122 F T F <---
79322024-03-06T21:01:06.272Z 24 480-499 63 63 63 105 105 105 T F T <---
79332024-03-06T21:01:06.272Z 25 500-519 72 70 72 122 120 122 T T T <---
79342024-03-06T21:01:06.272Z 26 520-539 70 70 70 120 120 120 F F F
79352024-03-06T21:01:06.272Z 27 540-559 70 70 70 120 120 120 F F F
79362024-03-06T21:01:06.279Z 28 560-579 69 69 69 115 115 115 T F T <---
79372024-03-06T21:01:06.279Z 29 580-599 70 70 70 120 120 120 F F F
79382024-03-06T21:01:06.279ZMax gen: 72, Max flush: 123
79392024-03-06T21:01:06.279ZError: Difference in extent metadata found!
79402024-03-06T21:01:06.279ZA Difference in extent metadata is expected here
79412024-03-06T21:01:06.279ZOn loop 36
79422024-03-06T21:01:06.279Z
79432024-03-06T21:01:06.279ZVerifying data now
79442024-03-06T21:01:11.302ZNo change after verify
79452024-03-06T21:01:11.303ZLoop: 36 Downstairs dump after verify (and repair):
79462024-03-06T21:01:11.313ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
79472024-03-06T21:01:11.313Z 0 000-019 70 70 70 118 118 118 F F F
79482024-03-06T21:01:11.314Z 1 020-039 72 72 72 123 123 123 F F F
79492024-03-06T21:01:11.314Z 2 040-059 72 72 72 122 122 122 F F F
79502024-03-06T21:01:11.314Z 3 060-079 64 64 64 109 109 109 F F F
79512024-03-06T21:01:11.314Z 4 080-099 61 61 61 102 102 102 F F F
79522024-03-06T21:01:11.315Z 5 100-119 70 70 70 120 120 120 F F F
79532024-03-06T21:01:11.315Z 6 120-139 56 56 56 93 93 93 F F F
79542024-03-06T21:01:11.315Z 7 140-159 69 69 69 115 115 115 F F F
79552024-03-06T21:01:11.316Z 8 160-179 68 68 68 114 114 114 F F F
79562024-03-06T21:01:11.316Z 9 180-199 70 70 70 120 120 120 F F F
79572024-03-06T21:01:11.316Z 10 200-219 73 73 73 124 124 124 F F F
79582024-03-06T21:01:11.322Z 11 220-239 73 73 73 124 124 124 F F F
79592024-03-06T21:01:11.323Z 12 240-259 72 72 72 122 122 122 F F F
79602024-03-06T21:01:11.323Z 13 260-279 70 70 70 118 118 118 F F F
79612024-03-06T21:01:11.323Z 14 280-299 70 70 70 118 118 118 F F F
79622024-03-06T21:01:11.323Z 15 300-319 72 72 72 122 122 122 F F F
79632024-03-06T21:01:11.324Z 16 320-339 72 72 72 122 122 122 F F F
79642024-03-06T21:01:11.324Z 17 340-359 69 69 69 115 115 115 F F F
79652024-03-06T21:01:11.324Z 18 360-379 70 70 70 120 120 120 F F F
79662024-03-06T21:01:11.324Z 19 380-399 70 70 70 118 118 118 F F F
79672024-03-06T21:01:11.324Z 20 400-419 70 70 70 120 120 120 F F F
79682024-03-06T21:01:11.324Z 21 420-439 71 71 71 121 121 121 F F F
79692024-03-06T21:01:11.325Z 22 440-459 71 71 71 121 121 121 F F F
79702024-03-06T21:01:11.325Z 23 460-479 72 72 72 122 122 122 F F F
79712024-03-06T21:01:11.325Z 24 480-499 73 73 73 124 124 124 F F F
79722024-03-06T21:01:11.326Z 25 500-519 73 73 73 124 124 124 F F F
79732024-03-06T21:01:11.326Z 26 520-539 70 70 70 120 120 120 F F F
79742024-03-06T21:01:11.326Z 27 540-559 70 70 70 120 120 120 F F F
79752024-03-06T21:01:11.327Z 28 560-579 73 73 73 124 124 124 F F F
79762024-03-06T21:01:11.327Z 29 580-599 70 70 70 120 120 120 F F F
79772024-03-06T21:01:11.327ZMax gen: 73, Max flush: 124
79782024-03-06T21:01:11.327Z
79792024-03-06T21:01:11.328ZBegin loop 37
79802024-03-06T21:01:11.328ZDownstairs to restart: 1
79812024-03-06T21:01:11.343Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.098370653Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435}
79822024-03-06T21:01:11.343Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.098641004Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435}
79832024-03-06T21:01:11.343Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.098659963Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435}
79842024-03-06T21:01:11.343Z{"msg":"Crucible 2b4e671a-72ef-4026-816d-d6cdb248b9bf has session id: 949b0fb5-e981-4f9d-9609-ee0815a7b24b","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.098677692Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
79852024-03-06T21:01:11.360Z{"msg":"Upstairs opts: Upstairs UUID: 2b4e671a-72ef-4026-816d-d6cdb248b9bf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.098727419Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
79862024-03-06T21:01:11.361Z{"msg":"Crucible stats registered with UUID: 2b4e671a-72ef-4026-816d-d6cdb248b9bf","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.098751757Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"Crucible runtime is spawned
79872024-03-06T21:01:11.361Zsession_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
79882024-03-06T21:01:11.361Z{"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.098868149Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435}
79892024-03-06T21:01:11.361Z{"msg":"2b4e671a-72ef-4026-816d-d6cdb248b9bf active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.098914795Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
79902024-03-06T21:01:11.361Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.098931644Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"client":"0","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
79912024-03-06T21:01:11.361Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.098961402Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"client":"0","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
79922024-03-06T21:01:11.362Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.098981791Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"client":"1","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
79932024-03-06T21:01:11.362Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.099017619Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"client":"1","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
79942024-03-06T21:01:11.362Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.099043657Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"client":"2","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
79952024-03-06T21:01:11.362Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.099077604Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"client":"2","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
79962024-03-06T21:01:11.362Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.099103422Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"":"io task","client":"0","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
79972024-03-06T21:01:11.362Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.09913803Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"":"io task","client":"1","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
79982024-03-06T21:01:11.362Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.099162558Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"":"io task","client":"2","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
79992024-03-06T21:01:11.362Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.099264702Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"":"io task","client":"2","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80002024-03-06T21:01:11.362Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.099338156Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"":"io task","client":"0","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80012024-03-06T21:01:11.362Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.099462058Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"":"io task","client":"1","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80022024-03-06T21:01:11.362Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.100109493Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"client":"2","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80032024-03-06T21:01:11.363Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.10014767Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"client":"2","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80042024-03-06T21:01:11.363Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.100164649Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"client":"0","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80052024-03-06T21:01:11.363Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.100178999Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"client":"0","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80062024-03-06T21:01:11.363Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.100850182Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"client":"2","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80072024-03-06T21:01:11.363Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.100908348Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"client":"0","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80082024-03-06T21:01:11.363Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.101304211Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"client":"1","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80092024-03-06T21:01:11.363Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.101368697Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"client":"1","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80102024-03-06T21:01:11.364Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.101513827Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"client":"2","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80112024-03-06T21:01:11.364Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.101551464Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"client":"2","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80122024-03-06T21:01:11.364Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.101567303Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80132024-03-06T21:01:11.364Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.101587551Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"client":"0","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80142024-03-06T21:01:11.364Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.1016015Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"client":"0","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80152024-03-06T21:01:11.364Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.101621749Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80162024-03-06T21:01:11.364Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.101917608Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"client":"1","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80172024-03-06T21:01:11.365Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.102358708Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"client":"1","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80182024-03-06T21:01:11.365Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.102398526Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"client":"1","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80192024-03-06T21:01:11.365Z{"msg":"[0]R flush_numbers[0..12]: [118, 123, 122, 109, 102, 120, 93, 115, 114, 120, 124, 124]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.102415624Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"":"downstairs","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80202024-03-06T21:01:11.365Z{"msg":"[0]R generation[0..12]: [70, 72, 72, 64, 61, 70, 56, 69, 68, 70, 73, 73]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.102438293Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"":"downstairs","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80212024-03-06T21:01:11.365Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.102453982Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"":"downstairs","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80222024-03-06T21:01:11.365Z{"msg":"[1]R flush_numbers[0..12]: [118, 123, 122, 109, 102, 120, 93, 115, 114, 120, 124, 124]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.102469241Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"":"downstairs","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80232024-03-06T21:01:11.365Z{"msg":"[1]R generation[0..12]: [70, 72, 72, 64, 61, 70, 56, 69, 68, 70, 73, 73]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.102490329Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"":"downstairs","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80242024-03-06T21:01:11.365Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
80252024-03-06T21:01:11.365Z,"time":"2024-03-06T21:01:10.102505308Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"":"downstairs","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80262024-03-06T21:01:11.365Z{"msg":"[2]R flush_numbers[0..12]: [118, 123, 122, 109, 102, 120, 93, 115, 114, 120, 124, 124]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.102531606Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"":"downstairs","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80272024-03-06T21:01:11.365Z{"msg":"[2]R generation[0..12]: [70, 72, 72, 64, 61, 70, 56, 69, 68, 70, 73, 73]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.102553695Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"":"downstairs","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80282024-03-06T21:01:11.365Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.102568404Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"":"downstairs","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80292024-03-06T21:01:11.365Z{"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.102583663Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"":"downstairs","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80302024-03-06T21:01:11.365Z{"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.102610871Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"":"downstairs","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80312024-03-06T21:01:11.365Z{"msg":"Next flush: 125","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.10262516Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"":"downstairs","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80322024-03-06T21:01:11.366Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.102646028Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"":"downstairs","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80332024-03-06T21:01:11.366Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.102659977Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80342024-03-06T21:01:11.366Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.102684016Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"client":"0","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80352024-03-06T21:01:11.366Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.102698985Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"client":"1","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80362024-03-06T21:01:11.366Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.102719973Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"client":"2","":"downstairs","session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80372024-03-06T21:01:11.366Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.102733692Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80382024-03-06T21:01:11.366Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.102746311Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80392024-03-06T21:01:11.366Z{"msg":"2b4e671a-72ef-4026-816d-d6cdb248b9bf is now active with session: 949b0fb5-e981-4f9d-9609-ee0815a7b24b","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.10276553Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80402024-03-06T21:01:11.366Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.102777649Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435,"session_id":"949b0fb5-e981-4f9d-9609-ee0815a7b24b"}
80412024-03-06T21:01:11.366Z{"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:10.102790048Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435}
80422024-03-06T21:01:11.366ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
80432024-03-06T21:01:11.366ZLoading write count information from file "/tmp/test_repair_verify.data"
80442024-03-06T21:01:11.366ZRun Repair workload
80452024-03-06T21:01:11.367Z01/30 Write block 236 len 36864 data: 9 10 11 11 9 10 9 7 6
80462024-03-06T21:01:11.367Z02/30 Write block 208 len 24576 data: 4 4 5 5 5 4
80472024-03-06T21:01:11.367Z03/30 Flush
80482024-03-06T21:01:11.373Z04/30 Write block 451 len 28672 data: 9 9 6 5 6 4 7
80492024-03-06T21:01:11.373Z05/30 Write block 560 len 4096 data: 6
80502024-03-06T21:01:11.373Z06/30 Read block 185 len 40960
80512024-03-06T21:01:11.373Z07/30 Write block 542 len 16384 data: 8 8 8 8
80522024-03-06T21:01:11.374Z08/30 Write block 262 len 8192 data: 3 5
80532024-03-06T21:01:11.374Z09/30 Write block 57 len 40960 data: 7 10 12 13 14 14 17 15 14 13
80542024-03-06T21:01:11.374Z10/30 Flush
80552024-03-06T21:01:11.374Z11/30 Write block 509 len 40960 data: 4 7 6 7 7 6 8 8 8 6
80562024-03-06T21:01:11.374Z12/30 Write block 395 len 36864 data: 4 4 6 6 3 3 2 3 4
80572024-03-06T21:01:11.374Z13/30 Read block 22 len 40960
80582024-03-06T21:01:11.374Z14/30 Write block 20 len 16384 data: 7 6 7 8
80592024-03-06T21:01:11.374Z15/30 Read block 73 len 16384
80602024-03-06T21:01:11.375Z16/30 Read block 352 len 32768
80612024-03-06T21:01:11.375Z17/30 Read block 170 len 24576
80622024-03-06T21:01:11.375Z18/30 Flush
80632024-03-06T21:01:11.380Z19/30 Write block 432 len 32768 data: 6 8 9 7 6 8 8 8
80642024-03-06T21:01:11.380Z20/30 Flush
80652024-03-06T21:01:11.380Z21/30 Flush
80662024-03-06T21:01:11.380Z22/30 Write block 178 len 40960 data: 5 8 9 11 9 7 7 6 6 5
80672024-03-06T21:01:11.380Z23/30 Read block 445 len 28672
80682024-03-06T21:01:11.380Z24/30 Read block 547 len 36864
80692024-03-06T21:01:11.389Z25/30 Read block 577 len 20480
80702024-03-06T21:01:11.389Z26/30 Write block 354 len 40960 data: 8 7 9 7 7 7 9 11 10 13
80712024-03-06T21:01:11.389Z27/30 Read block 524 len 32768
80722024-03-06T21:01:11.389Z28/30 Read block 248 len 4096
80732024-03-06T21:01:11.389Z29/30 Read block 121 len 4096
80742024-03-06T21:01:11.390Z30/30 Read block 161 len 16384
80752024-03-06T21:01:11.390Z----------------------------------------------------------------
80762024-03-06T21:01:11.390Z Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:30
80772024-03-06T21:01:11.390ZStates: Active Active Active
80782024-03-06T21:01:11.391ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
80792024-03-06T21:01:11.391Z 1 Acked 1000 Write 9 Done Done Done false
80802024-03-06T21:01:11.391Z 2 Acked 1001 Write 6 Done Sent Done false
80812024-03-06T21:01:11.391Z 3 Acked 1002 Flush 0 Done Sent Done false
80822024-03-06T21:01:11.391Z 4 Acked 1003 Write 7 Done Sent Done false
80832024-03-06T21:01:11.392Z 5 Acked 1004 Write 1 Done Sent Done false
80842024-03-06T21:01:11.392Z 6 Acked 1005 Read 10 Done Sent Done false
80852024-03-06T21:01:11.392Z 7 Acked 1006 Write 4 Done Sent Done false
80862024-03-06T21:01:11.392Z 8 Acked 1007 Write 2 Done Sent Done false
80872024-03-06T21:01:11.393Z 9 Acked 1008 Write 10 Done Sent Done false
80882024-03-06T21:01:11.393Z 10 Acked 1009 Flush 0 Done Sent Done false
80892024-03-06T21:01:11.393Z 11 Acked 1010 Write 10 Done Sent Done false
80902024-03-06T21:01:11.393Z 12 Acked 1011 Write 9 Done Sent Done false
80912024-03-06T21:01:11.393Z 13 Acked 1012 Read 10 Done Sent Done false
80922024-03-06T21:01:11.394Z 14 Acked 1013 Write 4 Done Sent Done false
80932024-03-06T21:01:11.394Z 15 Acked 1014 Read 4 Done Sent Done false
80942024-03-06T21:01:11.394Z 16 Acked 1015 Read 8 Done Sent Done false
80952024-03-06T21:01:11.394Z 17 Acked 1016 Read 6 Done Sent Done false
80962024-03-06T21:01:11.395Z 18 Acked 1017 Flush 0 Done Sent Done false
80972024-03-06T21:01:11.395Z 19 Acked 1018 Write 8 Done Sent Done false
80982024-03-06T21:01:11.395Z 20 Acked 1019 Flush 0 Done Sent Done false
80992024-03-06T21:01:11.395Z 21 Acked 1020 Flush 0 Done Sent Done false
81002024-03-06T21:01:11.395Z 22 Acked 1021 Write 10 Done Sent Done false
81012024-03-06T21:01:11.396Z 23 Acked 1022 Read 7 Done Sent Done false
81022024-03-06T21:01:11.396Z 24 Acked 1023 Read 9 Done Sent Done false
81032024-03-06T21:01:11.396Z 25 Acked 1024 Read 5 Done Sent Done false
81042024-03-06T21:01:11.396Z 26 Acked 1025 Write 10 Done Sent Done false
81052024-03-06T21:01:11.397Z 27 Acked 1026 Read 8 Done Sent Done false
81062024-03-06T21:01:11.397Z 28 Acked 1027 Read 1 Done Sent Done false
81072024-03-06T21:01:11.397Z 29 Acked 1028 Read 1 Done Sent Done false
81082024-03-06T21:01:11.397Z 30 Acked 1029 Read 4 Done Sent Done false
81092024-03-06T21:01:11.397Z STATES DS:0 DS:1 DS:2 TOTAL
81102024-03-06T21:01:11.397Z New 0 0 0 0
81112024-03-06T21:01:11.397Z Sent 0 29 0 29
81122024-03-06T21:01:11.398Z Done 30 1 30 61
81132024-03-06T21:01:11.398Z Skipped 0 0 0 0
81142024-03-06T21:01:11.398Z Error 0 0 0 0
81152024-03-06T21:01:11.398ZLast Flush: 1020 0 1020
81162024-03-06T21:01:11.398ZDownstairs last five completed:
81172024-03-06T21:01:11.398ZUpstairs last five completed: 30 29 28 27 26
81182024-03-06T21:01:11.399Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:01:10.146104913Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1435}
81192024-03-06T21:01:11.399ZWrote out file "/tmp/test_repair_verify.data"
81202024-03-06T21:01:11.415Z
81212024-03-06T21:01:11.416ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
81222024-03-06T21:01:11.434ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
81232024-03-06T21:01:11.434Z 0 000-019 70 70 70 118 118 118 F F F
81242024-03-06T21:01:11.434Z 1 020-039 74 72 74 127 123 127 F F F <---
81252024-03-06T21:01:11.435Z 2 040-059 74 72 74 126 122 126 F F F <---
81262024-03-06T21:01:11.435Z 3 060-079 74 64 74 126 109 126 F F F <---
81272024-03-06T21:01:11.435Z 4 080-099 61 61 61 102 102 102 F F F
81282024-03-06T21:01:11.435Z 5 100-119 70 70 70 120 120 120 F F F
81292024-03-06T21:01:11.436Z 6 120-139 56 56 56 93 93 93 F F F
81302024-03-06T21:01:11.436Z 7 140-159 69 69 69 115 115 115 F F F
81312024-03-06T21:01:11.436Z 8 160-179 68 68 68 114 114 114 T F T <---
81322024-03-06T21:01:11.436Z 9 180-199 70 70 70 120 120 120 T F T <---
81332024-03-06T21:01:11.437Z 10 200-219 74 73 74 125 124 125 F F F <---
81342024-03-06T21:01:11.437Z 11 220-239 74 73 74 125 124 125 F T F <---
81352024-03-06T21:01:11.437Z 12 240-259 74 72 74 125 122 125 F T F <---
81362024-03-06T21:01:11.437Z 13 260-279 74 70 74 126 118 126 F F F <---
81372024-03-06T21:01:11.437Z 14 280-299 70 70 70 118 118 118 F F F
81382024-03-06T21:01:11.437Z 15 300-319 72 72 72 122 122 122 F F F
81392024-03-06T21:01:11.438Z 16 320-339 72 72 72 122 122 122 F F F
81402024-03-06T21:01:11.438Z 17 340-359 69 69 69 115 115 115 T F T <---
81412024-03-06T21:01:11.438Z 18 360-379 70 70 70 120 120 120 T F T <---
81422024-03-06T21:01:11.438Z 19 380-399 74 70 74 127 118 127 F F F <---
81432024-03-06T21:01:11.438Z 20 400-419 74 70 74 127 120 127 F F F <---
81442024-03-06T21:01:11.439Z 21 420-439 74 71 74 128 121 128 F F F <---
81452024-03-06T21:01:11.439Z 22 440-459 74 71 74 126 121 126 F F F <---
81462024-03-06T21:01:11.439Z 23 460-479 72 72 72 122 122 122 F F F
81472024-03-06T21:01:11.439Z 24 480-499 73 73 73 124 124 124 F F F
81482024-03-06T21:01:11.439Z 25 500-519 74 73 74 127 124 127 F F F <---
81492024-03-06T21:01:11.440Z 26 520-539 70 70 70 120 120 120 F F F
81502024-03-06T21:01:11.440Z 27 540-559 74 70 74 126 120 126 F F F <---
81512024-03-06T21:01:11.440Z 28 560-579 74 73 74 126 124 126 F F F <---
81522024-03-06T21:01:11.440Z 29 580-599 70 70 70 120 120 120 F F F
81532024-03-06T21:01:11.441ZMax gen: 74, Max flush: 128
81542024-03-06T21:01:11.441ZError: Difference in extent metadata found!
81552024-03-06T21:01:11.441ZA Difference in extent metadata is expected here
81562024-03-06T21:01:11.441ZOn loop 37
81572024-03-06T21:01:11.441Z
81582024-03-06T21:01:11.442ZVerifying data now
81592024-03-06T21:01:17.330ZNo change after verify
81602024-03-06T21:01:17.330ZLoop: 37 Downstairs dump after verify (and repair):
81612024-03-06T21:01:17.345ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
81622024-03-06T21:01:17.345Z 0 000-019 70 70 70 118 118 118 F F F
81632024-03-06T21:01:17.345Z 1 020-039 74 74 74 127 127 127 F F F
81642024-03-06T21:01:17.345Z 2 040-059 74 74 74 126 126 126 F F F
81652024-03-06T21:01:17.345Z 3 060-079 74 74 74 126 126 126 F F F
81662024-03-06T21:01:17.345Z 4 080-099 61 61 61 102 102 102 F F F
81672024-03-06T21:01:17.345Z 5 100-119 70 70 70 120 120 120 F F F
81682024-03-06T21:01:17.345Z 6 120-139 56 56 56 93 93 93 F F F
81692024-03-06T21:01:17.345Z 7 140-159 69 69 69 115 115 115 F F F
81702024-03-06T21:01:17.345Z 8 160-179 75 75 75 129 129 129 F F F
81712024-03-06T21:01:17.345Z 9 180-199 75 75 75 129 129 129 F F F
81722024-03-06T21:01:17.345Z 10 200-219 74 74 74 125 125 125 F F F
81732024-03-06T21:01:17.345Z 11 220-239 74 74 74 125 125 125 F F F
81742024-03-06T21:01:17.345Z 12 240-259 74 74 74 125 125 125 F F F
81752024-03-06T21:01:17.345Z 13 260-279 74 74 74 126 126 126 F F F
81762024-03-06T21:01:17.345Z 14 280-299 70 70 70 118 118 118 F F F
81772024-03-06T21:01:17.345Z 15 300-319 72 72 72 122 122 122 F F F
81782024-03-06T21:01:17.346Z 16 320-339 72 72 72 122 122 122 F F F
81792024-03-06T21:01:17.346Z 17 340-359 75 75 75 129 129 129 F F F
81802024-03-06T21:01:17.346Z 18 360-379 75 75 75 129 129 129 F F F
81812024-03-06T21:01:17.346Z 19 380-399 74 74 74 127 127 127 F F F
81822024-03-06T21:01:17.346Z 20 400-419 74 74 74 127 127 127 F F F
81832024-03-06T21:01:17.349Z 21 420-439 74 74 74 128 128 128 F F F
81842024-03-06T21:01:17.349Z 22 440-459 74 74 74 126 126 126 F F F
81852024-03-06T21:01:17.349Z 23 460-479 72 72 72 122 122 122 F F F
81862024-03-06T21:01:17.349Z 24 480-499 73 73 73 124 124 124 F F F
81872024-03-06T21:01:17.349Z 25 500-519 74 74 74 127 127 127 F F F
81882024-03-06T21:01:17.349Z 26 520-539 70 70 70 120 120 120 F F F
81892024-03-06T21:01:17.349Z 27 540-559 74 74 74 126 126 126 F F F
81902024-03-06T21:01:17.350Z 28 560-579 74 74 74 126 126 126 F F F
81912024-03-06T21:01:17.350Z 29 580-599 70 70 70 120 120 120 F F F
81922024-03-06T21:01:17.350ZMax gen: 75, Max flush: 129
81932024-03-06T21:01:17.350Z
81942024-03-06T21:01:17.350ZBegin loop 38
81952024-03-06T21:01:17.350ZDownstairs to restart: 1
81962024-03-06T21:01:17.370Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.130394945Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443}
81972024-03-06T21:01:17.370Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.130644368Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443}
81982024-03-06T21:01:17.370Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.130668166Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443}
81992024-03-06T21:01:17.371Z{"msg":"Crucible e2947cee-14e2-460f-9bca-a8baaba77137 has session id: 7093f500-a4d5-4f9c-bc17-73107171298c","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.130690435Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82002024-03-06T21:01:17.371Z{"msg":"Crucible runtime is spawned
82012024-03-06T21:01:17.378ZUpstairs opts: Upstairs UUID: e2947cee-14e2-460f-9bca-a8baaba77137, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.13075214Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82022024-03-06T21:01:17.378Z{"msg":"Crucible stats registered with UUID: e2947cee-14e2-460f-9bca-a8baaba77137","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.130788558Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82032024-03-06T21:01:17.378Z{"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.130813766Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443}
82042024-03-06T21:01:17.378Z{"msg":"e2947cee-14e2-460f-9bca-a8baaba77137 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.131020552Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82052024-03-06T21:01:17.378Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.131067098Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"client":"0","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82062024-03-06T21:01:17.378Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.131094527Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"client":"0","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82072024-03-06T21:01:17.378Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.131138433Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"client":"1","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82082024-03-06T21:01:17.378Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.131181511Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"client":"1","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82092024-03-06T21:01:17.378Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.131214528Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"client":"2","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82102024-03-06T21:01:17.378Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.131251776Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"client":"2","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82112024-03-06T21:01:17.378Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.131297413Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"":"io task","client":"0","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82122024-03-06T21:01:17.378Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.131357478Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"":"io task","client":"1","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82132024-03-06T21:01:17.378Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.131397796Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"":"io task","client":"2","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82142024-03-06T21:01:17.378Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.131426704Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"":"io task","client":"0","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82152024-03-06T21:01:17.378Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.131569284Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"":"io task","client":"2","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82162024-03-06T21:01:17.378Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.131636449Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"":"io task","client":"1","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82172024-03-06T21:01:17.378Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.132291684Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"client":"0","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82182024-03-06T21:01:17.378Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.132329571Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"client":"0","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82192024-03-06T21:01:17.378Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.13234751Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"client":"2","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82202024-03-06T21:01:17.378Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.132366059Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"client":"2","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82212024-03-06T21:01:17.378Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.133002805Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"client":"0","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82222024-03-06T21:01:17.379Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.133148995Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"client":"2","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82232024-03-06T21:01:17.379Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.133302115Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"client":"1","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82242024-03-06T21:01:17.379Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.133338032Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"client":"1","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82252024-03-06T21:01:17.379Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.133465653Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"client":"0","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82262024-03-06T21:01:17.379Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.13350747Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"client":"0","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82272024-03-06T21:01:17.379Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.133525889Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82282024-03-06T21:01:17.379Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.133629772Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"client":"2","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82292024-03-06T21:01:17.379Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.133669999Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"client":"2","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82302024-03-06T21:01:17.379Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.133687198Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82312024-03-06T21:01:17.379Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.133906433Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"client":"1","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82322024-03-06T21:01:17.379Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.134309735Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"client":"1","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82332024-03-06T21:01:17.379Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.134344533Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"client":"1","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82342024-03-06T21:01:17.379Z{"msg":"[0]R flush_numbers[0..12]: [118, 127, 126, 126, 102, 120, 93, 115, 129, 129, 125, 125]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.134366721Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"":"downstairs","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82352024-03-06T21:01:17.379Z{"msg":"[0]R generation[0..12]: [70, 74, 74, 74, 61, 70, 56, 69, 75, 75, 74, 74]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.134389649Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"":"downstairs","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82362024-03-06T21:01:17.379Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.134404919Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"":"downstairs","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82372024-03-06T21:01:17.379Z{"msg":"[1]R flush_numbers[0..12]: [118, 127, 126, 126, 102, 120, 93, 115, 129, 129, 125, 125]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.134425687Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"":"downstairs","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82382024-03-06T21:01:17.379Z{"msg":"[1]R generation[0..12]: [70, 74, 74, 74, 61, 70, 56, 69, 75, 75, 74, 74]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.134439916Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"":"downstairs","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82392024-03-06T21:01:17.379Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.134461404Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"":"downstairs","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82402024-03-06T21:01:17.379Z{"msg":"[2]R flush_numbers[0..12]: [118, 127, 126, 126, 102, 120, 93, 115, 129, 129, 125, 125]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.134475694Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"":"downstairs","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82412024-03-06T21:01:17.379Z{"msg":"[2]R generation[0..12]: [70, 74, 74, 74, 61, 70, 56, 69, 75, 75, 74, 74]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.134496312Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"":"downstairs","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82422024-03-06T21:01:17.379Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.134510591Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO
82432024-03-06T21:01:17.379Zdownstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82442024-03-06T21:01:17.379Z{"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.1345285Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"":"downstairs","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82452024-03-06T21:01:17.380Z{"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.134548798Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"":"downstairs","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82462024-03-06T21:01:17.380Z{"msg":"Next flush: 130","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.134565737Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"":"downstairs","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82472024-03-06T21:01:17.380Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.134598455Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"":"downstairs","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82482024-03-06T21:01:17.380Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.134630713Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82492024-03-06T21:01:17.407Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.134652631Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"client":"0","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82502024-03-06T21:01:17.407Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.134684979Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"client":"1","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82512024-03-06T21:01:17.407Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.134706728Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"client":"2","":"downstairs","session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82522024-03-06T21:01:17.407Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.134736936Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82532024-03-06T21:01:17.407Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.134758804Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82542024-03-06T21:01:17.407Z{"msg":"e2947cee-14e2-460f-9bca-a8baaba77137 is now active with session: 7093f500-a4d5-4f9c-bc17-73107171298c","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.134788462Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82552024-03-06T21:01:17.407Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.13481014Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443,"session_id":"7093f500-a4d5-4f9c-bc17-73107171298c"}
82562024-03-06T21:01:17.407Z{"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:16.134843088Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443}
82572024-03-06T21:01:17.407ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
82582024-03-06T21:01:17.407ZLoading write count information from file "/tmp/test_repair_verify.data"
82592024-03-06T21:01:17.407ZRun Repair workload
82602024-03-06T21:01:17.407Z01/30 Write block 346 len 36864 data: 5 6 9 8 9 10 8 8 9
82612024-03-06T21:01:17.407Z02/30 Write block 101 len 12288 data: 6 7 7
82622024-03-06T21:01:17.407Z03/30 Write block 230 len 40960 data: 7 7 6 4 6 8 10 11 12 12
82632024-03-06T21:01:17.407Z04/30 Read block 402 len 4096
82642024-03-06T21:01:17.407Z05/30 Write block 142 len 4096 data: 3
82652024-03-06T21:01:17.407Z06/30 Write block 403 len 16384 data: 5 3 5 6
82662024-03-06T21:01:17.407Z07/30 Read block 215 len 40960
82672024-03-06T21:01:17.407Z08/30 Write block 22 len 40960 data: 8 9 7 6 7 7 5 5 6 6
82682024-03-06T21:01:17.407Z09/30 Read block 17 len 16384
82692024-03-06T21:01:17.408Z10/30 Read block 565 len 16384
82702024-03-06T21:01:17.408Z11/30 Read block 436 len 16384
82712024-03-06T21:01:17.408Z12/30 Read block 320 len 40960
82722024-03-06T21:01:17.408Z13/30 Write block 483 len 12288 data: 5 4 6
82732024-03-06T21:01:17.408Z14/30 Read block 168 len 12288
82742024-03-06T21:01:17.408Z15/30 Read block 173 len 40960
82752024-03-06T21:01:17.408Z16/30 Write block 421 len 36864 data: 12 10 8 7 7 7 7 7 7
82762024-03-06T21:01:17.408Z17/30 Read block 35 len 32768
82772024-03-06T21:01:17.408Z18/30 Read block 246 len 16384
82782024-03-06T21:01:17.408Z19/30 Read block 587 len 4096
82792024-03-06T21:01:17.408Z20/30 Flush
82802024-03-06T21:01:17.408Z21/30 Write block 134 len 40960 data: 4 4 5 4 3 3 3 3 4 4
82812024-03-06T21:01:17.408Z22/30 Write block 74 len 4096 data: 7
82822024-03-06T21:01:17.408Z23/30 Read block 465 len 40960
82832024-03-06T21:01:17.408Z24/30 Read block 214 len 40960
82842024-03-06T21:01:17.408Z25/30 Write block 405 len 24576 data: 6 7 6 7 5 5
82852024-03-06T21:01:17.408Z26/30 Flush
82862024-03-06T21:01:17.413Z27/30 Write block 356 len 12288 data: 10 8 8
82872024-03-06T21:01:17.413Z28/30 Read block 417 len 28672
82882024-03-06T21:01:17.413Z29/30 Write block 369 len 32768 data: 8 9 9 9 9 9 9 6
82892024-03-06T21:01:17.413Z30/30 Read block 169 len 4096
82902024-03-06T21:01:17.414Z----------------------------------------------------------------
82912024-03-06T21:01:17.414Z Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:30
82922024-03-06T21:01:17.414ZStates: Active Active Active
82932024-03-06T21:01:17.414ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
82942024-03-06T21:01:17.414Z 1 Acked 1000 Write 9 Done Done Done false
82952024-03-06T21:01:17.414Z 2 Acked 1001 Write 3 Done Done Done false
82962024-03-06T21:01:17.414Z 3 Acked 1002 Write 10 Done Sent Done false
82972024-03-06T21:01:17.414Z 4 Acked 1003 Read 1 Done Sent Done false
82982024-03-06T21:01:17.414Z 5 Acked 1004 Write 1 Done Sent Done false
82992024-03-06T21:01:17.414Z 6 Acked 1005 Write 4 Done Sent Done false
83002024-03-06T21:01:17.414Z 7 Acked 1006 Read 10 Done Sent Done false
83012024-03-06T21:01:17.414Z 8 Acked 1007 Write 10 Done Sent Done false
83022024-03-06T21:01:17.414Z 9 Acked 1008 Read 4 Done Sent Done false
83032024-03-06T21:01:17.414Z 10 Acked 1009 Read 4 Done Sent Done false
83042024-03-06T21:01:17.414Z 11 Acked 1010 Read 4 Done Sent Done false
83052024-03-06T21:01:17.414Z 12 Acked 1011 Read 10 Done Sent Done false
83062024-03-06T21:01:17.414Z 13 Acked 1012 Write 3 Done Sent Done false
83072024-03-06T21:01:17.414Z 14 Acked 1013 Read 3 Done Sent Done false
83082024-03-06T21:01:17.414Z 15 Acked 1014 Read 10 Done Sent Done false
83092024-03-06T21:01:17.414Z 16 Acked 1015 Write 9 Done Sent Done false
83102024-03-06T21:01:17.414Z 17 Acked 1016 Read 8 Done Sent Done false
83112024-03-06T21:01:17.414Z 18 Acked 1017 Read 4 Done Sent Done false
83122024-03-06T21:01:17.414Z 19 Acked 1018 Read 1 Done Sent Done false
83132024-03-06T21:01:17.414Z 20 Acked 1019 Flush 0 Done Sent Done false
83142024-03-06T21:01:17.415Z 21 Acked 1020 Write 10 Done Sent Done false
83152024-03-06T21:01:17.415Z 22 Acked 1021 Write 1 Done Sent Done false
83162024-03-06T21:01:17.415Z 23 Acked 1022 Read 10 Done Sent Done false
83172024-03-06T21:01:17.415Z 24 Acked 1023 Read 10 Done Sent Done false
83182024-03-06T21:01:17.415Z 25 Acked 1024 Write 6 Done Sent Done false
83192024-03-06T21:01:17.415Z 26 Acked 1025 Flush 0 Done Sent Done false
83202024-03-06T21:01:17.415Z 27 Acked 1026 Write 3 Done Sent Done false
83212024-03-06T21:01:17.415Z 28 Acked 1027 Read 7 Done Sent Done false
83222024-03-06T21:01:17.415Z 29 Acked 1028 Write 8 Done Sent Done false
83232024-03-06T21:01:17.415Z 30 Acked 1029 Read 1 Done Sent Done false
83242024-03-06T21:01:17.415Z STATES DS:0 DS:1 DS:2 TOTAL
83252024-03-06T21:01:17.415Z New 0 0 0 0
83262024-03-06T21:01:17.415Z Sent 0 28 0 28
83272024-03-06T21:01:17.415Z Done 30 2 30 62
83282024-03-06T21:01:17.415Z Skipped 0 0 0 0
83292024-03-06T21:01:17.415Z Error 0 0 0 0
83302024-03-06T21:01:17.415ZLast Flush: 1025 0 1025
83312024-03-06T21:01:17.415ZDownstairs last five completed:
83322024-03-06T21:01:17.415ZUpstairs last five completed: 30 29 28 27 26
83332024-03-06T21:01:17.415Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:01:16.171873396Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1443}
83342024-03-06T21:01:17.415ZWrote out file "/tmp/test_repair_verify.data"
83352024-03-06T21:01:17.422Z
83362024-03-06T21:01:17.428ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
83372024-03-06T21:01:17.443ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83382024-03-06T21:01:17.443Z 0 000-019 70 70 70 118 118 118 F F F
83392024-03-06T21:01:17.443Z 1 020-039 76 74 76 130 127 130 F F F <---
83402024-03-06T21:01:17.443Z 2 040-059 74 74 74 126 126 126 F F F
83412024-03-06T21:01:17.443Z 3 060-079 76 74 76 131 126 131 F F F <---
83422024-03-06T21:01:17.443Z 4 080-099 61 61 61 102 102 102 F F F
83432024-03-06T21:01:17.443Z 5 100-119 76 70 76 130 120 130 F T F <---
83442024-03-06T21:01:17.443Z 6 120-139 76 56 76 131 93 131 F F F <---
83452024-03-06T21:01:17.443Z 7 140-159 76 69 76 131 115 131 F F F <---
83462024-03-06T21:01:17.444Z 8 160-179 75 75 75 129 129 129 F F F
83472024-03-06T21:01:17.444Z 9 180-199 75 75 75 129 129 129 F F F
83482024-03-06T21:01:17.444Z 10 200-219 74 74 74 125 125 125 F F F
83492024-03-06T21:01:17.444Z 11 220-239 76 74 76 130 125 130 F F F <---
83502024-03-06T21:01:17.444Z 12 240-259 74 74 74 125 125 125 F F F
83512024-03-06T21:01:17.447Z 13 260-279 74 74 74 126 126 126 F F F
83522024-03-06T21:01:17.448Z 14 280-299 70 70 70 118 118 118 F F F
83532024-03-06T21:01:17.448Z 15 300-319 72 72 72 122 122 122 F F F
83542024-03-06T21:01:17.448Z 16 320-339 72 72 72 122 122 122 F F F
83552024-03-06T21:01:17.448Z 17 340-359 76 75 76 130 129 130 T T T <---
83562024-03-06T21:01:17.448Z 18 360-379 75 75 75 129 129 129 T F T <---
83572024-03-06T21:01:17.448Z 19 380-399 74 74 74 127 127 127 F F F
83582024-03-06T21:01:17.448Z 20 400-419 76 74 76 131 127 131 F F F <---
83592024-03-06T21:01:17.448Z 21 420-439 76 74 76 130 128 130 F F F <---
83602024-03-06T21:01:17.448Z 22 440-459 74 74 74 126 126 126 F F F
83612024-03-06T21:01:17.448Z 23 460-479 72 72 72 122 122 122 F F F
83622024-03-06T21:01:17.448Z 24 480-499 76 73 76 130 124 130 F F F <---
83632024-03-06T21:01:17.448Z 25 500-519 74 74 74 127 127 127 F F F
83642024-03-06T21:01:17.448Z 26 520-539 70 70 70 120 120 120 F F F
83652024-03-06T21:01:17.448Z 27 540-559 74 74 74 126 126 126 F F F
83662024-03-06T21:01:17.448Z 28 560-579 74 74 74 126 126 126 F F F
83672024-03-06T21:01:17.448Z 29 580-599 70 70 70 120 120 120 F F F
83682024-03-06T21:01:17.448ZMax gen: 76, Max flush: 131
83692024-03-06T21:01:17.448ZError: Difference in extent metadata found!
83702024-03-06T21:01:17.448ZA Difference in extent metadata is expected here
83712024-03-06T21:01:17.448ZOn loop 38
83722024-03-06T21:01:17.448Z
83732024-03-06T21:01:17.448ZVerifying data now
83742024-03-06T21:01:23.039ZNo change after verify
83752024-03-06T21:01:23.040ZLoop: 38 Downstairs dump after verify (and repair):
83762024-03-06T21:01:23.054ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83772024-03-06T21:01:23.054Z 0 000-019 70 70 70 118 118 118 F F F
83782024-03-06T21:01:23.054Z 1 020-039 76 76 76 130 130 130 F F F
83792024-03-06T21:01:23.054Z 2 040-059 74 74 74 126 126 126 F F F
83802024-03-06T21:01:23.055Z 3 060-079 76 76 76 131 131 131 F F F
83812024-03-06T21:01:23.055Z 4 080-099 61 61 61 102 102 102 F F F
83822024-03-06T21:01:23.055Z 5 100-119 76 76 76 130 130 130 F F F
83832024-03-06T21:01:23.055Z 6 120-139 76 76 76 131 131 131 F F F
83842024-03-06T21:01:23.055Z 7 140-159 76 76 76 131 131 131 F F F
83852024-03-06T21:01:23.055Z 8 160-179 75 75 75 129 129 129 F F F
83862024-03-06T21:01:23.055Z 9 180-199 75 75 75 129 129 129 F F F
83872024-03-06T21:01:23.055Z 10 200-219 74 74 74 125 125 125 F F F
83882024-03-06T21:01:23.055Z 11 220-239 76 76 76 130 130 130 F F F
83892024-03-06T21:01:23.055Z 12 240-259 74 74 74 125 125 125 F F F
83902024-03-06T21:01:23.055Z 13 260-279 74 74 74 126 126 126 F F F
83912024-03-06T21:01:23.055Z 14 280-299 70 70 70 118 118 118 F F F
83922024-03-06T21:01:23.055Z 15 300-319 72 72 72 122 122 122 F F F
83932024-03-06T21:01:23.055Z 16 320-339 72 72 72 122 122 122 F F F
83942024-03-06T21:01:23.055Z 17 340-359 77 77 77 132 132 132 F F F
83952024-03-06T21:01:23.055Z 18 360-379 77 77 77 132 132 132 F F F
83962024-03-06T21:01:23.055Z 19 380-399 74 74 74 127 127 127 F F F
83972024-03-06T21:01:23.062Z 20 400-419 76 76 76 131 131 131 F F F
83982024-03-06T21:01:23.062Z 21 420-439 76 76 76 130 130 130 F F F
83992024-03-06T21:01:23.062Z 22 440-459 74 74 74 126 126 126 F F F
84002024-03-06T21:01:23.062Z 23 460-479 72 72 72 122 122 122 F F F
84012024-03-06T21:01:23.062Z 24 480-499 76 76 76 130 130 130 F F F
84022024-03-06T21:01:23.062Z 25 500-519 74 74 74 127 127 127 F F F
84032024-03-06T21:01:23.062Z 26 520-539 70 70 70 120 120 120 F F F
84042024-03-06T21:01:23.062Z 27 540-559 74 74 74 126 126 126 F F F
84052024-03-06T21:01:23.062Z 28 560-579 74 74 74 126 126 126 F F F
84062024-03-06T21:01:23.062Z 29 580-599 70 70 70 120 120 120 F F F
84072024-03-06T21:01:23.062ZMax gen: 77, Max flush: 132
84082024-03-06T21:01:23.062Z
84092024-03-06T21:01:23.062ZBegin loop 39
84102024-03-06T21:01:23.062ZDownstairs to restart: 1
84112024-03-06T21:01:23.081Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.840855118Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451}
84122024-03-06T21:01:23.081Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
84132024-03-06T21:01:23.081Z,"time":"2024-03-06T21:01:21.841166207Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451}
84142024-03-06T21:01:23.086Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.841227852Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451}
84152024-03-06T21:01:23.086Z{"msg":"Crucible 6968b801-6166-4208-b729-b12a2d14f829 has session id: c7c7e82e-20c5-46a2-a6a6-0afab673266d","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.841273829Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84162024-03-06T21:01:23.086Z{"msg":"Upstairs opts: Upstairs UUID: 6968b801-6166-4208-b729-b12a2d14f829, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.841323896Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84172024-03-06T21:01:23.086Z{"msg":"Crucible stats registered with UUID: 6968b801-6166-4208-b729-b12a2d14f829","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.841355554Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84182024-03-06T21:01:23.086Z{"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.841393281Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451}
84192024-03-06T21:01:23.086Z{"msg":"6968b801-6166-4208-b729-b12a2d14f829 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.84141223Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84202024-03-06T21:01:23.086Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.841444088Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"client":"0","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84212024-03-06T21:01:23.086Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.841483245Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"client":"0","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84222024-03-06T21:01:23.086Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.841499534Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"client":"1","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84232024-03-06T21:01:23.086Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.841523672Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"client":"1","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84242024-03-06T21:01:23.086Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.841561229Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"client":"2","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84252024-03-06T21:01:23.086Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.841585398Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"client":"2","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84262024-03-06T21:01:23.086Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.841620615Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"":"io task","client":"0","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84272024-03-06T21:01:23.087Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.841654243Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"":"io task","client":"1","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84282024-03-06T21:01:23.087Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.841671812Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"":"io task","client":"2","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84292024-03-06T21:01:23.087Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.841822261Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"":"io task","client":"0","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84302024-03-06T21:01:23.087Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.841906936Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"":"io task","client":"2","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84312024-03-06T21:01:23.087Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.841969001Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"":"io task","client":"1","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84322024-03-06T21:01:23.087Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.842591088Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"client":"2","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84332024-03-06T21:01:23.087Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.842640755Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"client":"2","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84342024-03-06T21:01:23.087Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.842669293Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"client":"0","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84352024-03-06T21:01:23.087Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.842690962Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"client":"0","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84362024-03-06T21:01:23.087Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.843282161Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"client":"0","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84372024-03-06T21:01:23.087Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.84343709Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"client":"2","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84382024-03-06T21:01:23.087Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.843497186Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"client":"1","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84392024-03-06T21:01:23.087Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.843522204Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"client":"1","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84402024-03-06T21:01:23.087Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.843958934Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"client":"0","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84412024-03-06T21:01:23.087Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.843993322Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"client":"0","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84422024-03-06T21:01:23.087Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.844009061Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84432024-03-06T21:01:23.087Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.844080166Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"client":"2","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84442024-03-06T21:01:23.087Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.844125273Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"client":"2","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84452024-03-06T21:01:23.087Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.844150571Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84462024-03-06T21:01:23.087Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.844320189Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"client":"1","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84472024-03-06T21:01:23.092Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.844758109Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"client":"1","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84482024-03-06T21:01:23.092Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.844796706Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"client":"1","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84492024-03-06T21:01:23.092Z{"msg":"[0]R flush_numbers[0..12]: [118, 130, 126, 131, 102, 130, 131, 131, 129, 129, 125, 130]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.844812735Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"":"downstairs","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84502024-03-06T21:01:23.093Z{"msg":"[0]R generation[0..12]: [70, 76, 74, 76, 61, 76, 76, 76, 75, 75, 74, 76]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.844828424Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"":"downstairs","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84512024-03-06T21:01:23.093Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.844850893Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"":"downstairs","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84522024-03-06T21:01:23.093Z{"msg":"[1]R flush_numbers[0..12]: [118, 130, 126, 131, 102, 130, 131, 131, 129, 129, 125, 130]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.844865392Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"":"downstairs","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84532024-03-06T21:01:23.093Z{"msg":"[1]R generation[0..12]: [70, 76, 74, 76, 61, 76, 76, 76, 75, 75, 74, 76]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.844886Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"":"downstairs","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84542024-03-06T21:01:23.093Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.844900969Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"":"downstairs","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84552024-03-06T21:01:23.093Z{"msg":"[2]R flush_numbers[0..12]: [118, 130, 126, 131, 102, 130, 131, 131, 129, 129, 125, 130]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.844921398Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"":"downstairs","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84562024-03-06T21:01:23.093Z{"msg":"[2]R generation[0..12]: [70, 76, 74, 76, 61, 76, 76, 76, 75, 75, 74, 76]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.844935347Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"":"downstairs","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84572024-03-06T21:01:23.093Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.844955586Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"":"downstairs","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84582024-03-06T21:01:23.093Z{"msg":"Max found gen is 78","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
84592024-03-06T21:01:23.093Z,"time":"2024-03-06T21:01:21.844968685Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"":"downstairs","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84602024-03-06T21:01:23.093Z{"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.844982773Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"":"downstairs","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84612024-03-06T21:01:23.093Z{"msg":"Next flush: 133","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.844998872Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"":"downstairs","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84622024-03-06T21:01:23.093Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.84502805Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"":"downstairs","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84632024-03-06T21:01:23.093Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.845042789Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84642024-03-06T21:01:23.093Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.845055919Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"client":"0","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84652024-03-06T21:01:23.093Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.845069498Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"client":"1","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84662024-03-06T21:01:23.093Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.845083077Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"client":"2","":"downstairs","session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84672024-03-06T21:01:23.093Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.845104235Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84682024-03-06T21:01:23.093Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.845117044Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84692024-03-06T21:01:23.093Z{"msg":"6968b801-6166-4208-b729-b12a2d14f829 is now active with session: c7c7e82e-20c5-46a2-a6a6-0afab673266d","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.845136363Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84702024-03-06T21:01:23.093Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.845148452Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451,"session_id":"c7c7e82e-20c5-46a2-a6a6-0afab673266d"}
84712024-03-06T21:01:23.093Z{"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:21.845160651Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451}
84722024-03-06T21:01:23.093ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
84732024-03-06T21:01:23.094ZLoading write count information from file "/tmp/test_repair_verify.data"
84742024-03-06T21:01:23.094ZRun Repair workload
84752024-03-06T21:01:23.094Z01/30 Flush
84762024-03-06T21:01:23.094Z02/30 Write block 22 len 36864 data: 9 10 8 7 8 8 6 6 7
84772024-03-06T21:01:23.094Z03/30 Flush
84782024-03-06T21:01:23.097Z04/30 Write block 363 len 36864 data: 14 11 10 8 9 6 9 10 10
84792024-03-06T21:01:23.097Z05/30 Read block 21 len 4096
84802024-03-06T21:01:23.097Z06/30 Write block 162 len 4096 data: 6
84812024-03-06T21:01:23.098Z07/30 Read block 3 len 4096
84822024-03-06T21:01:23.098Z08/30 Read block 514 len 8192
84832024-03-06T21:01:23.098Z09/30 Read block 304 len 20480
84842024-03-06T21:01:23.098Z10/30 Flush
84852024-03-06T21:01:23.102Z11/30 Write block 276 len 40960 data: 9 7 7 8 8 9 9 7 9 10
84862024-03-06T21:01:23.102Z12/30 Read block 541 len 16384
84872024-03-06T21:01:23.105Z13/30 Read block 588 len 20480
84882024-03-06T21:01:23.105Z14/30 Write block 125 len 28672 data: 7 6 8 9 9 8 6
84892024-03-06T21:01:23.105Z15/30 Read block 495 len 12288
84902024-03-06T21:01:23.108Z16/30 Read block 539 len 12288
84912024-03-06T21:01:23.108Z17/30 Read block 72 len 4096
84922024-03-06T21:01:23.108Z18/30 Write block 192 len 28672 data: 8 7 6 6 8 7 9
84932024-03-06T21:01:23.108Z19/30 Read block 231 len 20480
84942024-03-06T21:01:23.112Z20/30 Flush
84952024-03-06T21:01:23.122Z21/30 Write block 117 len 24576 data: 4 3 3 3 3 4
84962024-03-06T21:01:23.122Z22/30 Read block 297 len 36864
84972024-03-06T21:01:23.122Z23/30 Write block 464 len 40960 data: 12 12 10 9 9 8 5 5 4 4
84982024-03-06T21:01:23.122Z24/30 Read block 285 len 16384
84992024-03-06T21:01:23.122Z25/30 Write block 233 len 8192 data: 5 7
85002024-03-06T21:01:23.122Z26/30 Flush
85012024-03-06T21:01:23.533Z27/30 Write block 336 len 12288 data: 5 6 8
85022024-03-06T21:01:23.533Z28/30 Read block 150 len 24576
85032024-03-06T21:01:23.533Z29/30 Write block 570 len 12288 data: 5 5 5
85042024-03-06T21:01:23.533Z30/30 Write block 595 len 16384 data: 5 4 4 3
85052024-03-06T21:01:23.533Z----------------------------------------------------------------
85062024-03-06T21:01:23.533Z Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:27
85072024-03-06T21:01:23.534ZStates: Active Active Active
85082024-03-06T21:01:23.534ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
85092024-03-06T21:01:23.534Z 4 Acked 1003 Write 9 Done Done Done false
85102024-03-06T21:01:23.534Z 5 Acked 1004 Read 1 Done Done Done false
85112024-03-06T21:01:23.534Z 6 Acked 1005 Write 1 Done Done Done false
85122024-03-06T21:01:23.534Z 7 Acked 1006 Read 1 Done Done Done false
85132024-03-06T21:01:23.534Z 8 Acked 1007 Read 2 Done Sent Done false
85142024-03-06T21:01:23.534Z 9 Acked 1008 Read 5 Done Sent Done false
85152024-03-06T21:01:23.534Z 10 Acked 1009 Flush 0 Done Sent Done false
85162024-03-06T21:01:23.534Z 11 Acked 1010 Write 10 Done Sent Done false
85172024-03-06T21:01:23.534Z 12 Acked 1011 Read 4 Done Sent Done false
85182024-03-06T21:01:23.534Z 13 Acked 1012 Read 5 Done Sent Done false
85192024-03-06T21:01:23.534Z 14 Acked 1013 Write 7 Done Sent Done false
85202024-03-06T21:01:23.534Z 15 Acked 1014 Read 3 Done Sent Done false
85212024-03-06T21:01:23.534Z 16 Acked 1015 Read 3 Done Sent Done false
85222024-03-06T21:01:23.534Z 17 Acked 1016 Read 1 Done Sent Done false
85232024-03-06T21:01:23.534Z 18 Acked 1017 Write 7 Done Sent Done false
85242024-03-06T21:01:23.534Z 19 Acked 1018 Read 5 Done Sent Done false
85252024-03-06T21:01:23.534Z 20 Acked 1019 Flush 0 Done Sent Done false
85262024-03-06T21:01:23.534Z 21 Acked 1020 Write 6 Done Sent Done false
85272024-03-06T21:01:23.534Z 22 Acked 1021 Read 9 Done Sent Done false
85282024-03-06T21:01:23.534Z 23 Acked 1022 Write 10 Done Sent Done false
85292024-03-06T21:01:23.534Z 24 Acked 1023 Read 4 Done Sent Done false
85302024-03-06T21:01:23.535Z 25 Acked 1024 Write 2 Done Sent Done false
85312024-03-06T21:01:23.535Z 26 Acked 1025 Flush 0 Done Sent Done false
85322024-03-06T21:01:23.535Z 27 Acked 1026 Write 3 Done Sent Done false
85332024-03-06T21:01:23.535Z 28 Acked 1027 Read 6 Done Sent Done false
85342024-03-06T21:01:23.535Z 29 Acked 1028 Write 3 Done Sent Done false
85352024-03-06T21:01:23.535Z 30 Acked 1029 Write 4 Sent Sent Sent false
85362024-03-06T21:01:23.535Z STATES DS:0 DS:1 DS:2 TOTAL
85372024-03-06T21:01:23.535Z New 0 0 0 0
85382024-03-06T21:01:23.535Z Sent 1 23 1 25
85392024-03-06T21:01:23.535Z Done 26 4 26 56
85402024-03-06T21:01:23.535Z Skipped 0 0 0 0
85412024-03-06T21:01:23.535Z Error 0 0 0 0
85422024-03-06T21:01:23.535ZLast Flush: 1025 1002 1025
85432024-03-06T21:01:23.535ZDownstairs last five completed: 1002 1001 1000
85442024-03-06T21:01:23.535ZUpstairs last five completed: 30 29 28 27 26
85452024-03-06T21:01:23.535Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:01:21.883218578Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1451}
85462024-03-06T21:01:23.535ZWrote out file "/tmp/test_repair_verify.data"
85472024-03-06T21:01:23.535Z
85482024-03-06T21:01:23.535ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
85492024-03-06T21:01:23.535ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
85502024-03-06T21:01:23.535Z 0 000-019 70 70 70 118 118 118 F F F
85512024-03-06T21:01:23.535Z 1 020-039 78 78 78 134 134 134 F F F
85522024-03-06T21:01:23.535Z 2 040-059 74 74 74 126 126 126 F F F
85532024-03-06T21:01:23.535Z 3 060-079 76 76 76 131 131 131 F F F
85542024-03-06T21:01:23.536Z 4 080-099 61 61 61 102 102 102 F F F
85552024-03-06T21:01:23.536Z 5 100-119 78 76 78 137 130 137 F F F <---
85562024-03-06T21:01:23.536Z 6 120-139 78 76 78 137 131 137 F F F <---
85572024-03-06T21:01:23.536Z 7 140-159 76 76 76 131 131 131 F F F
85582024-03-06T21:01:23.536Z 8 160-179 78 75 78 135 129 135 F T F <---
85592024-03-06T21:01:23.536Z 9 180-199 78 75 78 136 129 136 F F F <---
85602024-03-06T21:01:23.536Z 10 200-219 74 74 74 125 125 125 F F F
85612024-03-06T21:01:23.536Z 11 220-239 78 76 78 137 130 137 F F F <---
85622024-03-06T21:01:23.536Z 12 240-259 74 74 74 125 125 125 F F F
85632024-03-06T21:01:23.536Z 13 260-279 78 74 78 136 126 136 F F F <---
85642024-03-06T21:01:23.536Z 14 280-299 78 70 78 136 118 136 F F F <---
85652024-03-06T21:01:23.536Z 15 300-319 72 72 72 122 122 122 F F F
85662024-03-06T21:01:23.536Z 16 320-339 72 72 72 122 122 122 T F T <---
85672024-03-06T21:01:23.536Z 17 340-359 77 77 77 132 132 132 F F F
85682024-03-06T21:01:23.536Z 18 360-379 78 77 78 135 132 135 F T F <---
85692024-03-06T21:01:23.536Z 19 380-399 74 74 74 127 127 127 F F F
85702024-03-06T21:01:23.536Z 20 400-419 76 76 76 131 131 131 F F F
85712024-03-06T21:01:23.536Z 21 420-439 76 76 76 130 130 130 F F F
85722024-03-06T21:01:23.536Z 22 440-459 74 74 74 126 126 126 F F F
85732024-03-06T21:01:23.536Z 23 460-479 78 72 78 137 122 137 F F F <---
85742024-03-06T21:01:23.536Z 24 480-499 76 76 76 130 130 130 F F F
85752024-03-06T21:01:23.536Z 25 500-519 74 74 74 127 127 127 F F F
85762024-03-06T21:01:23.536Z 26 520-539 70 70 70 120 120 120 F F F
85772024-03-06T21:01:23.536Z 27 540-559 74 74 74 126 126 126 F F F
85782024-03-06T21:01:23.537Z 28 560-579 74 74 74 126 126 126 T F T <---
85792024-03-06T21:01:23.537Z 29 580-599 70 70 70 120 120 120 T F T <---
85802024-03-06T21:01:23.537ZMax gen: 78, Max flush: 137
85812024-03-06T21:01:23.537ZError: Difference in extent metadata found!
85822024-03-06T21:01:23.537ZA Difference in extent metadata is expected here
85832024-03-06T21:01:23.537ZOn loop 39
85842024-03-06T21:01:23.537Z
85852024-03-06T21:01:23.537ZVerifying data now
85862024-03-06T21:01:28.797ZNo change after verify
85872024-03-06T21:01:28.798ZLoop: 39 Downstairs dump after verify (and repair):
85882024-03-06T21:01:28.816ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
85892024-03-06T21:01:28.816Z 0 000-019 70 70 70 118 118 118 F F F
85902024-03-06T21:01:28.816Z 1 020-039 78 78 78 134 134 134 F F F
85912024-03-06T21:01:28.816Z 2 040-059 74 74 74 126 126 126 F F F
85922024-03-06T21:01:28.816Z 3 060-079 76 76 76 131 131 131 F F F
85932024-03-06T21:01:28.816Z 4 080-099 61 61 61 102 102 102 F F F
85942024-03-06T21:01:28.816Z 5 100-119 78 78 78 137 137 137 F F F
85952024-03-06T21:01:28.817Z 6 120-139 78 78 78 137 137 137 F F F
85962024-03-06T21:01:28.817Z 7 140-159 76 76 76 131 131 131 F F F
85972024-03-06T21:01:28.817Z 8 160-179 78 78 78 135 135 135 F F F
85982024-03-06T21:01:28.817Z 9 180-199 78 78 78 136 136 136 F F F
85992024-03-06T21:01:28.817Z 10 200-219 74 74 74 125 125 125 F F F
86002024-03-06T21:01:28.817Z 11 220-239 78 78 78 137 137 137 F F F
86012024-03-06T21:01:28.817Z 12 240-259 74 74 74 125 125 125 F F F
86022024-03-06T21:01:28.817Z 13 260-279 78 78 78 136 136 136 F F F
86032024-03-06T21:01:28.817Z 14 280-299 78 78 78 136 136 136 F F F
86042024-03-06T21:01:28.817Z 15 300-319 72 72 72 122 122 122 F F F
86052024-03-06T21:01:28.817Z 16 320-339 79 79 79 138 138 138 F F F
86062024-03-06T21:01:28.828Z 17 340-359 77 77 77 132 132 132 F F F
86072024-03-06T21:01:28.828Z 18 360-379 78 78 78 135 135 135 F F F
86082024-03-06T21:01:28.828Z 19 380-399 74 74 74 127 127 127 F F F
86092024-03-06T21:01:28.828Z 20 400-419 76 76 76 131 131 131 F F F
86102024-03-06T21:01:28.828Z 21 420-439 76 76 76 130 130 130 F F F
86112024-03-06T21:01:28.828Z 22 440-459 74 74 74 126 126 126 F F F
86122024-03-06T21:01:28.828Z 23 460-479 78 78 78 137 137 137 F F F
86132024-03-06T21:01:28.828Z 24 480-499 76 76 76 130 130 130 F F F
86142024-03-06T21:01:28.828Z 25 500-519 74 74 74 127 127 127 F F F
86152024-03-06T21:01:28.828Z 26 520-539 70 70 70 120 120 120 F F F
86162024-03-06T21:01:28.828Z 27 540-559 74 74 74 126 126 126 F F F
86172024-03-06T21:01:28.828Z 28 560-579 79 79 79 138 138 138 F F F
86182024-03-06T21:01:28.829Z 29 580-599 79 79 79 138 138 138 F F F
86192024-03-06T21:01:28.829ZMax gen: 79, Max flush: 138
86202024-03-06T21:01:28.829Z
86212024-03-06T21:01:28.829ZBegin loop 40
86222024-03-06T21:01:28.829ZDownstairs to restart: 2
86232024-03-06T21:01:28.840Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.596270992Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459}
86242024-03-06T21:01:28.840Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.596510645Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459}
86252024-03-06T21:01:28.840Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.596527204Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459}
86262024-03-06T21:01:28.840Z{"msg":"Crucible 1ba1d953-7de2-4e5e-aa50-436b8e7b8b6e has session id: 151016ed-0064-41ce-a900-4647591613bc","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.596540693Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86272024-03-06T21:01:28.840Z{"msg":"Upstairs opts: Upstairs UUID: 1ba1d953-7de2-4e5e-aa50-436b8e7b8b6e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
86282024-03-06T21:01:28.840Z,"time":"2024-03-06T21:01:27.59658953Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86292024-03-06T21:01:28.840Z{"msg":"Crucible stats registered with UUID: 1ba1d953-7de2-4e5e-aa50-436b8e7b8b6e","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.596630007Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86302024-03-06T21:01:28.840Z{"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.596662195Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459}
86312024-03-06T21:01:28.850Z{"msg":"1ba1d953-7de2-4e5e-aa50-436b8e7b8b6e active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.596890489Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86322024-03-06T21:01:28.850Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.596932526Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"client":"0","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86332024-03-06T21:01:28.850Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.596954455Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"client":"0","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86342024-03-06T21:01:28.850Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.596979633Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"client":"1","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86352024-03-06T21:01:28.851Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.59702725Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"client":"1","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86362024-03-06T21:01:28.851Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.597054138Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"client":"2","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86372024-03-06T21:01:28.851Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.597091215Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"client":"2","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86382024-03-06T21:01:28.851Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.597117593Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"":"io task","client":"2","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86392024-03-06T21:01:28.851Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.59716452Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"":"io task","client":"0","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86402024-03-06T21:01:28.851Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.597205478Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"":"io task","client":"1","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86412024-03-06T21:01:28.851Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.597392395Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"":"io task","client":"0","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86422024-03-06T21:01:28.852Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.597432132Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"":"io task","client":"1","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86432024-03-06T21:01:28.852Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.59746225Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"":"io task","client":"2","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86442024-03-06T21:01:28.852Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.598236456Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"client":"0","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86452024-03-06T21:01:28.852Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.598287103Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"client":"0","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86462024-03-06T21:01:28.852Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.598316731Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"client":"1","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86472024-03-06T21:01:28.852Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.598343499Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"client":"1","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86482024-03-06T21:01:28.852Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.598842895Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"client":"1","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86492024-03-06T21:01:28.853Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.599007303Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"client":"2","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86502024-03-06T21:01:28.853Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.599042091Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"client":"2","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86512024-03-06T21:01:28.853Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.59905882Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"client":"0","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86522024-03-06T21:01:28.853Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.599366509Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"client":"1","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86532024-03-06T21:01:28.853Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.599412346Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"client":"1","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86542024-03-06T21:01:28.853Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.599439693Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86552024-03-06T21:01:28.853Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.599569405Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"client":"0","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86562024-03-06T21:01:28.853Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.599605682Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"client":"0","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86572024-03-06T21:01:28.854Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.599621671Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86582024-03-06T21:01:28.854Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.599649239Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"client":"2","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86592024-03-06T21:01:28.854Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.600025993Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"client":"2","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86602024-03-06T21:01:28.854Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.60007224Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"client":"2","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86612024-03-06T21:01:28.854Z{"msg":"[0]R flush_numbers[0..12]: [118, 134, 126, 131, 102, 137, 137, 131, 135, 136, 125, 137]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.600090879Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"":"downstairs","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86622024-03-06T21:01:28.854Z{"msg":"[0]R generation[0..12]: [70, 78, 74, 76, 61, 78, 78, 76, 78, 78, 74, 78]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.600114137Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"":"downstairs","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86632024-03-06T21:01:28.854Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.600129846Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"":"downstairs","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86642024-03-06T21:01:28.855Z{"msg":"[1]R flush_numbers[0..12]: [118, 134, 126, 131, 102, 137, 137, 131, 135, 136, 125, 137]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.600150884Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"":"downstairs","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86652024-03-06T21:01:28.855Z{"msg":"[1]R generation[0..12]: [70, 78, 74, 76, 61, 78, 78, 76, 78, 78, 74, 78]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.600165594Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"":"downstairs","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86662024-03-06T21:01:28.855Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.600193162Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"":"downstairs","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86672024-03-06T21:01:28.855ZWait for a query_work_queue command to finish before sending IO
86682024-03-06T21:01:28.855Z{"msg":"[2]R flush_numbers[0..12]: [118, 134, 126, 131, 102, 137, 137, 131, 135, 136, 125, 137]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.600233469Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"":"downstairs","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86692024-03-06T21:01:28.855Z{"msg":"[2]R generation[0..12]: [70, 78, 74, 76, 61, 78, 78, 76, 78, 78, 74, 78]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.600270106Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"":"downstairs","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86702024-03-06T21:01:28.855Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.600308584Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"":"downstairs","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86712024-03-06T21:01:28.855Z{"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.600349341Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"":"downstairs","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86722024-03-06T21:01:28.855Z{"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.60036558Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"":"downstairs","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86732024-03-06T21:01:28.855Z{"msg":"Next flush: 139","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.600378689Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"":"downstairs","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86742024-03-06T21:01:28.855Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.600404197Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"":"downstairs","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86752024-03-06T21:01:28.856Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.600426385Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86762024-03-06T21:01:28.856Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.600461463Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"client":"0","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86772024-03-06T21:01:28.856Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.600493791Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"client":"1","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86782024-03-06T21:01:28.856Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.600531728Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"client":"2","":"downstairs","session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86792024-03-06T21:01:28.856Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.600557327Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86802024-03-06T21:01:28.856Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.600578225Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86812024-03-06T21:01:28.856Z{"msg":"1ba1d953-7de2-4e5e-aa50-436b8e7b8b6e is now active with session: 151016ed-0064-41ce-a900-4647591613bc","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.600614223Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86822024-03-06T21:01:28.856Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.600641421Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459,"session_id":"151016ed-0064-41ce-a900-4647591613bc"}
86832024-03-06T21:01:28.856Z{"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:27.60065444Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459}
86842024-03-06T21:01:28.856ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
86852024-03-06T21:01:28.856ZLoading write count information from file "/tmp/test_repair_verify.data"
86862024-03-06T21:01:28.856ZRun Repair workload
86872024-03-06T21:01:28.856Z01/30 Write block 18 len 4096 data: 6
86882024-03-06T21:01:28.856Z02/30 Write block 505 len 28672 data: 4 4 5 4 5 8 7
86892024-03-06T21:01:28.856Z03/30 Read block 164 len 32768
86902024-03-06T21:01:28.856Z04/30 Read block 308 len 12288
86912024-03-06T21:01:28.862Z05/30 Read block 210 len 12288
86922024-03-06T21:01:28.862Z06/30 Read block 510 len 32768
86932024-03-06T21:01:28.862Z07/30 Read block 444 len 28672
86942024-03-06T21:01:28.862Z08/30 Read block 259 len 28672
86952024-03-06T21:01:28.862Z09/30 Read block 451 len 32768
86962024-03-06T21:01:28.862Z10/30 Read block 3 len 36864
86972024-03-06T21:01:28.863Z11/30 Read block 548 len 40960
86982024-03-06T21:01:28.863Z12/30 Flush
86992024-03-06T21:01:28.863Z13/30 Write block 292 len 8192 data: 6 5
87002024-03-06T21:01:28.863Z14/30 Read block 496 len 32768
87012024-03-06T21:01:28.863Z15/30 Read block 77 len 12288
87022024-03-06T21:01:28.863Z16/30 Read block 497 len 24576
87032024-03-06T21:01:28.863Z17/30 Write block 13 len 12288 data: 7 6 7
87042024-03-06T21:01:28.863Z18/30 Flush
87052024-03-06T21:01:28.869Z19/30 Write block 0 len 4096 data: 2
87062024-03-06T21:01:28.869Z20/30 Read block 505 len 8192
87072024-03-06T21:01:28.869Z21/30 Read block 169 len 20480
87082024-03-06T21:01:28.869Z22/30 Write block 586 len 32768 data: 9 11 9 9 9 8 9 5
87092024-03-06T21:01:28.869Z23/30 Write block 124 len 24576 data: 5 8 7 9 10 10
87102024-03-06T21:01:28.870Z24/30 Read block 513 len 12288
87112024-03-06T21:01:28.887Z25/30 Read block 208 len 36864
87122024-03-06T21:01:28.887Z26/30 Read block 65 len 4096
87132024-03-06T21:01:28.887Z27/30 Read block 366 len 16384
87142024-03-06T21:01:28.887Z28/30 Read block 47 len 28672
87152024-03-06T21:01:28.887Z29/30 Read block 567 len 4096
87162024-03-06T21:01:28.887Z30/30 Read block 177 len 4096
87172024-03-06T21:01:28.887Z----------------------------------------------------------------
87182024-03-06T21:01:28.887Z Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:30
87192024-03-06T21:01:28.887ZStates: Active Active Active
87202024-03-06T21:01:28.887ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
87212024-03-06T21:01:28.887Z 1 Acked 1000 Write 1 Done Done Done false
87222024-03-06T21:01:28.888Z 2 Acked 1001 Write 7 Done Done Done false
87232024-03-06T21:01:28.888Z 3 Acked 1002 Read 8 Done Done Done false
87242024-03-06T21:01:28.888Z 4 Acked 1003 Read 3 Done Done Done false
87252024-03-06T21:01:28.888Z 5 Acked 1004 Read 3 Done Done Done false
87262024-03-06T21:01:28.888Z 6 Acked 1005 Read 8 Done Done Sent false
87272024-03-06T21:01:28.888Z 7 Acked 1006 Read 7 Done Done Sent false
87282024-03-06T21:01:28.888Z 8 Acked 1007 Read 7 Done Done Sent false
87292024-03-06T21:01:28.888Z 9 Acked 1008 Read 8 Done Done Sent false
87302024-03-06T21:01:28.888Z 10 Acked 1009 Read 9 Done Done Sent false
87312024-03-06T21:01:28.888Z 11 Acked 1010 Read 10 Done Done Sent false
87322024-03-06T21:01:28.888Z 12 Acked 1011 Flush 0 Done Done Sent false
87332024-03-06T21:01:28.888Z 13 Acked 1012 Write 2 Done Done Sent false
87342024-03-06T21:01:28.888Z 14 Acked 1013 Read 8 Done Done Sent false
87352024-03-06T21:01:28.889Z 15 Acked 1014 Read 3 Done Done Sent false
87362024-03-06T21:01:28.889Z 16 Acked 1015 Read 6 Done Done Sent false
87372024-03-06T21:01:28.889Z 17 Acked 1016 Write 3 Done Done Sent false
87382024-03-06T21:01:28.889Z 18 Acked 1017 Flush 0 Done Done Sent false
87392024-03-06T21:01:28.889Z 19 Acked 1018 Write 1 Done Done Sent false
87402024-03-06T21:01:28.889Z 20 Acked 1019 Read 2 Done Done Sent false
87412024-03-06T21:01:28.889Z 21 Acked 1020 Read 5 Done Done Sent false
87422024-03-06T21:01:28.889Z 22 Acked 1021 Write 8 Done Done Sent false
87432024-03-06T21:01:28.889Z 23 Acked 1022 Write 6 Done Done Sent false
87442024-03-06T21:01:28.889Z 24 Acked 1023 Read 3 Done Done Sent false
87452024-03-06T21:01:28.889Z 25 Acked 1024 Read 9 Done Done Sent false
87462024-03-06T21:01:28.889Z 26 Acked 1025 Read 1 Done Done Sent false
87472024-03-06T21:01:28.890Z 27 Acked 1026 Read 4 Done Done Sent false
87482024-03-06T21:01:28.890Z 28 Acked 1027 Read 7 Done Done Sent false
87492024-03-06T21:01:28.890Z 29 Acked 1028 Read 1 Done Done Sent false
87502024-03-06T21:01:28.890Z 30 Acked 1029 Read 1 Done Done Sent false
87512024-03-06T21:01:28.890Z STATES DS:0 DS:1 DS:2 TOTAL
87522024-03-06T21:01:28.890Z New 0 0 0 0
87532024-03-06T21:01:28.890Z Sent 0 0 25 25
87542024-03-06T21:01:28.891Z Done 30 30 5 65
87552024-03-06T21:01:28.891Z Skipped 0 0 0 0
87562024-03-06T21:01:28.891Z Error 0 0 0 0
87572024-03-06T21:01:28.891ZLast Flush: 1017 1017 0
87582024-03-06T21:01:28.891ZDownstairs last five completed:
87592024-03-06T21:01:28.891ZUpstairs last five completed: 30 29 28 27 26
87602024-03-06T21:01:28.891Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:01:27.632367564Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1459}
87612024-03-06T21:01:28.891ZWrote out file "/tmp/test_repair_verify.data"
87622024-03-06T21:01:28.897Z
87632024-03-06T21:01:28.898ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
87642024-03-06T21:01:28.909ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
87652024-03-06T21:01:28.910Z 0 000-019 80 80 70 140 140 118 T T T <---
87662024-03-06T21:01:28.910Z 1 020-039 78 78 78 134 134 134 F F F
87672024-03-06T21:01:28.910Z 2 040-059 74 74 74 126 126 126 F F F
87682024-03-06T21:01:28.911Z 3 060-079 76 76 76 131 131 131 F F F
87692024-03-06T21:01:28.911Z 4 080-099 61 61 61 102 102 102 F F F
87702024-03-06T21:01:28.911Z 5 100-119 78 78 78 137 137 137 F F F
87712024-03-06T21:01:28.911Z 6 120-139 78 78 78 137 137 137 T T F <---
87722024-03-06T21:01:28.911Z 7 140-159 76 76 76 131 131 131 F F F
87732024-03-06T21:01:28.911Z 8 160-179 78 78 78 135 135 135 F F F
87742024-03-06T21:01:28.912Z 9 180-199 78 78 78 136 136 136 F F F
87752024-03-06T21:01:28.912Z 10 200-219 74 74 74 125 125 125 F F F
87762024-03-06T21:01:28.912Z 11 220-239 78 78 78 137 137 137 F F F
87772024-03-06T21:01:28.912Z 12 240-259 74 74 74 125 125 125 F F F
87782024-03-06T21:01:28.913Z 13 260-279 78 78 78 136 136 136 F F F
87792024-03-06T21:01:28.913Z 14 280-299 80 80 78 140 140 136 F F F <---
87802024-03-06T21:01:28.913Z 15 300-319 72 72 72 122 122 122 F F F
87812024-03-06T21:01:28.913Z 16 320-339 79 79 79 138 138 138 F F F
87822024-03-06T21:01:28.914Z 17 340-359 77 77 77 132 132 132 F F F
87832024-03-06T21:01:28.914Z 18 360-379 78 78 78 135 135 135 F F F
87842024-03-06T21:01:28.914Z 19 380-399 74 74 74 127 127 127 F F F
87852024-03-06T21:01:28.914Z 20 400-419 76 76 76 131 131 131 F F F
87862024-03-06T21:01:28.915Z 21 420-439 76 76 76 130 130 130 F F F
87872024-03-06T21:01:28.915Z 22 440-459 74 74 74 126 126 126 F F F
87882024-03-06T21:01:28.915Z 23 460-479 78 78 78 137 137 137 F F F
87892024-03-06T21:01:28.916Z 24 480-499 76 76 76 130 130 130 F F F
87902024-03-06T21:01:28.916Z 25 500-519 80 80 74 139 139 127 F F T <---
87912024-03-06T21:01:28.916Z 26 520-539 70 70 70 120 120 120 F F F
87922024-03-06T21:01:28.916Z 27 540-559 74 74 74 126 126 126 F F F
87932024-03-06T21:01:28.916Z 28 560-579 79 79 79 138 138 138 F F F
87942024-03-06T21:01:28.917Z 29 580-599 79 79 79 138 138 138 T T F <---
87952024-03-06T21:01:28.917ZMax gen: 80, Max flush: 140
87962024-03-06T21:01:28.917ZError: Difference in extent metadata found!
87972024-03-06T21:01:28.929ZA Difference in extent metadata is expected here
87982024-03-06T21:01:28.930ZOn loop 40
87992024-03-06T21:01:28.930Z
88002024-03-06T21:01:28.930ZVerifying data now
88012024-03-06T21:01:33.244ZNo change after verify
88022024-03-06T21:01:33.244ZLoop: 40 Downstairs dump after verify (and repair):
88032024-03-06T21:01:33.259ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
88042024-03-06T21:01:33.259Z 0 000-019 81 81 81 141 141 141 F F F
88052024-03-06T21:01:33.260Z 1 020-039 78 78 78 134 134 134 F F F
88062024-03-06T21:01:33.260Z 2 040-059 74 74 74 126 126 126 F F F
88072024-03-06T21:01:33.260Z 3 060-079 76 76 76 131 131 131 F F F
88082024-03-06T21:01:33.260Z 4 080-099 61 61 61 102 102 102 F F F
88092024-03-06T21:01:33.260Z 5 100-119 78 78 78 137 137 137 F F F
88102024-03-06T21:01:33.260Z 6 120-139 81 81 81 141 141 141 F F F
88112024-03-06T21:01:33.260Z 7 140-159 76 76 76 131 131 131 F F F
88122024-03-06T21:01:33.260Z 8 160-179 78 78 78 135 135 135 F F F
88132024-03-06T21:01:33.260Z 9 180-199 78 78 78 136 136 136 F F F
88142024-03-06T21:01:33.261Z 10 200-219 74 74 74 125 125 125 F F F
88152024-03-06T21:01:33.261Z 11 220-239 78 78 78 137 137 137 F F F
88162024-03-06T21:01:33.261Z 12 240-259 74 74 74 125 125 125 F F F
88172024-03-06T21:01:33.261Z 13 260-279 78 78 78 136 136 136 F F F
88182024-03-06T21:01:33.261Z 14 280-299 80 80 80 140 140 140 F F F
88192024-03-06T21:01:33.261Z 15 300-319 72 72 72 122 122 122 F F F
88202024-03-06T21:01:33.261Z 16 320-339 79 79 79 138 138 138 F F F
88212024-03-06T21:01:33.261Z 17 340-359 77 77 77 132 132 132 F F F
88222024-03-06T21:01:33.261Z 18 360-379 78 78 78 135 135 135 F F F
88232024-03-06T21:01:33.262Z 19 380-399 74 74 74 127 127 127 F F F
88242024-03-06T21:01:33.262Z 20 400-419 76 76 76 131 131 131 F F F
88252024-03-06T21:01:33.262Z 21 420-439 76 76 76 130 130 130 F F F
88262024-03-06T21:01:33.262Z 22 440-459 74 74 74 126 126 126 F F F
88272024-03-06T21:01:33.262Z 23 460-479 78 78 78 137 137 137 F F F
88282024-03-06T21:01:33.262Z 24 480-499 76 76 76 130 130 130 F F F
88292024-03-06T21:01:33.262Z 25 500-519 80 80 80 139 139 139 F F F
88302024-03-06T21:01:33.266Z 26 520-539 70 70 70 120 120 120 F F F
88312024-03-06T21:01:33.267Z 27 540-559 74 74 74 126 126 126 F F F
88322024-03-06T21:01:33.267Z 28 560-579 79 79 79 138 138 138 F F F
88332024-03-06T21:01:33.267Z 29 580-599 81 81 81 141 141 141 F F F
88342024-03-06T21:01:33.267ZMax gen: 81, Max flush: 141
88352024-03-06T21:01:33.267Z
88362024-03-06T21:01:33.268ZBegin loop 41
88372024-03-06T21:01:33.268ZDownstairs to restart: 2
88382024-03-06T21:01:33.285Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.043601206Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467}
88392024-03-06T21:01:33.285Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.043871047Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467}
88402024-03-06T21:01:33.285Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.043907254Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467}
88412024-03-06T21:01:33.286Z{"msg":"Crucible 88a6567f-8cd0-4e4d-9cae-452d53e27db9 has session id: 82edfd21-9191-464f-bb1e-901ad1b25e3a","v":0,"name":"crucible","levelCrucible runtime is spawned
88422024-03-06T21:01:33.293Z":30,"time":"2024-03-06T21:01:32.04396275Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88432024-03-06T21:01:33.293Z{"msg":"Upstairs opts: Upstairs UUID: 88a6567f-8cd0-4e4d-9cae-452d53e27db9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.044010537Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88442024-03-06T21:01:33.293Z{"msg":"Crucible stats registered with UUID: 88a6567f-8cd0-4e4d-9cae-452d53e27db9","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.044042815Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88452024-03-06T21:01:33.293Z{"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.044084072Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467}
88462024-03-06T21:01:33.293Z{"msg":"88a6567f-8cd0-4e4d-9cae-452d53e27db9 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.044201944Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88472024-03-06T21:01:33.293Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.044239902Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"client":"0","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88482024-03-06T21:01:33.294Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.04426237Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"client":"0","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88492024-03-06T21:01:33.294Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.044298667Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"client":"1","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88502024-03-06T21:01:33.294Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.044325636Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"client":"1","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88512024-03-06T21:01:33.294Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.044349794Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"client":"2","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88522024-03-06T21:01:33.294Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.044396681Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"client":"2","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88532024-03-06T21:01:33.294Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.044421319Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"":"io task","client":"1","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88542024-03-06T21:01:33.294Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.044459296Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"":"io task","client":"0","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88552024-03-06T21:01:33.295Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.044484625Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"":"io task","client":"2","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88562024-03-06T21:01:33.295Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.044659533Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"":"io task","client":"1","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88572024-03-06T21:01:33.295Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.04470385Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"":"io task","client":"2","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88582024-03-06T21:01:33.295Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.044731068Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"":"io task","client":"0","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88592024-03-06T21:01:33.295Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.04555962Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"client":"0","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88602024-03-06T21:01:33.295Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.045600798Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"client":"0","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88612024-03-06T21:01:33.295Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.045633015Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"client":"1","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88622024-03-06T21:01:33.295Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.045668693Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"client":"1","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88632024-03-06T21:01:33.296Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.04629343Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"client":"1","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88642024-03-06T21:01:33.296Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.046355526Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"client":"0","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88652024-03-06T21:01:33.296Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.046385324Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"client":"2","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88662024-03-06T21:01:33.296Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.046408882Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"client":"2","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88672024-03-06T21:01:33.296Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.046977703Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"client":"1","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88682024-03-06T21:01:33.296Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.047027009Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"client":"1","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88692024-03-06T21:01:33.296Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.047046498Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88702024-03-06T21:01:33.296Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.047077776Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"client":"2","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88712024-03-06T21:01:33.297Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.047105494Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"client":"0","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88722024-03-06T21:01:33.297Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.047135902Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"client":"0","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88732024-03-06T21:01:33.297Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.047150351Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88742024-03-06T21:01:33.297Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.047559203Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"client":"2","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88752024-03-06T21:01:33.297Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.04759332Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"client":"2","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88762024-03-06T21:01:33.297Z{"msg":"[0]R flush_numbers[0..12]: [141, 134, 126, 131, 102, 137, 141, 131, 135, 136, 125, 137]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.047609739Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"":"downstairs","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88772024-03-06T21:01:33.297Z{"msg":"[0]R generation[0..12]: [81, 78, 74, 76, 61, 78, 81, 76, 78, 78, 74, 78]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.047631858Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"":"downstairs","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88782024-03-06T21:01:33.297Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.047647987Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"":"downstairs","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88792024-03-06T21:01:33.298Z{"msg":"[1]R flush_numbers[0..12]: [141, 134, 126, 131, 102, 137, 141, 131, 135, 136, 125, 137]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.047669255Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"":"downstairs","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88802024-03-06T21:01:33.298Z{"msg":"[1]R generation[0..12]: [81, 78, 74, 76, 61, 78, 81, 76, 78, 78, 74, 78]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.047683704Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"":"downstairs","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88812024-03-06T21:01:33.298Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.047705993Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"":"downstairs","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88822024-03-06T21:01:33.298Z{"msg":"[2]R flush_numbers[0..12]: [141, 134, 126, 131, 102, 137, 141, 131, 135, 136, 125, 137]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.047732291Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"":"downstairs","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88832024-03-06T21:01:33.298Z{"msg":"[2]R generation[0..12]: [81, 78, 74, 76, 61, 78, 81, 76, 78, 78, 74, 78]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.0477466Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"":"downstairs","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88842024-03-06T21:01:33.299ZWait for a query_work_queue command to finish before sending IO
88852024-03-06T21:01:33.299Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.047777038Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"":"downstairs","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88862024-03-06T21:01:33.299Z{"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.047802146Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"":"downstairs","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88872024-03-06T21:01:33.299Z{"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.047826874Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"":"downstairs","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88882024-03-06T21:01:33.299Z{"msg":"Next flush: 142","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.047840853Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"":"downstairs","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88892024-03-06T21:01:33.299Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.047862312Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"":"downstairs","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88902024-03-06T21:01:33.299Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.047876551Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88912024-03-06T21:01:33.300Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.047897119Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"client":"0","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88922024-03-06T21:01:33.300Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.047911858Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"client":"1","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88932024-03-06T21:01:33.300Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.047925197Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"client":"2","":"downstairs","session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88942024-03-06T21:01:33.300Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.047946426Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88952024-03-06T21:01:33.300Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.047959185Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88962024-03-06T21:01:33.300Z{"msg":"88a6567f-8cd0-4e4d-9cae-452d53e27db9 is now active with session: 82edfd21-9191-464f-bb1e-901ad1b25e3a","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.047971944Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88972024-03-06T21:01:33.301Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.047991383Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467,"session_id":"82edfd21-9191-464f-bb1e-901ad1b25e3a"}
88982024-03-06T21:01:33.301Z{"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:32.048004182Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467}
88992024-03-06T21:01:33.301ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
89002024-03-06T21:01:33.306ZLoading write count information from file "/tmp/test_repair_verify.data"
89012024-03-06T21:01:33.306ZRun Repair workload
89022024-03-06T21:01:33.306Z01/30 Write block 424 len 12288 data: 8 8 8
89032024-03-06T21:01:33.306Z02/30 Write block 237 len 12288 data: 12 13 13
89042024-03-06T21:01:33.307Z03/30 Read block 245 len 16384
89052024-03-06T21:01:33.307Z04/30 Read block 356 len 16384
89062024-03-06T21:01:33.307Z05/30 Read block 15 len 28672
89072024-03-06T21:01:33.307Z06/30 Read block 441 len 20480
89082024-03-06T21:01:33.307Z07/30 Write block 441 len 24576 data: 5 4 3 5 4 5
89092024-03-06T21:01:33.307Z08/30 Write block 2 len 28672 data: 4 5 6 7 6 7 6
89102024-03-06T21:01:33.308Z09/30 Read block 157 len 28672
89112024-03-06T21:01:33.308Z10/30 Read block 213 len 16384
89122024-03-06T21:01:33.308Z11/30 Read block 264 len 40960
89132024-03-06T21:01:33.308Z12/30 Read block 591 len 36864
89142024-03-06T21:01:33.308Z13/30 Read block 408 len 36864
89152024-03-06T21:01:33.308Z14/30 Read block 315 len 20480
89162024-03-06T21:01:33.308Z15/30 Flush
89172024-03-06T21:01:33.313Z16/30 Write block 579 len 32768 data: 3 3 3 3 5 8 9 10
89182024-03-06T21:01:33.313Z17/30 Write block 480 len 24576 data: 4 5 5 6 5 7
89192024-03-06T21:01:33.313Z18/30 Read block 312 len 24576
89202024-03-06T21:01:33.313Z19/30 Read block 110 len 20480
89212024-03-06T21:01:33.313Z20/30 Read block 363 len 32768
89222024-03-06T21:01:33.317Z21/30 Read block 512 len 12288
89232024-03-06T21:01:33.317Z22/30 Read block 65 len 8192
89242024-03-06T21:01:33.317Z23/30 Write block 36 len 16384 data: 4 2 4 5
89252024-03-06T21:01:33.317Z24/30 Read block 103 len 16384
89262024-03-06T21:01:33.317Z25/30 Read block 566 len 8192
89272024-03-06T21:01:33.317Z26/30 Read block 468 len 32768
89282024-03-06T21:01:33.321Z27/30 Write block 197 len 24576 data: 8 10 7 6 5 4
89292024-03-06T21:01:33.321Z28/30 Write block 586 len 36864 data: 11 12 10 10 10 9 10 6 6
89302024-03-06T21:01:33.321Z29/30 Read block 128 len 36864
89312024-03-06T21:01:33.328Z30/30 Read block 131 len 4096
89322024-03-06T21:01:33.328Z----------------------------------------------------------------
89332024-03-06T21:01:33.328Z Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:30
89342024-03-06T21:01:33.328ZStates: Active Active Active
89352024-03-06T21:01:33.328ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
89362024-03-06T21:01:33.328Z 1 Acked 1000 Write 3 Done Done Done false
89372024-03-06T21:01:33.328Z 2 Acked 1001 Write 3 Done Done Done false
89382024-03-06T21:01:33.328Z 3 Acked 1002 Read 4 Done Done Done false
89392024-03-06T21:01:33.329Z 4 Acked 1003 Read 4 Done Done Done false
89402024-03-06T21:01:33.329Z 5 Acked 1004 Read 7 Done Done Done false
89412024-03-06T21:01:33.329Z 6 Acked 1005 Read 5 Done Done Done false
89422024-03-06T21:01:33.329Z 7 Acked 1006 Write 6 Done Done Done false
89432024-03-06T21:01:33.329Z 8 Acked 1007 Write 7 Done Done Done false
89442024-03-06T21:01:33.329Z 9 Acked 1008 Read 7 Done Done Done false
89452024-03-06T21:01:33.329Z 10 Acked 1009 Read 4 Done Done Sent false
89462024-03-06T21:01:33.329Z 11 Acked 1010 Read 10 Done Done Sent false
89472024-03-06T21:01:33.329Z 12 Acked 1011 Read 9 Done Done Sent false
89482024-03-06T21:01:33.329Z 13 Acked 1012 Read 9 Done Done Sent false
89492024-03-06T21:01:33.330Z 14 Acked 1013 Read 5 Done Done Sent false
89502024-03-06T21:01:33.330Z 15 Acked 1014 Flush 0 Done Done Sent false
89512024-03-06T21:01:33.330Z 16 Acked 1015 Write 8 Done Done Sent false
89522024-03-06T21:01:33.330Z 17 Acked 1016 Write 6 Done Done Sent false
89532024-03-06T21:01:33.330Z 18 Acked 1017 Read 6 Done Done Sent false
89542024-03-06T21:01:33.330Z 19 Acked 1018 Read 5 Done Done Sent false
89552024-03-06T21:01:33.330Z 20 Acked 1019 Read 8 Done Done Sent false
89562024-03-06T21:01:33.331Z 21 Acked 1020 Read 3 Done Done Sent false
89572024-03-06T21:01:33.331Z 22 Acked 1021 Read 2 Done Done Sent false
89582024-03-06T21:01:33.331Z 23 Acked 1022 Write 4 Done Done Sent false
89592024-03-06T21:01:33.331Z 24 Acked 1023 Read 4 Done Done Sent false
89602024-03-06T21:01:33.331Z 25 Acked 1024 Read 2 Done Done Sent false
89612024-03-06T21:01:33.331Z 26 Acked 1025 Read 8 Done Done Sent false
89622024-03-06T21:01:33.331Z 27 Acked 1026 Write 6 Done Done Sent false
89632024-03-06T21:01:33.331Z 28 Acked 1027 Write 9 Done Done Sent false
89642024-03-06T21:01:33.331Z 29 Acked 1028 Read 9 Done Done Sent false
89652024-03-06T21:01:33.332Z 30 Acked 1029 Read 1 Done Done Sent false
89662024-03-06T21:01:33.332Z STATES DS:0 DS:1 DS:2 TOTAL
89672024-03-06T21:01:33.332Z New 0 0 0 0
89682024-03-06T21:01:33.332Z Sent 0 0 21 21
89692024-03-06T21:01:33.332Z Done 30 30 9 69
89702024-03-06T21:01:33.332Z Skipped 0 0 0 0
89712024-03-06T21:01:33.332Z Error 0 0 0 0
89722024-03-06T21:01:33.332ZLast Flush: 1014 1014 0
89732024-03-06T21:01:33.333ZDownstairs last five completed:
89742024-03-06T21:01:33.333ZUpstairs last five completed: 30 29 28 27 26
89752024-03-06T21:01:33.333Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:01:32.080877726Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1467}
89762024-03-06T21:01:33.333ZWrote out file "/tmp/test_repair_verify.data"
89772024-03-06T21:01:33.343Z
89782024-03-06T21:01:33.343ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
89792024-03-06T21:01:33.365ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
89802024-03-06T21:01:33.365Z 0 000-019 82 82 81 142 142 141 F F T <---
89812024-03-06T21:01:33.365Z 1 020-039 78 78 78 134 134 134 T T F <---
89822024-03-06T21:01:33.365Z 2 040-059 74 74 74 126 126 126 F F F
89832024-03-06T21:01:33.366Z 3 060-079 76 76 76 131 131 131 F F F
89842024-03-06T21:01:33.366Z 4 080-099 61 61 61 102 102 102 F F F
89852024-03-06T21:01:33.366Z 5 100-119 78 78 78 137 137 137 F F F
89862024-03-06T21:01:33.366Z 6 120-139 81 81 81 141 141 141 F F F
89872024-03-06T21:01:33.366Z 7 140-159 76 76 76 131 131 131 F F F
89882024-03-06T21:01:33.366Z 8 160-179 78 78 78 135 135 135 F F F
89892024-03-06T21:01:33.367Z 9 180-199 78 78 78 136 136 136 T T F <---
89902024-03-06T21:01:33.367Z 10 200-219 74 74 74 125 125 125 T T F <---
89912024-03-06T21:01:33.367Z 11 220-239 82 82 78 142 142 137 F F T <---
89922024-03-06T21:01:33.367Z 12 240-259 74 74 74 125 125 125 F F F
89932024-03-06T21:01:33.367Z 13 260-279 78 78 78 136 136 136 F F F
89942024-03-06T21:01:33.384Z 14 280-299 80 80 80 140 140 140 F F F
89952024-03-06T21:01:33.384Z 15 300-319 72 72 72 122 122 122 F F F
89962024-03-06T21:01:33.384Z 16 320-339 79 79 79 138 138 138 F F F
89972024-03-06T21:01:33.384Z 17 340-359 77 77 77 132 132 132 F F F
89982024-03-06T21:01:33.385Z 18 360-379 78 78 78 135 135 135 F F F
89992024-03-06T21:01:33.385Z 19 380-399 74 74 74 127 127 127 F F F
90002024-03-06T21:01:33.385Z 20 400-419 76 76 76 131 131 131 F F F
90012024-03-06T21:01:33.385Z 21 420-439 82 82 76 142 142 130 F F T <---
90022024-03-06T21:01:33.386Z 22 440-459 82 82 74 142 142 126 F F T <---
90032024-03-06T21:01:33.386Z 23 460-479 78 78 78 137 137 137 F F F
90042024-03-06T21:01:33.386Z 24 480-499 76 76 76 130 130 130 T T F <---
90052024-03-06T21:01:33.386Z 25 500-519 80 80 80 139 139 139 F F F
90062024-03-06T21:01:33.386Z 26 520-539 70 70 70 120 120 120 F F F
90072024-03-06T21:01:33.386Z 27 540-559 74 74 74 126 126 126 F F F
90082024-03-06T21:01:33.387Z 28 560-579 79 79 79 138 138 138 T T F <---
90092024-03-06T21:01:33.387Z 29 580-599 81 81 81 141 141 141 T T F <---
90102024-03-06T21:01:33.387ZMax gen: 82, Max flush: 142
90112024-03-06T21:01:33.388ZError: Difference in extent metadata found!
90122024-03-06T21:01:33.388ZA Difference in extent metadata is expected here
90132024-03-06T21:01:33.388ZOn loop 41
90142024-03-06T21:01:33.388Z
90152024-03-06T21:01:33.388ZVerifying data now
90162024-03-06T21:01:38.926ZNo change after verify
90172024-03-06T21:01:38.926ZLoop: 41 Downstairs dump after verify (and repair):
90182024-03-06T21:01:38.946ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
90192024-03-06T21:01:38.946Z 0 000-019 82 82 82 142 142 142 F F F
90202024-03-06T21:01:38.946Z 1 020-039 83 83 83 143 143 143 F F F
90212024-03-06T21:01:38.947Z 2 040-059 74 74 74 126 126 126 F F F
90222024-03-06T21:01:38.947Z 3 060-079 76 76 76 131 131 131 F F F
90232024-03-06T21:01:38.947Z 4 080-099 61 61 61 102 102 102 F F F
90242024-03-06T21:01:38.947Z 5 100-119 78 78 78 137 137 137 F F F
90252024-03-06T21:01:38.947Z 6 120-139 81 81 81 141 141 141 F F F
90262024-03-06T21:01:38.948Z 7 140-159 76 76 76 131 131 131 F F F
90272024-03-06T21:01:38.948Z 8 160-179 78 78 78 135 135 135 F F F
90282024-03-06T21:01:38.948Z 9 180-199 83 83 83 143 143 143 F F F
90292024-03-06T21:01:38.948Z 10 200-219 83 83 83 143 143 143 F F F
90302024-03-06T21:01:38.948Z 11 220-239 82 82 82 142 142 142 F F F
90312024-03-06T21:01:38.948Z 12 240-259 74 74 74 125 125 125 F F F
90322024-03-06T21:01:38.949Z 13 260-279 78 78 78 136 136 136 F F F
90332024-03-06T21:01:38.949Z 14 280-299 80 80 80 140 140 140 F F F
90342024-03-06T21:01:38.949Z 15 300-319 72 72 72 122 122 122 F F F
90352024-03-06T21:01:38.949Z 16 320-339 79 79 79 138 138 138 F F F
90362024-03-06T21:01:38.955Z 17 340-359 77 77 77 132 132 132 F F F
90372024-03-06T21:01:38.955Z 18 360-379 78 78 78 135 135 135 F F F
90382024-03-06T21:01:38.955Z 19 380-399 74 74 74 127 127 127 F F F
90392024-03-06T21:01:38.955Z 20 400-419 76 76 76 131 131 131 F F F
90402024-03-06T21:01:38.955Z 21 420-439 82 82 82 142 142 142 F F F
90412024-03-06T21:01:38.955Z 22 440-459 82 82 82 142 142 142 F F F
90422024-03-06T21:01:38.955Z 23 460-479 78 78 78 137 137 137 F F F
90432024-03-06T21:01:38.956Z 24 480-499 83 83 83 143 143 143 F F F
90442024-03-06T21:01:38.956Z 25 500-519 80 80 80 139 139 139 F F F
90452024-03-06T21:01:38.956Z 26 520-539 70 70 70 120 120 120 F F F
90462024-03-06T21:01:38.956Z 27 540-559 74 74 74 126 126 126 F F F
90472024-03-06T21:01:38.956Z 28 560-579 83 83 83 143 143 143 F F F
90482024-03-06T21:01:38.956Z 29 580-599 83 83 83 143 143 143 F F F
90492024-03-06T21:01:38.956ZMax gen: 83, Max flush: 143
90502024-03-06T21:01:38.956Z
90512024-03-06T21:01:38.956ZBegin loop 42
90522024-03-06T21:01:38.956ZDownstairs to restart: 1
90532024-03-06T21:01:38.969Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.725349216Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475}
90542024-03-06T21:01:38.969Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.725618238Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475}
90552024-03-06T21:01:38.969Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.725639676Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475}
90562024-03-06T21:01:38.969Z{"msg":"Crucible 7e671ca6-bb5f-48b0-b494-c47e8156b94f has session id: ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4","v":0,"name":"crucible","level":30Crucible runtime is spawned
90572024-03-06T21:01:38.969Z,"time":"2024-03-06T21:01:37.725663504Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90582024-03-06T21:01:38.979Z{"msg":"Upstairs opts: Upstairs UUID: 7e671ca6-bb5f-48b0-b494-c47e8156b94f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.72572617Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90592024-03-06T21:01:38.979Z{"msg":"Crucible stats registered with UUID: 7e671ca6-bb5f-48b0-b494-c47e8156b94f","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.725770177Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90602024-03-06T21:01:38.979Z{"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.725814644Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475}
90612024-03-06T21:01:38.979Z{"msg":"7e671ca6-bb5f-48b0-b494-c47e8156b94f active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.72602281Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90622024-03-06T21:01:38.980Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.726075696Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"client":"0","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90632024-03-06T21:01:38.980Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.726126283Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"client":"0","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90642024-03-06T21:01:38.980Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.726173069Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"client":"1","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90652024-03-06T21:01:38.980Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.726214747Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"client":"1","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90662024-03-06T21:01:38.980Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.726350767Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"client":"2","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90672024-03-06T21:01:38.980Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.726405504Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"client":"2","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90682024-03-06T21:01:38.980Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.726432142Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"":"io task","client":"2","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90692024-03-06T21:01:38.981Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.726491807Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"":"io task","client":"0","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90702024-03-06T21:01:38.981Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.726523035Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"":"io task","client":"1","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90712024-03-06T21:01:38.981Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:01:37.726555533Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"":"io task","client":"1","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90722024-03-06T21:01:38.981Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:01:37.72659876Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"":"io task","client":"1","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90732024-03-06T21:01:38.982Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:01:37.726632588Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90742024-03-06T21:01:38.982Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.726661356Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"client":"1","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90752024-03-06T21:01:38.982Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.726688044Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"client":"1","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90762024-03-06T21:01:38.982Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.726731301Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"":"io task","client":"0","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90772024-03-06T21:01:38.982Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.726773938Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"":"io task","client":"2","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90782024-03-06T21:01:38.982Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.726811045Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"":"io task","client":"1","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90792024-03-06T21:01:38.983Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.727306621Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"client":"2","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90802024-03-06T21:01:38.983Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.727343969Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"client":"2","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90812024-03-06T21:01:38.983Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.727360948Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"client":"0","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90822024-03-06T21:01:38.983Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.727375586Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"client":"0","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90832024-03-06T21:01:38.983Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.727989804Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"client":"2","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90842024-03-06T21:01:38.983Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.728127645Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"client":"0","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90852024-03-06T21:01:38.983Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.728584573Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"client":"2","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90862024-03-06T21:01:38.983Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.728641899Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"client":"2","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90872024-03-06T21:01:38.983Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.728680867Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90882024-03-06T21:01:38.984Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.728703805Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"client":"0","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90892024-03-06T21:01:38.984Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.728726993Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"client":"0","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90902024-03-06T21:01:38.984Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:37.728751242Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90912024-03-06T21:01:39.970Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:38.727398918Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"":"io task","client":"1","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90922024-03-06T21:01:39.970Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:38.727499061Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"":"io task","client":"1","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90932024-03-06T21:01:39.978Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:38.728506291Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"client":"1","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90942024-03-06T21:01:39.978Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:38.728551798Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"client":"1","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90952024-03-06T21:01:39.978Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:38.729040615Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"client":"1","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90962024-03-06T21:01:39.978Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:38.729530371Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"client":"1","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90972024-03-06T21:01:39.979Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:38.729568338Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"client":"1","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90982024-03-06T21:01:39.979Z{"msg":"[0]R flush_numbers[0..12]: [142, 143, 126, 131, 102, 137, 141, 131, 135, 143, 143, 142]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:38.729584987Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"":"downstairs","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
90992024-03-06T21:01:39.979Z{"msg":"[0]R generation[0..12]: [82, 83, 74, 76, 61, 78, 81, 76, 78, 83, 83, 82]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:38.729607986Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"":"downstairs","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
91002024-03-06T21:01:39.979Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:38.729639903Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"":"downstairs","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
91012024-03-06T21:01:39.979Z{"msg":"[1]R flush_numbers[0..12]: [142, 143, 126, 131, 102, 137, 141, 131, 135, 143, 143, 142]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:38.729658662Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"":"downstairs","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
91022024-03-06T21:01:39.979Z{"msg":"[1]R generation[0..12]: [82, 83, 74, 76, 61, 78, 81, 76, 78, 83, 83, 82]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:38.72969158Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"":"downstairs","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
91032024-03-06T21:01:39.979Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
91042024-03-06T21:01:39.979Z,"time":"2024-03-06T21:01:38.729725358Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"":"downstairs","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
91052024-03-06T21:01:39.979Z{"msg":"[2]R flush_numbers[0..12]: [142, 143, 126, 131, 102, 137, 141, 131, 135, 143, 143, 142]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:38.729760385Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"":"downstairs","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
91062024-03-06T21:01:39.979Z{"msg":"[2]R generation[0..12]: [82, 83, 74, 76, 61, 78, 81, 76, 78, 83, 83, 82]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:38.729791283Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"":"downstairs","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
91072024-03-06T21:01:39.979Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:38.72982892Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"":"downstairs","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
91082024-03-06T21:01:39.979Z{"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:38.729850909Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"":"downstairs","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
91092024-03-06T21:01:39.980Z{"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:38.729882877Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"":"downstairs","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
91102024-03-06T21:01:39.980Z{"msg":"Next flush: 144","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:38.729898655Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"":"downstairs","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
91112024-03-06T21:01:39.980Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:38.729920554Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"":"downstairs","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
91122024-03-06T21:01:39.980Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:38.729951172Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
91132024-03-06T21:01:39.980Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:38.729965851Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"client":"0","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
91142024-03-06T21:01:39.980Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:38.729996699Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"client":"1","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
91152024-03-06T21:01:39.980Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:38.730015058Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
91162024-03-06T21:01:39.980Zw-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"client":"2","":"downstairs","session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
91172024-03-06T21:01:39.980Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:38.730048095Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
91182024-03-06T21:01:39.980Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:38.730063694Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
91192024-03-06T21:01:39.980Z{"msg":"7e671ca6-bb5f-48b0-b494-c47e8156b94f is now active with session: ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:38.730092092Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
91202024-03-06T21:01:39.980Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:38.730109111Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475,"session_id":"ab0e5678-1c5d-4dd2-a4f0-1cc27e2241b4"}
91212024-03-06T21:01:39.980Z{"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:38.730136639Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475}
91222024-03-06T21:01:39.980ZLoading write count information from file "/tmp/test_repair_verify.data"
91232024-03-06T21:01:39.981ZRun Repair workload
91242024-03-06T21:01:39.981Z01/30 Write block 164 len 20480 data: 4 3 2 3 3
91252024-03-06T21:01:39.987Z02/30 Read block 274 len 28672
91262024-03-06T21:01:39.987Z03/30 Read block 311 len 8192
91272024-03-06T21:01:39.987Z04/30 Write block 536 len 40960 data: 7 9 7 5 8 6 9 9 9 9
91282024-03-06T21:01:39.987Z05/30 Write block 61 len 4096 data: 15
91292024-03-06T21:01:39.987Z06/30 Read block 527 len 8192
91302024-03-06T21:01:39.987Z07/30 Write block 208 len 12288 data: 5 5 6
91312024-03-06T21:01:39.987Z08/30 Write block 318 len 12288 data: 7 7 5
91322024-03-06T21:01:39.987Z09/30 Write block 543 len 32768 data: 10 10 10 8 8 7 7 6
91332024-03-06T21:01:39.987Z10/30 Read block 274 len 28672
91342024-03-06T21:01:39.987Z11/30 Write block 150 len 36864 data: 9 8 6 7 7 8 9 9 9
91352024-03-06T21:01:39.991Z12/30 Read block 192 len 36864
91362024-03-06T21:01:39.991Z13/30 Write block 453 len 8192 data: 7 6
91372024-03-06T21:01:39.991Z14/30 Write block 338 len 20480 data: 9 7 8 8 7
91382024-03-06T21:01:39.991Z15/30 Read block 3 len 28672
91392024-03-06T21:01:39.991Z16/30 Read block 484 len 40960
91402024-03-06T21:01:39.991Z17/30 Read block 495 len 8192
91412024-03-06T21:01:39.991Z18/30 Read block 216 len 20480
91422024-03-06T21:01:39.994Z19/30 Write block 133 len 32768 data: 5 5 5 6 5 4 4 4
91432024-03-06T21:01:39.994Z20/30 Read block 337 len 36864
91442024-03-06T21:01:39.998Z21/30 Read block 510 len 28672
91452024-03-06T21:01:39.998Z22/30 Write block 17 len 20480 data: 5 7 7 8 7
91462024-03-06T21:01:39.998Z23/30 Write block 118 len 40960 data: 4 4 4 4 5 4 6 9 8 10
91472024-03-06T21:01:39.998Z24/30 Read block 83 len 12288
91482024-03-06T21:01:40.003Z25/30 Read block 59 len 28672
91492024-03-06T21:01:40.003Z26/30 Write block 38 len 28672 data: 5 6 5 7 8 8 7
91502024-03-06T21:01:40.003Z27/30 Read block 572 len 28672
91512024-03-06T21:01:40.003Z28/30 Write block 484 len 32768 data: 6 8 6 7 8 9 7 8
91522024-03-06T21:01:40.003Z29/30 Read block 394 len 36864
91532024-03-06T21:01:40.011Z30/30 Write block 182 len 40960 data: 10 8 8 7 7 6 6 7 7 6
91542024-03-06T21:01:40.011Z----------------------------------------------------------------
91552024-03-06T21:01:40.011Z Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:30
91562024-03-06T21:01:40.011ZStates: Active Active Active
91572024-03-06T21:01:40.011ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
91582024-03-06T21:01:40.011Z 1 Acked 1000 Write 5 Done Done Done false
91592024-03-06T21:01:40.011Z 2 Acked 1001 Read 7 Done Done Done false
91602024-03-06T21:01:40.011Z 3 Acked 1002 Read 2 Done Sent Done false
91612024-03-06T21:01:40.011Z 4 Acked 1003 Write 10 Done Sent Done false
91622024-03-06T21:01:40.011Z 5 Acked 1004 Write 1 Done Sent Done false
91632024-03-06T21:01:40.011Z 6 Acked 1005 Read 2 Done Sent Done false
91642024-03-06T21:01:40.011Z 7 Acked 1006 Write 3 Done Sent Done false
91652024-03-06T21:01:40.012Z 8 Acked 1007 Write 3 Done Sent Done false
91662024-03-06T21:01:40.012Z 9 Acked 1008 Write 8 Done Sent Done false
91672024-03-06T21:01:40.012Z 10 Acked 1009 Read 7 Done Sent Done false
91682024-03-06T21:01:40.012Z 11 Acked 1010 Write 9 Done Sent Done false
91692024-03-06T21:01:40.012Z 12 Acked 1011 Read 9 Done Sent Done false
91702024-03-06T21:01:40.012Z 13 Acked 1012 Write 2 Done Sent Done false
91712024-03-06T21:01:40.012Z 14 Acked 1013 Write 5 Done Sent Done false
91722024-03-06T21:01:40.012Z 15 Acked 1014 Read 7 Done Sent Done false
91732024-03-06T21:01:40.012Z 16 Acked 1015 Read 10 Done Sent Done false
91742024-03-06T21:01:40.012Z 17 Acked 1016 Read 2 Done Sent Done false
91752024-03-06T21:01:40.012Z 18 Acked 1017 Read 5 Done Sent Done false
91762024-03-06T21:01:40.012Z 19 Acked 1018 Write 8 Done Sent Done false
91772024-03-06T21:01:40.013Z 20 Acked 1019 Read 9 Done Sent Done false
91782024-03-06T21:01:40.013Z 21 Acked 1020 Read 7 Done Sent Done false
91792024-03-06T21:01:40.013Z 22 Acked 1021 Write 5 Done Sent Done false
91802024-03-06T21:01:40.013Z 23 Acked 1022 Write 10 Done Sent Done false
91812024-03-06T21:01:40.013Z 24 Acked 1023 Read 3 Done Sent Done false
91822024-03-06T21:01:40.013Z 25 Acked 1024 Read 7 Done Sent Done false
91832024-03-06T21:01:40.013Z 26 Acked 1025 Write 7 Done Sent Done false
91842024-03-06T21:01:40.013Z 27 Acked 1026 Read 7 Done Sent Done false
91852024-03-06T21:01:40.013Z 28 Acked 1027 Write 8 Done Sent Done false
91862024-03-06T21:01:40.013Z 29 Acked 1028 Read 9 Done Sent Done false
91872024-03-06T21:01:40.013Z 30 Acked 1029 Write 10 Sent Sent Sent false
91882024-03-06T21:01:40.013Z STATES DS:0 DS:1 DS:2 TOTAL
91892024-03-06T21:01:40.013Z New 0 0 0 0
91902024-03-06T21:01:40.013Z Sent 1 28 1 30
91912024-03-06T21:01:40.014Z Done 29 2 29 60
91922024-03-06T21:01:40.014Z Skipped 0 0 0 0
91932024-03-06T21:01:40.014Z Error 0 0 0 0
91942024-03-06T21:01:40.014ZLast Flush: 0 0 0
91952024-03-06T21:01:40.014ZDownstairs last five completed:
91962024-03-06T21:01:40.014ZUpstairs last five completed: 30 29 28 27 26
91972024-03-06T21:01:40.014Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:01:38.764443325Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1475}
91982024-03-06T21:01:40.014ZWrote out file "/tmp/test_repair_verify.data"
91992024-03-06T21:01:40.030Z
92002024-03-06T21:01:40.030ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
92012024-03-06T21:01:40.039ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
92022024-03-06T21:01:40.039Z 0 000-019 82 82 82 142 142 142 T F T <---
92032024-03-06T21:01:40.039Z 1 020-039 83 83 83 143 143 143 T F T <---
92042024-03-06T21:01:40.039Z 2 040-059 74 74 74 126 126 126 T F T <---
92052024-03-06T21:01:40.039Z 3 060-079 76 76 76 131 131 131 T F T <---
92062024-03-06T21:01:40.039Z 4 080-099 61 61 61 102 102 102 F F F
92072024-03-06T21:01:40.039Z 5 100-119 78 78 78 137 137 137 T F T <---
92082024-03-06T21:01:40.039Z 6 120-139 81 81 81 141 141 141 T F T <---
92092024-03-06T21:01:40.040Z 7 140-159 76 76 76 131 131 131 T F T <---
92102024-03-06T21:01:40.040Z 8 160-179 78 78 78 135 135 135 T T T
92112024-03-06T21:01:40.040Z 9 180-199 83 83 83 143 143 143 T F F <---
92122024-03-06T21:01:40.040Z 10 200-219 83 83 83 143 143 143 T F T <---
92132024-03-06T21:01:40.040Z 11 220-239 82 82 82 142 142 142 F F F
92142024-03-06T21:01:40.040Z 12 240-259 74 74 74 125 125 125 F F F
92152024-03-06T21:01:40.044Z 13 260-279 78 78 78 136 136 136 F F F
92162024-03-06T21:01:40.044Z 14 280-299 80 80 80 140 140 140 F F F
92172024-03-06T21:01:40.044Z 15 300-319 72 72 72 122 122 122 T F T <---
92182024-03-06T21:01:40.045Z 16 320-339 79 79 79 138 138 138 T F T <---
92192024-03-06T21:01:40.045Z 17 340-359 77 77 77 132 132 132 T F T <---
92202024-03-06T21:01:40.045Z 18 360-379 78 78 78 135 135 135 F F F
92212024-03-06T21:01:40.045Z 19 380-399 74 74 74 127 127 127 F F F
92222024-03-06T21:01:40.045Z 20 400-419 76 76 76 131 131 131 F F F
92232024-03-06T21:01:40.045Z 21 420-439 82 82 82 142 142 142 F F F
92242024-03-06T21:01:40.045Z 22 440-459 82 82 82 142 142 142 T F T <---
92252024-03-06T21:01:40.045Z 23 460-479 78 78 78 137 137 137 F F F
92262024-03-06T21:01:40.045Z 24 480-499 83 83 83 143 143 143 T F T <---
92272024-03-06T21:01:40.045Z 25 500-519 80 80 80 139 139 139 F F F
92282024-03-06T21:01:40.045Z 26 520-539 70 70 70 120 120 120 T F T <---
92292024-03-06T21:01:40.045Z 27 540-559 74 74 74 126 126 126 T F T <---
92302024-03-06T21:01:40.045Z 28 560-579 83 83 83 143 143 143 F F F
92312024-03-06T21:01:40.046Z 29 580-599 83 83 83 143 143 143 F F F
92322024-03-06T21:01:40.046ZMax gen: 83, Max flush: 143
92332024-03-06T21:01:40.046ZError: Difference in extent metadata found!
92342024-03-06T21:01:40.046ZA Difference in extent metadata is expected here
92352024-03-06T21:01:40.046ZOn loop 42
92362024-03-06T21:01:40.046Z
92372024-03-06T21:01:40.050ZVerifying data now
92382024-03-06T21:01:44.941ZNo change after verify
92392024-03-06T21:01:44.941ZLoop: 42 Downstairs dump after verify (and repair):
92402024-03-06T21:01:44.963ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
92412024-03-06T21:01:44.963Z 0 000-019 84 84 84 144 144 144 F F F
92422024-03-06T21:01:44.964Z 1 020-039 84 84 84 144 144 144 F F F
92432024-03-06T21:01:44.965Z 2 040-059 84 84 84 144 144 144 F F F
92442024-03-06T21:01:44.965Z 3 060-079 84 84 84 144 144 144 F F F
92452024-03-06T21:01:44.965Z 4 080-099 61 61 61 102 102 102 F F F
92462024-03-06T21:01:44.965Z 5 100-119 84 84 84 144 144 144 F F F
92472024-03-06T21:01:44.966Z 6 120-139 84 84 84 144 144 144 F F F
92482024-03-06T21:01:44.967Z 7 140-159 84 84 84 144 144 144 F F F
92492024-03-06T21:01:44.967Z 8 160-179 84 84 84 144 144 144 F F F
92502024-03-06T21:01:44.972Z 9 180-199 84 84 84 144 144 144 F F F
92512024-03-06T21:01:44.972Z 10 200-219 84 84 84 144 144 144 F F F
92522024-03-06T21:01:44.972Z 11 220-239 82 82 82 142 142 142 F F F
92532024-03-06T21:01:44.972Z 12 240-259 74 74 74 125 125 125 F F F
92542024-03-06T21:01:44.972Z 13 260-279 78 78 78 136 136 136 F F F
92552024-03-06T21:01:44.972Z 14 280-299 80 80 80 140 140 140 F F F
92562024-03-06T21:01:44.972Z 15 300-319 84 84 84 144 144 144 F F F
92572024-03-06T21:01:44.972Z 16 320-339 84 84 84 144 144 144 F F F
92582024-03-06T21:01:44.973Z 17 340-359 84 84 84 144 144 144 F F F
92592024-03-06T21:01:44.973Z 18 360-379 78 78 78 135 135 135 F F F
92602024-03-06T21:01:44.973Z 19 380-399 74 74 74 127 127 127 F F F
92612024-03-06T21:01:44.981Z 20 400-419 76 76 76 131 131 131 F F F
92622024-03-06T21:01:44.981Z 21 420-439 82 82 82 142 142 142 F F F
92632024-03-06T21:01:44.981Z 22 440-459 84 84 84 144 144 144 F F F
92642024-03-06T21:01:44.981Z 23 460-479 78 78 78 137 137 137 F F F
92652024-03-06T21:01:44.981Z 24 480-499 84 84 84 144 144 144 F F F
92662024-03-06T21:01:44.981Z 25 500-519 80 80 80 139 139 139 F F F
92672024-03-06T21:01:44.981Z 26 520-539 84 84 84 144 144 144 F F F
92682024-03-06T21:01:44.981Z 27 540-559 84 84 84 144 144 144 F F F
92692024-03-06T21:01:44.981Z 28 560-579 83 83 83 143 143 143 F F F
92702024-03-06T21:01:44.981Z 29 580-599 83 83 83 143 143 143 F F F
92712024-03-06T21:01:44.981ZMax gen: 84, Max flush: 144
92722024-03-06T21:01:44.981Z
92732024-03-06T21:01:44.981ZBegin loop 43
92742024-03-06T21:01:44.982ZDownstairs to restart: 1
92752024-03-06T21:01:44.988Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.738484499Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483}
92762024-03-06T21:01:44.988Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.738765699Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483}
92772024-03-06T21:01:44.988Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.738799607Z","hostname":"Crucible runtime is spawned
92782024-03-06T21:01:44.988Zw-01HRARQTS0TVE14DKZ437MF5TB","pid":1483}
92792024-03-06T21:01:44.988Z{"msg":"Crucible 2f9f98ee-19a7-494d-8d65-2d868450f982 has session id: fc60fd6f-6368-45e4-aa55-61840a2692e9","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.738873882Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
92802024-03-06T21:01:44.988Z{"msg":"Upstairs opts: Upstairs UUID: 2f9f98ee-19a7-494d-8d65-2d868450f982, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.738931908Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
92812024-03-06T21:01:44.988Z{"msg":"Crucible stats registered with UUID: 2f9f98ee-19a7-494d-8d65-2d868450f982","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.738974875Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
92822024-03-06T21:01:44.988Z{"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.739004443Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483}
92832024-03-06T21:01:44.988Z{"msg":"2f9f98ee-19a7-494d-8d65-2d868450f982 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.739066709Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
92842024-03-06T21:01:44.988Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.739093897Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"client":"0","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
92852024-03-06T21:01:44.988Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.739111086Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"client":"0","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
92862024-03-06T21:01:44.988Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.739133874Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"client":"1","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
92872024-03-06T21:01:44.989Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.739152183Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"client":"1","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
92882024-03-06T21:01:44.989Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.739179531Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"client":"2","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
92892024-03-06T21:01:44.989Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.739207199Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"client":"2","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
92902024-03-06T21:01:44.989Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.739228318Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"":"io task","client":"0","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
92912024-03-06T21:01:44.989Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.739268155Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"":"io task","client":"1","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
92922024-03-06T21:01:44.989Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.739302682Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"":"io task","client":"2","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
92932024-03-06T21:01:44.989Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.739422704Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"":"io task","client":"0","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
92942024-03-06T21:01:44.989Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.739503159Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"":"io task","client":"2","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
92952024-03-06T21:01:44.989Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.739572994Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"":"io task","client":"1","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
92962024-03-06T21:01:44.989Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.740177992Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"client":"2","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
92972024-03-06T21:01:44.989Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.740217359Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"client":"2","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
92982024-03-06T21:01:44.989Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.740234848Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"client":"0","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
92992024-03-06T21:01:44.989Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.740305583Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"client":"0","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93002024-03-06T21:01:44.989Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.740896572Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"client":"0","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93012024-03-06T21:01:44.989Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.741105048Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"client":"2","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93022024-03-06T21:01:44.989Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.74151767Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"client":"0","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93032024-03-06T21:01:44.989Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.741569336Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"client":"0","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93042024-03-06T21:01:44.989Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.741588685Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93052024-03-06T21:01:44.989Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.741681089Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"client":"2","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93062024-03-06T21:01:44.989Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.741712966Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"client":"2","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93072024-03-06T21:01:44.989Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.741729145Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93082024-03-06T21:01:44.989Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.741766363Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"client":"1","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93092024-03-06T21:01:44.989Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.74180012Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"client":"1","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93102024-03-06T21:01:44.990Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.742334724Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"client":"1","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93112024-03-06T21:01:44.990Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.742731846Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"client":"1","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93122024-03-06T21:01:44.990Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.742764344Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"client":"1","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93132024-03-06T21:01:44.990Z{"msg":"[0]R flush_numbers[0..12]: [144, 144, 144, 144, 102, 144, 144, 144, 144, 144, 144, 142]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.742781283Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"":"downstairs","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93142024-03-06T21:01:44.990Z{"msg":"[0]R generation[0..12]: [84, 84, 84, 84, 61, 84, 84, 84, 84, 84, 84, 82]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.74281611Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"":"downstairs","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93152024-03-06T21:01:44.990Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.742846648Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"":"downstairs","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93162024-03-06T21:01:44.990Z{"msg":"[1]R flush_numbers[0..12]: [144, 144, 144, 144, 102, 144, 144, 144, 144, 144, 144, 142]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.742874666Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"":"downstairs","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93172024-03-06T21:01:44.990Z{"msg":"[1]R generation[0..12]: [84, 84, 84, 84, 61, 84, 84, 84, 84, 84, 84, 82]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.742894585Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"":"downstairs","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93182024-03-06T21:01:44.990Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.742913194Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483Wait for a query_work_queue command to finish before sending IO
93192024-03-06T21:01:44.990Z,"":"downstairs","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93202024-03-06T21:01:44.990Z{"msg":"[2]R flush_numbers[0..12]: [144, 144, 144, 144, 102, 144, 144, 144, 144, 144, 144, 142]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.742941622Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"":"downstairs","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93212024-03-06T21:01:44.990Z{"msg":"[2]R generation[0..12]: [84, 84, 84, 84, 61, 84, 84, 84, 84, 84, 84, 82]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.74296588Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"":"downstairs","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93222024-03-06T21:01:44.990Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.742989988Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"":"downstairs","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93232024-03-06T21:01:44.990Z{"msg":"Max found gen is 85","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.743020276Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"":"downstairs","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93242024-03-06T21:01:44.990Z{"msg":"Generation requested: 86 >= found:85","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.743035625Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"":"downstairs","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93252024-03-06T21:01:44.990Z{"msg":"Next flush: 145","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.743058954Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"":"downstairs","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93262024-03-06T21:01:44.990Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.743093811Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"":"downstairs","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93272024-03-06T21:01:44.990Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.743130949Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93282024-03-06T21:01:44.990Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.743173346Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"client":"0","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93292024-03-06T21:01:44.990Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.743202144Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"client":"1","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93302024-03-06T21:01:44.990Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.743236381Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"client":"2","":"downstairs","session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93312024-03-06T21:01:44.990Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.743271939Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93322024-03-06T21:01:44.990Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.743308966Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93332024-03-06T21:01:44.991Z{"msg":"2f9f98ee-19a7-494d-8d65-2d868450f982 is now active with session: fc60fd6f-6368-45e4-aa55-61840a2692e9","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.743346614Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93342024-03-06T21:01:44.991Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
93352024-03-06T21:01:44.991Z2024-03-06T21:01:43.743379101Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483,"session_id":"fc60fd6f-6368-45e4-aa55-61840a2692e9"}
93362024-03-06T21:01:44.991Z{"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:43.743411379Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483}
93372024-03-06T21:01:45.004ZLoading write count information from file "/tmp/test_repair_verify.data"
93382024-03-06T21:01:45.004ZRun Repair workload
93392024-03-06T21:01:45.004Z01/30 Write block 451 len 36864 data: 10 10 8 7 7 5 8 7 7
93402024-03-06T21:01:45.004Z02/30 Read block 310 len 40960
93412024-03-06T21:01:45.004Z03/30 Flush
93422024-03-06T21:01:45.004Z04/30 Flush
93432024-03-06T21:01:45.004Z05/30 Flush
93442024-03-06T21:01:45.004Z06/30 Flush
93452024-03-06T21:01:45.004Z07/30 Flush
93462024-03-06T21:01:45.004Z08/30 Write block 40 len 12288 data: 6 8 9
93472024-03-06T21:01:45.008Z09/30 Write block 433 len 40960 data: 9 10 8 7 9 9 9 8 6 5
93482024-03-06T21:01:45.008Z10/30 Flush
93492024-03-06T21:01:45.009Z11/30 Write block 125 len 8192 data: 10 9
93502024-03-06T21:01:45.009Z12/30 Write block 17 len 40960 data: 6 8 8 9 8 10 11 9 8 9
93512024-03-06T21:01:45.009Z13/30 Read block 397 len 36864
93522024-03-06T21:01:45.009Z14/30 Write block 200 len 28672 data: 7 6 5 4 4 4 5
93532024-03-06T21:01:45.009Z15/30 Read block 543 len 16384
93542024-03-06T21:01:45.009Z16/30 Read block 322 len 40960
93552024-03-06T21:01:45.009Z17/30 Write block 196 len 24576 data: 9 9 11 8 8 7
93562024-03-06T21:01:45.009Z18/30 Read block 331 len 32768
93572024-03-06T21:01:45.013Z19/30 Write block 581 len 8192 data: 4 4
93582024-03-06T21:01:45.013Z20/30 Read block 472 len 40960
93592024-03-06T21:01:45.013Z21/30 Write block 385 len 4096 data: 9
93602024-03-06T21:01:45.013Z22/30 Read block 130 len 12288
93612024-03-06T21:01:45.013Z23/30 Write block 173 len 4096 data: 8
93622024-03-06T21:01:45.013Z24/30 Write block 447 len 20480 data: 3 7 7 8 11
93632024-03-06T21:01:45.017Z25/30 Write block 298 len 20480 data: 5 5 5 4 5
93642024-03-06T21:01:45.017Z26/30 Read block 490 len 20480
93652024-03-06T21:01:45.017Z27/30 Read block 174 len 24576
93662024-03-06T21:01:45.017Z28/30 Read block 447 len 4096
93672024-03-06T21:01:45.022Z29/30 Write block 439 len 8192 data: 10 9
93682024-03-06T21:01:45.022Z30/30 Read block 111 len 24576
93692024-03-06T21:01:45.022Z----------------------------------------------------------------
93702024-03-06T21:01:45.022Z Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:30
93712024-03-06T21:01:45.022ZStates: Active Active Active
93722024-03-06T21:01:45.022ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
93732024-03-06T21:01:45.023Z 1 Acked 1000 Write 9 Done Sent Done false
93742024-03-06T21:01:45.023Z 2 Acked 1001 Read 10 Done Sent Done false
93752024-03-06T21:01:45.023Z 3 Acked 1002 Flush 0 Done Sent Done false
93762024-03-06T21:01:45.023Z 4 Acked 1003 Flush 0 Done Sent Done false
93772024-03-06T21:01:45.023Z 5 Acked 1004 Flush 0 Done Sent Done false
93782024-03-06T21:01:45.023Z 6 Acked 1005 Flush 0 Done Sent Done false
93792024-03-06T21:01:45.023Z 7 Acked 1006 Flush 0 Done Sent Done false
93802024-03-06T21:01:45.023Z 8 Acked 1007 Write 3 Done Sent Done false
93812024-03-06T21:01:45.023Z 9 Acked 1008 Write 10 Done Sent Done false
93822024-03-06T21:01:45.023Z 10 Acked 1009 Flush 0 Done Sent Done false
93832024-03-06T21:01:45.023Z 11 Acked 1010 Write 2 Done Sent Done false
93842024-03-06T21:01:45.023Z 12 Acked 1011 Write 10 Done Sent Done false
93852024-03-06T21:01:45.024Z 13 Acked 1012 Read 9 Done Sent Done false
93862024-03-06T21:01:45.024Z 14 Acked 1013 Write 7 Done Sent Done false
93872024-03-06T21:01:45.024Z 15 Acked 1014 Read 4 Done Sent Done false
93882024-03-06T21:01:45.024Z 16 Acked 1015 Read 10 Done Sent Done false
93892024-03-06T21:01:45.024Z 17 Acked 1016 Write 6 Done Sent Done false
93902024-03-06T21:01:45.024Z 18 Acked 1017 Read 8 Done Sent Done false
93912024-03-06T21:01:45.024Z 19 Acked 1018 Write 2 Done Sent Done false
93922024-03-06T21:01:45.024Z 20 Acked 1019 Read 10 Done Sent Done false
93932024-03-06T21:01:45.024Z 21 Acked 1020 Write 1 Done Sent Done false
93942024-03-06T21:01:45.024Z 22 Acked 1021 Read 3 Done Sent Done false
93952024-03-06T21:01:45.024Z 23 Acked 1022 Write 1 Done Sent Done false
93962024-03-06T21:01:45.024Z 24 Acked 1023 Write 5 Done Sent Done false
93972024-03-06T21:01:45.024Z 25 Acked 1024 Write 5 Done Sent Done false
93982024-03-06T21:01:45.025Z 26 Acked 1025 Read 5 Done Sent Done false
93992024-03-06T21:01:45.025Z 27 Acked 1026 Read 6 Done Sent Done false
94002024-03-06T21:01:45.025Z 28 Acked 1027 Read 1 Done Sent Done false
94012024-03-06T21:01:45.025Z 29 Acked 1028 Write 2 Done Sent Sent false
94022024-03-06T21:01:45.025Z 30 Acked 1029 Read 6 Done Sent Sent false
94032024-03-06T21:01:45.025Z STATES DS:0 DS:1 DS:2 TOTAL
94042024-03-06T21:01:45.025Z New 0 0 0 0
94052024-03-06T21:01:45.025Z Sent 0 30 2 32
94062024-03-06T21:01:45.025Z Done 30 0 28 58
94072024-03-06T21:01:45.025Z Skipped 0 0 0 0
94082024-03-06T21:01:45.025Z Error 0 0 0 0
94092024-03-06T21:01:45.025ZLast Flush: 1009 0 1009
94102024-03-06T21:01:45.025ZDownstairs last five completed:
94112024-03-06T21:01:45.026ZUpstairs last five completed: 30 29 28 27 26
94122024-03-06T21:01:45.026Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:01:43.777078919Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1483}
94132024-03-06T21:01:45.026ZWrote out file "/tmp/test_repair_verify.data"
94142024-03-06T21:01:45.030Z
94152024-03-06T21:01:45.030ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
94162024-03-06T21:01:45.049ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
94172024-03-06T21:01:45.049Z 0 000-019 84 84 84 144 144 144 T F T <---
94182024-03-06T21:01:45.049Z 1 020-039 84 84 84 144 144 144 T F T <---
94192024-03-06T21:01:45.049Z 2 040-059 86 84 86 150 144 150 F F F <---
94202024-03-06T21:01:45.049Z 3 060-079 84 84 84 144 144 144 F F F
94212024-03-06T21:01:45.049Z 4 080-099 61 61 61 102 102 102 F F F
94222024-03-06T21:01:45.049Z 5 100-119 84 84 84 144 144 144 F F F
94232024-03-06T21:01:45.050Z 6 120-139 84 84 84 144 144 144 T F T <---
94242024-03-06T21:01:45.050Z 7 140-159 84 84 84 144 144 144 F F F
94252024-03-06T21:01:45.050Z 8 160-179 84 84 84 144 144 144 T F T <---
94262024-03-06T21:01:45.050Z 9 180-199 84 84 84 144 144 144 T F T <---
94272024-03-06T21:01:45.050Z 10 200-219 84 84 84 144 144 144 T F T <---
94282024-03-06T21:01:45.050Z 11 220-239 82 82 82 142 142 142 F F F
94292024-03-06T21:01:45.050Z 12 240-259 74 74 74 125 125 125 F F F
94302024-03-06T21:01:45.050Z 13 260-279 78 78 78 136 136 136 F F F
94312024-03-06T21:01:45.050Z 14 280-299 80 80 80 140 140 140 T F T <---
94322024-03-06T21:01:45.050Z 15 300-319 84 84 84 144 144 144 T F T <---
94332024-03-06T21:01:45.050Z 16 320-339 84 84 84 144 144 144 F F F
94342024-03-06T21:01:45.050Z 17 340-359 84 84 84 144 144 144 F F F
94352024-03-06T21:01:45.054Z 18 360-379 78 78 78 135 135 135 F F F
94362024-03-06T21:01:45.054Z 19 380-399 74 74 74 127 127 127 T F T <---
94372024-03-06T21:01:45.054Z 20 400-419 76 76 76 131 131 131 F F F
94382024-03-06T21:01:45.054Z 21 420-439 86 82 86 150 142 150 T F T <---
94392024-03-06T21:01:45.054Z 22 440-459 86 84 86 150 144 150 T F T <---
94402024-03-06T21:01:45.054Z 23 460-479 78 78 78 137 137 137 F F F
94412024-03-06T21:01:45.054Z 24 480-499 84 84 84 144 144 144 F F F
94422024-03-06T21:01:45.054Z 25 500-519 80 80 80 139 139 139 F F F
94432024-03-06T21:01:45.054Z 26 520-539 84 84 84 144 144 144 F F F
94442024-03-06T21:01:45.054Z 27 540-559 84 84 84 144 144 144 F F F
94452024-03-06T21:01:45.054Z 28 560-579 83 83 83 143 143 143 F F F
94462024-03-06T21:01:45.054Z 29 580-599 83 83 83 143 143 143 T F T <---
94472024-03-06T21:01:45.054ZMax gen: 86, Max flush: 150
94482024-03-06T21:01:45.054ZError: Difference in extent metadata found!
94492024-03-06T21:01:45.054ZA Difference in extent metadata is expected here
94502024-03-06T21:01:45.054ZOn loop 43
94512024-03-06T21:01:45.054Z
94522024-03-06T21:01:45.058ZVerifying data now
94532024-03-06T21:01:49.760ZNo change after verify
94542024-03-06T21:01:49.761ZLoop: 43 Downstairs dump after verify (and repair):
94552024-03-06T21:01:50.200ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
94562024-03-06T21:01:50.200Z 0 000-019 87 87 87 151 151 151 F F F
94572024-03-06T21:01:50.200Z 1 020-039 87 87 87 151 151 151 F F F
94582024-03-06T21:01:50.200Z 2 040-059 86 86 86 150 150 150 F F F
94592024-03-06T21:01:50.200Z 3 060-079 84 84 84 144 144 144 F F F
94602024-03-06T21:01:50.200Z 4 080-099 61 61 61 102 102 102 F F F
94612024-03-06T21:01:50.200Z 5 100-119 84 84 84 144 144 144 F F F
94622024-03-06T21:01:50.200Z 6 120-139 87 87 87 151 151 151 F F F
94632024-03-06T21:01:50.200Z 7 140-159 84 84 84 144 144 144 F F F
94642024-03-06T21:01:50.200Z 8 160-179 87 87 87 151 151 151 F F F
94652024-03-06T21:01:50.200Z 9 180-199 87 87 87 151 151 151 F F F
94662024-03-06T21:01:50.200Z 10 200-219 87 87 87 151 151 151 F F F
94672024-03-06T21:01:50.200Z 11 220-239 82 82 82 142 142 142 F F F
94682024-03-06T21:01:50.201Z 12 240-259 74 74 74 125 125 125 F F F
94692024-03-06T21:01:50.201Z 13 260-279 78 78 78 136 136 136 F F F
94702024-03-06T21:01:50.201Z 14 280-299 87 87 87 151 151 151 F F F
94712024-03-06T21:01:50.201Z 15 300-319 87 87 87 151 151 151 F F F
94722024-03-06T21:01:50.201Z 16 320-339 84 84 84 144 144 144 F F F
94732024-03-06T21:01:50.237Z 17 340-359 84 84 84 144 144 144 F F F
94742024-03-06T21:01:50.237Z 18 360-379 78 78 78 135 135 135 F F F
94752024-03-06T21:01:50.237Z 19 380-399 87 87 87 151 151 151 F F F
94762024-03-06T21:01:50.237Z 20 400-419 76 76 76 131 131 131 F F F
94772024-03-06T21:01:50.237Z 21 420-439 87 87 87 151 151 151 F F F
94782024-03-06T21:01:50.237Z 22 440-459 87 87 87 151 151 151 F F F
94792024-03-06T21:01:50.237Z 23 460-479 78 78 78 137 137 137 F F F
94802024-03-06T21:01:50.237Z 24 480-499 84 84 84 144 144 144 F F F
94812024-03-06T21:01:50.237Z 25 500-519 80 80 80 139 139 139 F F F
94822024-03-06T21:01:50.237Z 26 520-539 84 84 84 144 144 144 F F F
94832024-03-06T21:01:50.237Z 27 540-559 84 84 84 144 144 144 F F F
94842024-03-06T21:01:50.237Z 28 560-579 83 83 83 143 143 143 F F F
94852024-03-06T21:01:50.237Z 29 580-599 87 87 87 151 151 151 F F F
94862024-03-06T21:01:50.237ZMax gen: 87, Max flush: 151
94872024-03-06T21:01:50.237Z
94882024-03-06T21:01:50.237ZBegin loop 44
94892024-03-06T21:01:50.237ZDownstairs to restart: 0
94902024-03-06T21:01:50.237Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.560849907Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491}
94912024-03-06T21:01:50.237Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
94922024-03-06T21:01:50.238Z2024-03-06T21:01:48.561151165Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491}
94932024-03-06T21:01:50.238Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.561190263Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491}
94942024-03-06T21:01:50.238Z{"msg":"Crucible d8bd737c-374c-4cda-8a2f-b3dd53019d50 has session id: 9198772e-c0d2-4972-8542-7f153124da1a","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.561238559Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
94952024-03-06T21:01:50.238Z{"msg":"Upstairs opts: Upstairs UUID: d8bd737c-374c-4cda-8a2f-b3dd53019d50, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.561296145Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
94962024-03-06T21:01:50.238Z{"msg":"Crucible stats registered with UUID: d8bd737c-374c-4cda-8a2f-b3dd53019d50","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.561326393Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
94972024-03-06T21:01:50.238Z{"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.561362431Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491}
94982024-03-06T21:01:50.238Z{"msg":"d8bd737c-374c-4cda-8a2f-b3dd53019d50 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.561387369Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
94992024-03-06T21:01:50.238Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.561423736Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"client":"0","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95002024-03-06T21:01:50.238Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.561455424Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"client":"0","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95012024-03-06T21:01:50.238Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.561488562Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"client":"1","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95022024-03-06T21:01:50.238Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.56151841Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"client":"1","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95032024-03-06T21:01:50.238Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.561632012Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"client":"2","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95042024-03-06T21:01:50.238Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.561677779Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"client":"2","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95052024-03-06T21:01:50.239Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.561714826Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"":"io task","client":"2","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95062024-03-06T21:01:50.239Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.561791181Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"":"io task","client":"0","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95072024-03-06T21:01:50.239Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.561837728Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"":"io task","client":"1","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95082024-03-06T21:01:50.239Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.561880795Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"":"io task","client":"0","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95092024-03-06T21:01:50.239Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.561920262Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"":"io task","client":"1","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95102024-03-06T21:01:50.239Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.561960989Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"":"io task","client":"2","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95112024-03-06T21:01:50.239Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.562593436Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"client":"1","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95122024-03-06T21:01:50.239Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.562657681Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"client":"1","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95132024-03-06T21:01:50.239Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.562712288Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"client":"2","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95142024-03-06T21:01:50.239Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.562731866Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"client":"2","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95152024-03-06T21:01:50.239Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.563314556Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"client":"1","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95162024-03-06T21:01:50.239Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.563350744Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"client":"2","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95172024-03-06T21:01:50.240Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.563868458Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"client":"2","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95182024-03-06T21:01:50.240Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.563919364Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"client":"2","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95192024-03-06T21:01:50.240Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.563960282Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95202024-03-06T21:01:50.240Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.564002579Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"client":"1","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95212024-03-06T21:01:50.240Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.564032337Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"client":"1","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95222024-03-06T21:01:50.240Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.564069774Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95232024-03-06T21:01:50.240Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.564105702Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"client":"0","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95242024-03-06T21:01:50.240Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.564144669Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"client":"0","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95252024-03-06T21:01:50.240Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.564609237Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"client":"0","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95262024-03-06T21:01:50.240Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.565051776Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"client":"0","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95272024-03-06T21:01:50.240Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.565091214Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"client":"0","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95282024-03-06T21:01:50.240Z{"msg":"[0]R flush_numbers[0..12]: [151, 151, 150, 144, 102, 144, 151, 144, 151, 151, 151, 142]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.565107793Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"":"downstairs","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95292024-03-06T21:01:50.240Z{"msg":"[0]R generation[0..12]: [87, 87, 86, 84, 61, 84, 87, 84, 87, 87, 87, 82]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.565123161Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"":"downstairs","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95302024-03-06T21:01:50.240Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.565138131Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"":"downstairs","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95312024-03-06T21:01:50.241Z{"msg":"[1]R flush_numbers[0..12]: [151, 151, 150, 144, 102, 144, 151, 144, 151, 151, 151, 142]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.565159859Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"":"downstairs","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95322024-03-06T21:01:50.241Z{"msg":"[1]R generation[0..12]: [87, 87, 86, 84, 61, 84, 87, 84, 87, 87, 87, 82]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.565174018Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"":"downstairs","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95332024-03-06T21:01:50.241Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.565194647Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"":"downstairs","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95342024-03-06T21:01:50.241Z{"msg":"[2]R flush_numbers[0..12]: [151, 151, 150, 144, 102, 144, 151, 144, 151, 151, 151, 142]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.565208906Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"":"downstairs","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95352024-03-06T21:01:50.241Z{"msg":"[2]R generation[0..12]: [87, 87, 86, 84, 61, 84, 87, 84, 87, 87, 87, 82]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.565229104Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"":"downstairs","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95362024-03-06T21:01:50.241Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.565243573Z","hostname":"Wait for a query_work_queue command to finish before sending IO
95372024-03-06T21:01:50.241Zw-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"":"downstairs","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95382024-03-06T21:01:50.241Z{"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.565260842Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"":"downstairs","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95392024-03-06T21:01:50.241Z{"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.565280181Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"":"downstairs","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95402024-03-06T21:01:50.241Z{"msg":"Next flush: 152","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.56529357Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"":"downstairs","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95412024-03-06T21:01:50.241Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.565313258Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"":"downstairs","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95422024-03-06T21:01:50.241Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.565326627Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95432024-03-06T21:01:50.241Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.565344896Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"client":"0","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95442024-03-06T21:01:50.241Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.565358445Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"client":"1","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95452024-03-06T21:01:50.241Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.565372144Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"client":"2","":"downstairs","session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95462024-03-06T21:01:50.241Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.565385663Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95472024-03-06T21:01:50.241Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.565404652Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95482024-03-06T21:01:50.241Z{"msg":"d8bd737c-374c-4cda-8a2f-b3dd53019d50 is now active with session: 9198772e-c0d2-4972-8542-7f153124da1a","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.565421711Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95492024-03-06T21:01:50.242Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.565452679Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491,"session_id":"9198772e-c0d2-4972-8542-7f153124da1a"}
95502024-03-06T21:01:50.242Z{"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:48.565483967Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491}
95512024-03-06T21:01:50.242ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
95522024-03-06T21:01:50.242ZLoading write count information from file "/tmp/test_repair_verify.data"
95532024-03-06T21:01:50.242ZRun Repair workload
95542024-03-06T21:01:50.242Z01/30 Write block 371 len 12288 data: 11 10 10
95552024-03-06T21:01:50.242Z02/30 Flush
95562024-03-06T21:01:50.242Z03/30 Write block 438 len 8192 data: 10 11
95572024-03-06T21:01:50.242Z04/30 Read block 370 len 40960
95582024-03-06T21:01:50.242Z05/30 Write block 543 len 24576 data: 11 11 11 9 9 8
95592024-03-06T21:01:50.242Z06/30 Flush
95602024-03-06T21:01:50.242Z07/30 Write block 502 len 32768 data: 5 5 5 5 5 6 5 6
95612024-03-06T21:01:50.242Z08/30 Read block 127 len 24576
95622024-03-06T21:01:50.242Z09/30 Read block 346 len 36864
95632024-03-06T21:01:50.242Z10/30 Write block 388 len 12288 data: 7 7 7
95642024-03-06T21:01:50.242Z11/30 Write block 523 len 40960 data: 4 6 6 6 7 7 8 10 9 8
95652024-03-06T21:01:50.242Z12/30 Write block 124 len 16384 data: 7 11 10 11
95662024-03-06T21:01:50.242Z13/30 Read block 423 len 12288
95672024-03-06T21:01:50.242Z14/30 Write block 375 len 40960 data: 10 7 6 5 6 7 6 8 7 9
95682024-03-06T21:01:50.243Z15/30 Write block 545 len 8192 data: 12 10
95692024-03-06T21:01:50.243Z16/30 Read block 537 len 24576
95702024-03-06T21:01:50.243Z17/30 Read block 196 len 40960
95712024-03-06T21:01:50.243Z18/30 Read block 207 len 36864
95722024-03-06T21:01:50.243Z19/30 Read block 211 len 16384
95732024-03-06T21:01:50.257Z20/30 Read block 500 len 40960
95742024-03-06T21:01:50.257Z21/30 Read block 477 len 8192
95752024-03-06T21:01:50.257Z22/30 Write block 271 len 36864 data: 6 6 6 10 8 10 8 8 9
95762024-03-06T21:01:50.257Z23/30 Read block 213 len 16384
95772024-03-06T21:01:50.257Z24/30 Read block 272 len 36864
95782024-03-06T21:01:50.257Z25/30 Read block 536 len 20480
95792024-03-06T21:01:50.257Z26/30 Read block 558 len 16384
95802024-03-06T21:01:50.257Z27/30 Write block 361 len 16384 data: 12 11 15 12
95812024-03-06T21:01:50.257Z28/30 Read block 92 len 24576
95822024-03-06T21:01:50.257Z29/30 Write block 72 len 12288 data: 8 7 8
95832024-03-06T21:01:50.258Z30/30 Read block 173 len 20480
95842024-03-06T21:01:50.258Z----------------------------------------------------------------
95852024-03-06T21:01:50.258Z Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:28
95862024-03-06T21:01:50.258ZStates: Active Active Active
95872024-03-06T21:01:50.258ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
95882024-03-06T21:01:50.258Z 3 Acked 1002 Write 2 Done Done Done false
95892024-03-06T21:01:50.258Z 4 Acked 1003 Read 10 Done Done Done false
95902024-03-06T21:01:50.258Z 5 Acked 1004 Write 6 Done Done Done false
95912024-03-06T21:01:50.258Z 6 Acked 1005 Flush 0 Sent Done Done false
95922024-03-06T21:01:50.258Z 7 Acked 1006 Write 8 Sent Done Done false
95932024-03-06T21:01:50.258Z 8 Acked 1007 Read 6 Sent Done Done false
95942024-03-06T21:01:50.258Z 9 Acked 1008 Read 9 Sent Done Done false
95952024-03-06T21:01:50.259Z 10 Acked 1009 Write 3 Sent Done Done false
95962024-03-06T21:01:50.259Z 11 Acked 1010 Write 10 Sent Done Done false
95972024-03-06T21:01:50.259Z 12 Acked 1011 Write 4 Sent Done Done false
95982024-03-06T21:01:50.259Z 13 Acked 1012 Read 3 Sent Done Done false
95992024-03-06T21:01:50.259Z 14 Acked 1013 Write 10 Sent Done Done false
96002024-03-06T21:01:50.259Z 15 Acked 1014 Write 2 Sent Done Done false
96012024-03-06T21:01:50.259Z 16 Acked 1015 Read 6 Sent Done Done false
96022024-03-06T21:01:50.259Z 17 Acked 1016 Read 10 Sent Done Done false
96032024-03-06T21:01:50.259Z 18 Acked 1017 Read 9 Sent Done Done false
96042024-03-06T21:01:50.259Z 19 Acked 1018 Read 4 Sent Done Done false
96052024-03-06T21:01:50.259Z 20 Acked 1019 Read 10 Sent Done Done false
96062024-03-06T21:01:50.259Z 21 Acked 1020 Read 2 Sent Done Done false
96072024-03-06T21:01:50.259Z 22 Acked 1021 Write 9 Sent Done Done false
96082024-03-06T21:01:50.259Z 23 Acked 1022 Read 4 Sent Done Done false
96092024-03-06T21:01:50.260Z 24 Acked 1023 Read 9 Sent Done Done false
96102024-03-06T21:01:50.260Z 25 Acked 1024 Read 5 Sent Done Done false
96112024-03-06T21:01:50.260Z 26 Acked 1025 Read 4 Sent Done Done false
96122024-03-06T21:01:50.260Z 27 Acked 1026 Write 4 Sent Done Done false
96132024-03-06T21:01:50.260Z 28 Acked 1027 Read 6 Sent Done Done false
96142024-03-06T21:01:50.260Z 29 Acked 1028 Write 3 Sent Done Done false
96152024-03-06T21:01:50.260Z 30 Acked 1029 Read 5 Sent Done Sent false
96162024-03-06T21:01:50.260Z STATES DS:0 DS:1 DS:2 TOTAL
96172024-03-06T21:01:50.261Z New 0 0 0 0
96182024-03-06T21:01:50.261Z Sent 25 0 1 26
96192024-03-06T21:01:50.261Z Done 3 28 27 58
96202024-03-06T21:01:50.261Z Skipped 0 0 0 0
96212024-03-06T21:01:50.261Z Error 0 0 0 0
96222024-03-06T21:01:50.261ZLast Flush: 1001 1005 1005
96232024-03-06T21:01:50.261ZDownstairs last five completed: 1001 1000
96242024-03-06T21:01:50.261ZUpstairs last five completed: 30 29 28 27 26
96252024-03-06T21:01:50.261Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:01:48.601524463Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1491}
96262024-03-06T21:01:50.261ZWrote out file "/tmp/test_repair_verify.data"
96272024-03-06T21:01:50.261Z
96282024-03-06T21:01:50.261ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
96292024-03-06T21:01:50.262ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
96302024-03-06T21:01:50.262Z 0 000-019 87 87 87 151 151 151 F F F
96312024-03-06T21:01:50.262Z 1 020-039 87 87 87 151 151 151 F F F
96322024-03-06T21:01:50.262Z 2 040-059 86 86 86 150 150 150 F F F
96332024-03-06T21:01:50.262Z 3 060-079 84 84 84 144 144 144 F T T <---
96342024-03-06T21:01:50.262Z 4 080-099 61 61 61 102 102 102 F F F
96352024-03-06T21:01:50.262Z 5 100-119 84 84 84 144 144 144 F F F
96362024-03-06T21:01:50.262Z 6 120-139 87 87 87 151 151 151 F T T <---
96372024-03-06T21:01:50.262Z 7 140-159 84 84 84 144 144 144 F F F
96382024-03-06T21:01:50.262Z 8 160-179 87 87 87 151 151 151 F F F
96392024-03-06T21:01:50.262Z 9 180-199 87 87 87 151 151 151 F F F
96402024-03-06T21:01:50.262Z 10 200-219 87 87 87 151 151 151 F F F
96412024-03-06T21:01:50.262Z 11 220-239 82 82 82 142 142 142 F F F
96422024-03-06T21:01:50.262Z 12 240-259 74 74 74 125 125 125 F F F
96432024-03-06T21:01:50.262Z 13 260-279 78 78 78 136 136 136 F T T <---
96442024-03-06T21:01:50.262Z 14 280-299 87 87 87 151 151 151 F F F
96452024-03-06T21:01:50.262Z 15 300-319 87 87 87 151 151 151 F F F
96462024-03-06T21:01:50.262Z 16 320-339 84 84 84 144 144 144 F F F
96472024-03-06T21:01:50.263Z 17 340-359 84 84 84 144 144 144 F F F
96482024-03-06T21:01:50.263Z 18 360-379 88 88 88 152 152 152 F T T <---
96492024-03-06T21:01:50.263Z 19 380-399 87 87 87 151 151 151 F T T <---
96502024-03-06T21:01:50.263Z 20 400-419 76 76 76 131 131 131 F F F
96512024-03-06T21:01:50.263Z 21 420-439 87 88 88 151 153 153 T F F <---
96522024-03-06T21:01:50.263Z 22 440-459 87 87 87 151 151 151 F F F
96532024-03-06T21:01:50.263Z 23 460-479 78 78 78 137 137 137 F F F
96542024-03-06T21:01:50.263Z 24 480-499 84 84 84 144 144 144 F F F
96552024-03-06T21:01:50.263Z 25 500-519 80 80 80 139 139 139 F T T <---
96562024-03-06T21:01:50.263Z 26 520-539 84 84 84 144 144 144 F T T <---
96572024-03-06T21:01:50.263Z 27 540-559 84 88 88 144 153 153 T T T <---
96582024-03-06T21:01:50.263Z 28 560-579 83 83 83 143 143 143 F F F
96592024-03-06T21:01:50.263Z 29 580-599 87 87 87 151 151 151 F F F
96602024-03-06T21:01:50.263ZMax gen: 88, Max flush: 153
96612024-03-06T21:01:50.263ZError: Difference in extent metadata found!
96622024-03-06T21:01:50.263ZA Difference in extent metadata is expected here
96632024-03-06T21:01:50.264ZOn loop 44
96642024-03-06T21:01:50.264Z
96652024-03-06T21:01:50.264ZVerifying data now
96662024-03-06T21:01:55.406ZNo change after verify
96672024-03-06T21:01:55.406ZLoop: 44 Downstairs dump after verify (and repair):
96682024-03-06T21:01:55.421ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
96692024-03-06T21:01:55.421Z 0 000-019 87 87 87 151 151 151 F F F
96702024-03-06T21:01:55.421Z 1 020-039 87 87 87 151 151 151 F F F
96712024-03-06T21:01:55.422Z 2 040-059 86 86 86 150 150 150 F F F
96722024-03-06T21:01:55.422Z 3 060-079 89 89 89 154 154 154 F F F
96732024-03-06T21:01:55.422Z 4 080-099 61 61 61 102 102 102 F F F
96742024-03-06T21:01:55.422Z 5 100-119 84 84 84 144 144 144 F F F
96752024-03-06T21:01:55.422Z 6 120-139 89 89 89 154 154 154 F F F
96762024-03-06T21:01:55.422Z 7 140-159 84 84 84 144 144 144 F F F
96772024-03-06T21:01:55.422Z 8 160-179 87 87 87 151 151 151 F F F
96782024-03-06T21:01:55.422Z 9 180-199 87 87 87 151 151 151 F F F
96792024-03-06T21:01:55.422Z 10 200-219 87 87 87 151 151 151 F F F
96802024-03-06T21:01:55.422Z 11 220-239 82 82 82 142 142 142 F F F
96812024-03-06T21:01:55.422Z 12 240-259 74 74 74 125 125 125 F F F
96822024-03-06T21:01:55.422Z 13 260-279 89 89 89 154 154 154 F F F
96832024-03-06T21:01:55.422Z 14 280-299 87 87 87 151 151 151 F F F
96842024-03-06T21:01:55.423Z 15 300-319 87 87 87 151 151 151 F F F
96852024-03-06T21:01:55.423Z 16 320-339 84 84 84 144 144 144 F F F
96862024-03-06T21:01:55.423Z 17 340-359 84 84 84 144 144 144 F F F
96872024-03-06T21:01:55.427Z 18 360-379 89 89 89 154 154 154 F F F
96882024-03-06T21:01:55.427Z 19 380-399 89 89 89 154 154 154 F F F
96892024-03-06T21:01:55.427Z 20 400-419 76 76 76 131 131 131 F F F
96902024-03-06T21:01:55.427Z 21 420-439 88 88 88 153 153 153 F F F
96912024-03-06T21:01:55.427Z 22 440-459 87 87 87 151 151 151 F F F
96922024-03-06T21:01:55.427Z 23 460-479 78 78 78 137 137 137 F F F
96932024-03-06T21:01:55.427Z 24 480-499 84 84 84 144 144 144 F F F
96942024-03-06T21:01:55.427Z 25 500-519 89 89 89 154 154 154 F F F
96952024-03-06T21:01:55.427Z 26 520-539 89 89 89 154 154 154 F F F
96962024-03-06T21:01:55.427Z 27 540-559 89 89 89 154 154 154 F F F
96972024-03-06T21:01:55.427Z 28 560-579 83 83 83 143 143 143 F F F
96982024-03-06T21:01:55.427Z 29 580-599 87 87 87 151 151 151 F F F
96992024-03-06T21:01:55.427ZMax gen: 89, Max flush: 154
97002024-03-06T21:01:55.427Z
97012024-03-06T21:01:55.427ZBegin loop 45
97022024-03-06T21:01:55.427ZDownstairs to restart: 2
97032024-03-06T21:01:55.448Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.205274739Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499}
97042024-03-06T21:01:55.448Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.205548331Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499}
97052024-03-06T21:01:55.449Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.205574269Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499}
97062024-03-06T21:01:55.449Z{"msg":"Crucible b0a1374b-cb37-4656-84e6-da10ebefe6ce has session id: 83eccf65-2048-440e-8ba2-5d53b991f07d"Crucible runtime is spawned
97072024-03-06T21:01:55.455Z,"v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.205618256Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97082024-03-06T21:01:55.455Z{"msg":"Upstairs opts: Upstairs UUID: b0a1374b-cb37-4656-84e6-da10ebefe6ce, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.205657063Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97092024-03-06T21:01:55.455Z{"msg":"Crucible stats registered with UUID: b0a1374b-cb37-4656-84e6-da10ebefe6ce","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.205683201Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97102024-03-06T21:01:55.455Z{"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.205717149Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499}
97112024-03-06T21:01:55.455Z{"msg":"b0a1374b-cb37-4656-84e6-da10ebefe6ce active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.205738418Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97122024-03-06T21:01:55.455Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.205762606Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"client":"0","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97132024-03-06T21:01:55.455Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.205786814Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"client":"0","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97142024-03-06T21:01:55.455Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.205823401Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"client":"1","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97152024-03-06T21:01:55.455Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.20584823Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"client":"1","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97162024-03-06T21:01:55.455Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.205890837Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"client":"2","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97172024-03-06T21:01:55.455Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.205951143Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"client":"2","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97182024-03-06T21:01:55.455Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.205979191Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"":"io task","client":"0","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97192024-03-06T21:01:55.456Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.206020808Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"":"io task","client":"2","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97202024-03-06T21:01:55.456Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.206051156Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"":"io task","client":"1","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97212024-03-06T21:01:55.456Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.20613321Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"":"io task","client":"0","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97222024-03-06T21:01:55.456Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.206187687Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"":"io task","client":"2","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97232024-03-06T21:01:55.456Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.206224714Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"":"io task","client":"1","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97242024-03-06T21:01:55.456Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.206993041Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"client":"0","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97252024-03-06T21:01:55.456Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.207047567Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"client":"0","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97262024-03-06T21:01:55.456Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.207108933Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"client":"1","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97272024-03-06T21:01:55.456Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.20714641Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"client":"1","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97282024-03-06T21:01:55.456Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.207805085Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"client":"1","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97292024-03-06T21:01:55.456Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.207851932Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"client":"0","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97302024-03-06T21:01:55.456Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.208132382Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"client":"2","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97312024-03-06T21:01:55.456Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.208184099Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"client":"2","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97322024-03-06T21:01:55.457Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.208530065Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"client":"0","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97332024-03-06T21:01:55.457Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.208570682Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"client":"0","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97342024-03-06T21:01:55.457Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.208587511Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97352024-03-06T21:01:55.457Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.20860061Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"client":"1","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97362024-03-06T21:01:55.457Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.208623468Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"client":"1","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97372024-03-06T21:01:55.457Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.208637978Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97382024-03-06T21:01:55.457Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.208774398Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"client":"2","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97392024-03-06T21:01:55.457Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.209236656Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"client":"2","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97402024-03-06T21:01:55.457Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.209273414Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"client":"2","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97412024-03-06T21:01:55.457Z{"msg":"[0]R flush_numbers[0..12]: [151, 151, 150, 154, 102, 144, 154, 144, 151, 151, 151, 142]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.209290143Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"":"downstairs","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97422024-03-06T21:01:55.457Z{"msg":"[0]R generation[0..12]: [87, 87, 86, 89, 61, 84, 89, 84, 87, 87, 87, 82]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.209304792Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"":"downstairs","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97432024-03-06T21:01:55.457Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.20932732Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"":"downstairs","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97442024-03-06T21:01:55.457Z{"msg":"[1]R flush_numbers[0..12]: [151, 151, 150, 154, 102, 144, 154, 144, 151, 151, 151, 142]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.209342679Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"":"downstairs","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97452024-03-06T21:01:55.458Z{"msg":"[1]R generation[0..12]: [87, 87, 86, 89, 61, 84, 89, 84, 87, 87, 87, 82]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.209356518Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"":"downstairs","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97462024-03-06T21:01:55.458Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
97472024-03-06T21:01:55.458Z,"time":"2024-03-06T21:01:54.209370547Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"":"downstairs","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97482024-03-06T21:01:55.458Z{"msg":"[2]R flush_numbers[0..12]: [151, 151, 150, 154, 102, 144, 154, 144, 151, 151, 151, 142]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.209386966Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"":"downstairs","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97492024-03-06T21:01:55.458Z{"msg":"[2]R generation[0..12]: [87, 87, 86, 89, 61, 84, 89, 84, 87, 87, 87, 82]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.209408465Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"":"downstairs","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97502024-03-06T21:01:55.458Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.209423113Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"":"downstairs","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97512024-03-06T21:01:55.458Z{"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.209436612Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"":"downstairs","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97522024-03-06T21:01:55.458Z{"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.209456971Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"":"downstairs","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97532024-03-06T21:01:55.458Z{"msg":"Next flush: 155","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.20947084Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"":"downstairs","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97542024-03-06T21:01:55.458Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.209485229Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"":"downstairs","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97552024-03-06T21:01:55.458Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.209518757Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97562024-03-06T21:01:55.458Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.209538985Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"client":"0","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97572024-03-06T21:01:55.459Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.209572693Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"client":"1","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97582024-03-06T21:01:55.459Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.209591622Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"client":"2","":"downstairs","session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97592024-03-06T21:01:55.459Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.20962469Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97602024-03-06T21:01:55.459Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.209642208Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97612024-03-06T21:01:55.459Z{"msg":"b0a1374b-cb37-4656-84e6-da10ebefe6ce is now active with session: 83eccf65-2048-440e-8ba2-5d53b991f07d","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.209673506Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97622024-03-06T21:01:55.459Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
97632024-03-06T21:01:55.459Z,"time":"2024-03-06T21:01:54.209691855Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499,"session_id":"83eccf65-2048-440e-8ba2-5d53b991f07d"}
97642024-03-06T21:01:55.459Z{"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:01:54.209715643Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499}
97652024-03-06T21:01:55.463ZLoading write count information from file "/tmp/test_repair_verify.data"
97662024-03-06T21:01:55.463ZRun Repair workload
97672024-03-06T21:01:55.464Z01/30 Write block 285 len 20480 data: 11 12 12 14 14
97682024-03-06T21:01:55.464Z02/30 Read block 468 len 28672
97692024-03-06T21:01:55.464Z03/30 Write block 344 len 24576 data: 8 7 6 7 10 9
97702024-03-06T21:01:55.464Z04/30 Flush
97712024-03-06T21:01:55.464Z05/30 Write block 318 len 28672 data: 8 8 6 5 6 5 5
97722024-03-06T21:01:55.464Z06/30 Read block 233 len 8192
97732024-03-06T21:01:55.464Z07/30 Write block 390 len 8192 data: 8 7
97742024-03-06T21:01:55.464Z08/30 Write block 139 len 12288 data: 5 5 4
97752024-03-06T21:01:55.472Z09/30 Read block 30 len 12288
97762024-03-06T21:01:55.472Z10/30 Read block 243 len 8192
97772024-03-06T21:01:55.472Z11/30 Write block 539 len 40960 data: 6 9 7 10 12 12 13 11 10 9
97782024-03-06T21:01:55.472Z12/30 Read block 488 len 32768
97792024-03-06T21:01:55.472Z13/30 Write block 400 len 24576 data: 4 3 4 6 4 7
97802024-03-06T21:01:55.473Z14/30 Write block 443 len 24576 data: 4 6 5 6 4 8
97812024-03-06T21:01:55.473Z15/30 Write block 291 len 32768 data: 9 7 6 5 8 7 6 6
97822024-03-06T21:01:55.473Z16/30 Read block 522 len 40960
97832024-03-06T21:01:55.476Z17/30 Write block 554 len 32768 data: 5 6 5 5 5 4 7 7
97842024-03-06T21:01:55.476Z18/30 Read block 484 len 20480
97852024-03-06T21:01:55.476Z19/30 Read block 372 len 4096
97862024-03-06T21:01:55.476Z20/30 Read block 240 len 36864
97872024-03-06T21:01:55.479Z21/30 Read block 250 len 40960
97882024-03-06T21:01:55.479Z22/30 Write block 190 len 12288 data: 8 7 9
97892024-03-06T21:01:55.479Z23/30 Read block 101 len 32768
97902024-03-06T21:01:55.482Z24/30 Write block 407 len 16384 data: 7 8 6 6
97912024-03-06T21:01:55.482Z25/30 Read block 484 len 36864
97922024-03-06T21:01:55.482Z26/30 Write block 407 len 12288 data: 8 9 7
97932024-03-06T21:01:55.486Z27/30 Write block 222 len 8192 data: 4 6
97942024-03-06T21:01:55.486Z28/30 Write block 2 len 28672 data: 5 6 7 8 7 8 7
97952024-03-06T21:01:55.486Z29/30 Read block 127 len 36864
97962024-03-06T21:01:55.491Z30/30 Write block 390 len 24576 data: 9 8 7 7 6 5
97972024-03-06T21:01:55.491Z----------------------------------------------------------------
97982024-03-06T21:01:55.491Z Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:26
97992024-03-06T21:01:55.491ZStates: Active Active Active
98002024-03-06T21:01:55.491ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
98012024-03-06T21:01:55.491Z 5 Acked 1004 Write 7 Done Done Done false
98022024-03-06T21:01:55.491Z 6 Acked 1005 Read 2 Done Done Done false
98032024-03-06T21:01:55.491Z 7 Acked 1006 Write 2 Done Done Done false
98042024-03-06T21:01:55.491Z 8 Acked 1007 Write 3 Done Done Done false
98052024-03-06T21:01:55.491Z 9 Acked 1008 Read 3 Done Done Done false
98062024-03-06T21:01:55.492Z 10 Acked 1009 Read 2 Done Done Sent false
98072024-03-06T21:01:55.492Z 11 Acked 1010 Write 10 Done Done Sent false
98082024-03-06T21:01:55.492Z 12 Acked 1011 Read 8 Done Done Sent false
98092024-03-06T21:01:55.492Z 13 Acked 1012 Write 6 Done Done Sent false
98102024-03-06T21:01:55.492Z 14 Acked 1013 Write 6 Done Done Sent false
98112024-03-06T21:01:55.492Z 15 Acked 1014 Write 8 Done Done Sent false
98122024-03-06T21:01:55.492Z 16 Acked 1015 Read 10 Done Done Sent false
98132024-03-06T21:01:55.492Z 17 Acked 1016 Write 8 Done Done Sent false
98142024-03-06T21:01:55.492Z 18 Acked 1017 Read 5 Done Done Sent false
98152024-03-06T21:01:55.492Z 19 Acked 1018 Read 1 Done Done Sent false
98162024-03-06T21:01:55.492Z 20 Acked 1019 Read 9 Done Done Sent false
98172024-03-06T21:01:55.492Z 21 Acked 1020 Read 10 Done Done Sent false
98182024-03-06T21:01:55.492Z 22 Acked 1021 Write 3 Done Done Sent false
98192024-03-06T21:01:55.492Z 23 Acked 1022 Read 8 Done Done Sent false
98202024-03-06T21:01:55.493Z 24 Acked 1023 Write 4 Done Done Sent false
98212024-03-06T21:01:55.493Z 25 Acked 1024 Read 9 Done Done Sent false
98222024-03-06T21:01:55.493Z 26 Acked 1025 Write 3 Done Done Sent false
98232024-03-06T21:01:55.493Z 27 Acked 1026 Write 2 Done Done Sent false
98242024-03-06T21:01:55.493Z 28 Acked 1027 Write 7 Done Done Sent false
98252024-03-06T21:01:55.493Z 29 Acked 1028 Read 9 Done Done Sent false
98262024-03-06T21:01:55.493Z 30 Acked 1029 Write 6 Sent Sent Sent false
98272024-03-06T21:01:55.494Z STATES DS:0 DS:1 DS:2 TOTAL
98282024-03-06T21:01:55.494Z New 0 0 0 0
98292024-03-06T21:01:55.494Z Sent 1 1 21 23
98302024-03-06T21:01:55.494Z Done 25 25 5 55
98312024-03-06T21:01:55.494Z Skipped 0 0 0 0
98322024-03-06T21:01:55.494Z Error 0 0 0 0
98332024-03-06T21:01:55.494ZLast Flush: 1003 1003 1003
98342024-03-06T21:01:55.494ZDownstairs last five completed: 1003 1002 1001 1000
98352024-03-06T21:01:55.494ZUpstairs last five completed: 30 29 28 27 26
98362024-03-06T21:01:55.494Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:01:54.244653676Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1499}
98372024-03-06T21:01:55.494ZWrote out file "/tmp/test_repair_verify.data"
98382024-03-06T21:01:55.504Z
98392024-03-06T21:01:55.504ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
98402024-03-06T21:01:55.526ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
98412024-03-06T21:01:55.526Z 0 000-019 87 87 87 151 151 151 T T F <---
98422024-03-06T21:01:55.526Z 1 020-039 87 87 87 151 151 151 F F F
98432024-03-06T21:01:55.526Z 2 040-059 86 86 86 150 150 150 F F F
98442024-03-06T21:01:55.526Z 3 060-079 89 89 89 154 154 154 F F F
98452024-03-06T21:01:55.527Z 4 080-099 61 61 61 102 102 102 F F F
98462024-03-06T21:01:55.527Z 5 100-119 84 84 84 144 144 144 F F F
98472024-03-06T21:01:55.527Z 6 120-139 89 89 89 154 154 154 T T T
98482024-03-06T21:01:55.527Z 7 140-159 84 84 84 144 144 144 T T T
98492024-03-06T21:01:55.527Z 8 160-179 87 87 87 151 151 151 F F F
98502024-03-06T21:01:55.527Z 9 180-199 87 87 87 151 151 151 T T F <---
98512024-03-06T21:01:55.527Z 10 200-219 87 87 87 151 151 151 F F F
98522024-03-06T21:01:55.527Z 11 220-239 82 82 82 142 142 142 T T F <---
98532024-03-06T21:01:55.527Z 12 240-259 74 74 74 125 125 125 F F F
98542024-03-06T21:01:55.527Z 13 260-279 89 89 89 154 154 154 F F F
98552024-03-06T21:01:55.527Z 14 280-299 90 90 90 155 155 155 T T F <---
98562024-03-06T21:01:55.527Z 15 300-319 87 87 87 151 151 151 T T T
98572024-03-06T21:01:55.527Z 16 320-339 84 84 84 144 144 144 T T T
98582024-03-06T21:01:55.527Z 17 340-359 90 90 90 155 155 155 F F F
98592024-03-06T21:01:55.527Z 18 360-379 89 89 89 154 154 154 F F F
98602024-03-06T21:01:55.527Z 19 380-399 89 89 89 154 154 154 T T T
98612024-03-06T21:01:55.527Z 20 400-419 76 76 76 131 131 131 T T F <---
98622024-03-06T21:01:55.537Z 21 420-439 88 88 88 153 153 153 F F F
98632024-03-06T21:01:55.537Z 22 440-459 87 87 87 151 151 151 T T F <---
98642024-03-06T21:01:55.537Z 23 460-479 78 78 78 137 137 137 F F F
98652024-03-06T21:01:55.537Z 24 480-499 84 84 84 144 144 144 F F F
98662024-03-06T21:01:55.537Z 25 500-519 89 89 89 154 154 154 F F F
98672024-03-06T21:01:55.537Z 26 520-539 89 89 89 154 154 154 T T F <---
98682024-03-06T21:01:55.537Z 27 540-559 89 89 89 154 154 154 T T F <---
98692024-03-06T21:01:55.537Z 28 560-579 83 83 83 143 143 143 T T F <---
98702024-03-06T21:01:55.537Z 29 580-599 87 87 87 151 151 151 F F F
98712024-03-06T21:01:55.537ZMax gen: 90, Max flush: 155
98722024-03-06T21:01:55.537ZError: Difference in extent metadata found!
98732024-03-06T21:01:55.537ZA Difference in extent metadata is expected here
98742024-03-06T21:01:55.537ZOn loop 45
98752024-03-06T21:01:55.538Z
98762024-03-06T21:01:55.538ZVerifying data now
98772024-03-06T21:02:01.281ZNo change after verify
98782024-03-06T21:02:01.281ZLoop: 45 Downstairs dump after verify (and repair):
98792024-03-06T21:02:01.296ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
98802024-03-06T21:02:01.296Z 0 000-019 91 91 91 156 156 156 F F F
98812024-03-06T21:02:01.297Z 1 020-039 87 87 87 151 151 151 F F F
98822024-03-06T21:02:01.297Z 2 040-059 86 86 86 150 150 150 F F F
98832024-03-06T21:02:01.297Z 3 060-079 89 89 89 154 154 154 F F F
98842024-03-06T21:02:01.297Z 4 080-099 61 61 61 102 102 102 F F F
98852024-03-06T21:02:01.297Z 5 100-119 84 84 84 144 144 144 F F F
98862024-03-06T21:02:01.297Z 6 120-139 91 91 91 156 156 156 F F F
98872024-03-06T21:02:01.297Z 7 140-159 91 91 91 156 156 156 F F F
98882024-03-06T21:02:01.297Z 8 160-179 87 87 87 151 151 151 F F F
98892024-03-06T21:02:01.297Z 9 180-199 91 91 91 156 156 156 F F F
98902024-03-06T21:02:01.297Z 10 200-219 87 87 87 151 151 151 F F F
98912024-03-06T21:02:01.297Z 11 220-239 91 91 91 156 156 156 F F F
98922024-03-06T21:02:01.297Z 12 240-259 74 74 74 125 125 125 F F F
98932024-03-06T21:02:01.297Z 13 260-279 89 89 89 154 154 154 F F F
98942024-03-06T21:02:01.297Z 14 280-299 91 91 91 156 156 156 F F F
98952024-03-06T21:02:01.297Z 15 300-319 91 91 91 156 156 156 F F F
98962024-03-06T21:02:01.297Z 16 320-339 91 91 91 156 156 156 F F F
98972024-03-06T21:02:01.302Z 17 340-359 90 90 90 155 155 155 F F F
98982024-03-06T21:02:01.302Z 18 360-379 89 89 89 154 154 154 F F F
98992024-03-06T21:02:01.302Z 19 380-399 91 91 91 156 156 156 F F F
99002024-03-06T21:02:01.302Z 20 400-419 91 91 91 156 156 156 F F F
99012024-03-06T21:02:01.302Z 21 420-439 88 88 88 153 153 153 F F F
99022024-03-06T21:02:01.302Z 22 440-459 91 91 91 156 156 156 F F F
99032024-03-06T21:02:01.302Z 23 460-479 78 78 78 137 137 137 F F F
99042024-03-06T21:02:01.302Z 24 480-499 84 84 84 144 144 144 F F F
99052024-03-06T21:02:01.302Z 25 500-519 89 89 89 154 154 154 F F F
99062024-03-06T21:02:01.302Z 26 520-539 91 91 91 156 156 156 F F F
99072024-03-06T21:02:01.302Z 27 540-559 91 91 91 156 156 156 F F F
99082024-03-06T21:02:01.302Z 28 560-579 91 91 91 156 156 156 F F F
99092024-03-06T21:02:01.302Z 29 580-599 87 87 87 151 151 151 F F F
99102024-03-06T21:02:01.302ZMax gen: 91, Max flush: 156
99112024-03-06T21:02:01.302Z
99122024-03-06T21:02:01.302ZBegin loop 46
99132024-03-06T21:02:01.302ZDownstairs to restart: 2
99142024-03-06T21:02:01.322Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.078112071Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507}
99152024-03-06T21:02:01.322Z{"Crucible runtime is spawned
99162024-03-06T21:02:01.330Zmsg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.078450958Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507}
99172024-03-06T21:02:01.330Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.078501774Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507}
99182024-03-06T21:02:01.330Z{"msg":"Crucible 8b55f244-e2f9-4d0f-8241-362f2faefebe has session id: 784027a2-4b00-4cd1-94c9-e7787dca857e","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.078540532Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99192024-03-06T21:02:01.330Z{"msg":"Upstairs opts: Upstairs UUID: 8b55f244-e2f9-4d0f-8241-362f2faefebe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.078597388Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99202024-03-06T21:02:01.330Z{"msg":"Crucible stats registered with UUID: 8b55f244-e2f9-4d0f-8241-362f2faefebe","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.078638185Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99212024-03-06T21:02:01.330Z{"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.078685452Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507}
99222024-03-06T21:02:01.330Z{"msg":"8b55f244-e2f9-4d0f-8241-362f2faefebe active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.07871412Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99232024-03-06T21:02:01.330Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.078759207Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"client":"0","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99242024-03-06T21:02:01.330Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.078794704Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"client":"0","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99252024-03-06T21:02:01.330Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.078841311Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"client":"1","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99262024-03-06T21:02:01.330Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.078939664Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"client":"1","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99272024-03-06T21:02:01.330Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.078976182Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"client":"2","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99282024-03-06T21:02:01.330Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.079016619Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"client":"2","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99292024-03-06T21:02:01.330Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.079045537Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"":"io task","client":"1","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99302024-03-06T21:02:01.330Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.079099033Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"":"io task","client":"0","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99312024-03-06T21:02:01.330Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.079129281Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"":"io task","client":"2","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99322024-03-06T21:02:01.330Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.079180958Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"":"io task","client":"0","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99332024-03-06T21:02:01.331Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.079233834Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"":"io task","client":"2","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99342024-03-06T21:02:01.331Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.079258942Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"":"io task","client":"1","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99352024-03-06T21:02:01.331Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.079845572Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"client":"0","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99362024-03-06T21:02:01.331Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.079889149Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"client":"0","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99372024-03-06T21:02:01.331Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.079921027Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"client":"1","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99382024-03-06T21:02:01.331Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.079968203Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"client":"1","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99392024-03-06T21:02:01.331Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.080526015Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"client":"1","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99402024-03-06T21:02:01.331Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.080563692Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"client":"2","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99412024-03-06T21:02:01.331Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.080586821Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"client":"2","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99422024-03-06T21:02:01.331Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.080626818Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"client":"0","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99432024-03-06T21:02:01.331Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.081245855Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"client":"1","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99442024-03-06T21:02:01.331Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.081281713Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"client":"1","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99452024-03-06T21:02:01.331Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.081297152Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99462024-03-06T21:02:01.331Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.081310831Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"client":"2","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99472024-03-06T21:02:01.331Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.081331829Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"client":"0","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99482024-03-06T21:02:01.331Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.081345818Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"client":"0","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99492024-03-06T21:02:01.331Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.081371447Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99502024-03-06T21:02:01.331Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.081748601Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"client":"2","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99512024-03-06T21:02:01.331Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.081781228Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"client":"2","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99522024-03-06T21:02:01.331Z{"msg":"[0]R flush_numbers[0..12]: [156, 151, 150, 154, 102, 144, 156, 156, 151, 156, 151, 156]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.081796347Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"":"downstairs","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99532024-03-06T21:02:01.331Z{"msg":"[0]R generation[0..12]: [91, 87, 86, 89, 61, 84, 91, 91, 87, 91, 87, 91]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.081811456Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"":"downstairs","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99542024-03-06T21:02:01.331Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.081832705Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"":"downstairs","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99552024-03-06T21:02:01.331Z{"msg":"[1]R flush_numbers[0..12]: [156, 151, 150, 154, 102, 144, 156, 156, 151, 156, 151, 156]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.081847114Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"":"downstairs","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99562024-03-06T21:02:01.332Z{"msg":"[1]R generation[0..12]: [91, 87, 86, 89, 61, 84, 91, 91, 87, 91, 87, 91]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.081860973Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"":"downstairs","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99572024-03-06T21:02:01.332Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.081881491Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"":"downstairs","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99582024-03-06T21:02:01.332Z{"msg":"[2]R flush_numbers[0..12]: [156, 151, 150, 154, 102, 144, 156, 156, 151, 156, 151, 156]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.081895731Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"":"downstairs","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99592024-03-06T21:02:01.332Z{"msg":"[2]R generation[0..12]: [91, 87, 86, 89, 61, 84, 91, 91, 87, 91, 87, 91]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.08191045Z","hostname":"Wait for a query_work_queue command to finish before sending IO
99602024-03-06T21:02:01.332Zw-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"":"downstairs","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99612024-03-06T21:02:01.332Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.081927748Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"":"downstairs","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99622024-03-06T21:02:01.332Z{"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.081941037Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"":"downstairs","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99632024-03-06T21:02:01.332Z{"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.081962086Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"":"downstairs","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99642024-03-06T21:02:01.332Z{"msg":"Next flush: 157","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.081975445Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"":"downstairs","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99652024-03-06T21:02:01.332Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.081988514Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"":"downstairs","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99662024-03-06T21:02:01.332Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.082002283Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99672024-03-06T21:02:01.332Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.082014552Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"client":"0","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99682024-03-06T21:02:01.332Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.082027911Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"client":"1","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99692024-03-06T21:02:01.332Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.08205012Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"client":"2","":"downstairs","session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99702024-03-06T21:02:01.332Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.082064089Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99712024-03-06T21:02:01.332Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.082076338Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99722024-03-06T21:02:01.332Z{"msg":"8b55f244-e2f9-4d0f-8241-362f2faefebe is now active with session: 784027a2-4b00-4cd1-94c9-e7787dca857e","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.082096377Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99732024-03-06T21:02:01.332Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.082108646Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507,"session_id":"784027a2-4b00-4cd1-94c9-e7787dca857e"}
99742024-03-06T21:02:01.332Z{"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:00.082121125Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507}
99752024-03-06T21:02:01.332ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
99762024-03-06T21:02:01.332ZLoading write count information from file "/tmp/test_repair_verify.data"
99772024-03-06T21:02:01.332ZRun Repair workload
99782024-03-06T21:02:01.332Z01/30 Write block 270 len 36864 data: 6 7 7 7 11 9 11 9 9
99792024-03-06T21:02:01.337Z02/30 Read block 374 len 8192
99802024-03-06T21:02:01.337Z03/30 Read block 316 len 32768
99812024-03-06T21:02:01.337Z04/30 Write block 549 len 20480 data: 8 7 5 5 4
99822024-03-06T21:02:01.337Z05/30 Read block 27 len 8192
99832024-03-06T21:02:01.337Z06/30 Write block 51 len 20480 data: 5 7 6 4 7
99842024-03-06T21:02:01.337Z07/30 Write block 423 len 40960 data: 9 9 9 9 8 8 8 8 6 7
99852024-03-06T21:02:01.337Z08/30 Write block 107 len 20480 data: 11 8 8 7 8
99862024-03-06T21:02:01.337Z09/30 Write block 210 len 36864 data: 7 6 6 5 5 8 10 11 9
99872024-03-06T21:02:01.337Z10/30 Flush
99882024-03-06T21:02:01.341Z11/30 Write block 321 len 32768 data: 6 7 6 6 5 6 4 4
99892024-03-06T21:02:01.345Z12/30 Write block 261 len 20480 data: 5 4 6 5 6
99902024-03-06T21:02:01.345Z13/30 Read block 221 len 4096
99912024-03-06T21:02:01.345Z14/30 Write block 318 len 16384 data: 9 9 7 7
99922024-03-06T21:02:01.346Z15/30 Write block 230 len 28672 data: 8 8 7 6 8 9 11
99932024-03-06T21:02:01.346Z16/30 Write block 156 len 8192 data: 10 10
99942024-03-06T21:02:01.350Z17/30 Write block 543 len 20480 data: 13 13 14 12 11
99952024-03-06T21:02:01.350Z18/30 Read block 168 len 8192
99962024-03-06T21:02:01.350Z19/30 Read block 417 len 28672
99972024-03-06T21:02:01.350Z20/30 Read block 209 len 8192
99982024-03-06T21:02:01.350Z21/30 Read block 543 len 8192
99992024-03-06T21:02:01.353Z22/30 Flush
100002024-03-06T21:02:01.353Z23/30 Write block 133 len 16384 data: 6 6 6 7
100012024-03-06T21:02:01.353Z24/30 Write block 421 len 36864 data: 13 11 10 10 10 10 9 9 9
100022024-03-06T21:02:01.357Z25/30 Flush
100032024-03-06T21:02:01.357Z26/30 Write block 538 len 4096 data: 8
100042024-03-06T21:02:01.357Z27/30 Read block 115 len 8192
100052024-03-06T21:02:01.376Z28/30 Read block 88 len 12288
100062024-03-06T21:02:01.376Z29/30 Read block 176 len 40960
100072024-03-06T21:02:01.376Z30/30 Read block 445 len 4096
100082024-03-06T21:02:01.376Z----------------------------------------------------------------
100092024-03-06T21:02:01.376Z Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:30
100102024-03-06T21:02:01.376ZStates: Active Active Active
100112024-03-06T21:02:01.376ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
100122024-03-06T21:02:01.377Z 1 Acked 1000 Write 9 Done Done Sent false
100132024-03-06T21:02:01.377Z 2 Acked 1001 Read 2 Done Done Sent false
100142024-03-06T21:02:01.377Z 3 Acked 1002 Read 8 Done Done Sent false
100152024-03-06T21:02:01.377Z 4 Acked 1003 Write 5 Done Done Sent false
100162024-03-06T21:02:01.377Z 5 Acked 1004 Read 2 Done Done Sent false
100172024-03-06T21:02:01.377Z 6 Acked 1005 Write 5 Done Done Sent false
100182024-03-06T21:02:01.377Z 7 Acked 1006 Write 10 Done Done Sent false
100192024-03-06T21:02:01.377Z 8 Acked 1007 Write 5 Done Done Sent false
100202024-03-06T21:02:01.377Z 9 Acked 1008 Write 9 Done Done Sent false
100212024-03-06T21:02:01.377Z 10 Acked 1009 Flush 0 Done Done Sent false
100222024-03-06T21:02:01.377Z 11 Acked 1010 Write 8 Done Done Sent false
100232024-03-06T21:02:01.377Z 12 Acked 1011 Write 5 Done Done Sent false
100242024-03-06T21:02:01.377Z 13 Acked 1012 Read 1 Done Done Sent false
100252024-03-06T21:02:01.377Z 14 Acked 1013 Write 4 Done Done Sent false
100262024-03-06T21:02:01.378Z 15 Acked 1014 Write 7 Done Done Sent false
100272024-03-06T21:02:01.378Z 16 Acked 1015 Write 2 Done Done Sent false
100282024-03-06T21:02:01.378Z 17 Acked 1016 Write 5 Done Done Sent false
100292024-03-06T21:02:01.378Z 18 Acked 1017 Read 2 Done Done Sent false
100302024-03-06T21:02:01.378Z 19 Acked 1018 Read 7 Done Done Sent false
100312024-03-06T21:02:01.378Z 20 Acked 1019 Read 2 Done Done Sent false
100322024-03-06T21:02:01.378Z 21 Acked 1020 Read 2 Done Done Sent false
100332024-03-06T21:02:01.378Z 22 Acked 1021 Flush 0 Done Done Sent false
100342024-03-06T21:02:01.378Z 23 Acked 1022 Write 4 Done Done Sent false
100352024-03-06T21:02:01.379Z 24 Acked 1023 Write 9 Done Done Sent false
100362024-03-06T21:02:01.379Z 25 Acked 1024 Flush 0 Done Done Sent false
100372024-03-06T21:02:01.379Z 26 Acked 1025 Write 1 Done Done Sent false
100382024-03-06T21:02:01.379Z 27 Acked 1026 Read 2 Done Done Sent false
100392024-03-06T21:02:01.379Z 28 Acked 1027 Read 3 Done Done Sent false
100402024-03-06T21:02:01.379Z 29 Acked 1028 Read 10 Done Done Sent false
100412024-03-06T21:02:01.379Z 30 Acked 1029 Read 1 Done Done Sent false
100422024-03-06T21:02:01.379Z STATES DS:0 DS:1 DS:2 TOTAL
100432024-03-06T21:02:01.379Z New 0 0 0 0
100442024-03-06T21:02:01.379Z Sent 0 0 30 30
100452024-03-06T21:02:01.380Z Done 30 30 0 60
100462024-03-06T21:02:01.380Z Skipped 0 0 0 0
100472024-03-06T21:02:01.380Z Error 0 0 0 0
100482024-03-06T21:02:01.380ZLast Flush: 1024 1024 0
100492024-03-06T21:02:01.380ZDownstairs last five completed:
100502024-03-06T21:02:01.380ZUpstairs last five completed: 30 29 28 27 26
100512024-03-06T21:02:01.380Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:02:00.11730408Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1507}
100522024-03-06T21:02:01.380ZWrote out file "/tmp/test_repair_verify.data"
100532024-03-06T21:02:01.388Z
100542024-03-06T21:02:01.388ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
100552024-03-06T21:02:01.392ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
100562024-03-06T21:02:01.392Z 0 000-019 91 91 91 156 156 156 F F F
100572024-03-06T21:02:01.392Z 1 020-039 87 87 87 151 151 151 F F F
100582024-03-06T21:02:01.392Z 2 040-059 92 92 86 157 157 150 F F F <---
100592024-03-06T21:02:01.392Z 3 060-079 89 89 89 154 154 154 F F F
100602024-03-06T21:02:01.392Z 4 080-099 61 61 61 102 102 102 F F F
100612024-03-06T21:02:01.392Z 5 100-119 92 92 84 157 157 144 F F F <---
100622024-03-06T21:02:01.392Z 6 120-139 92 92 91 159 159 156 F F F <---
100632024-03-06T21:02:01.392Z 7 140-159 92 92 91 158 158 156 F F F <---
100642024-03-06T21:02:01.392Z 8 160-179 87 87 87 151 151 151 F F F
100652024-03-06T21:02:01.392Z 9 180-199 91 91 91 156 156 156 F F F
100662024-03-06T21:02:01.392Z 10 200-219 92 92 87 157 157 151 F F F <---
100672024-03-06T21:02:01.392Z 11 220-239 92 92 91 158 158 156 F F F <---
100682024-03-06T21:02:01.392Z 12 240-259 74 74 74 125 125 125 F F F
100692024-03-06T21:02:01.392Z 13 260-279 92 92 89 158 158 154 F F F <---
100702024-03-06T21:02:01.392Z 14 280-299 91 91 91 156 156 156 F F F
100712024-03-06T21:02:01.392Z 15 300-319 92 92 91 158 158 156 F F F <---
100722024-03-06T21:02:01.392Z 16 320-339 92 92 91 158 158 156 F F F <---
100732024-03-06T21:02:01.393Z 17 340-359 90 90 90 155 155 155 F F F
100742024-03-06T21:02:01.393Z 18 360-379 89 89 89 154 154 154 F F F
100752024-03-06T21:02:01.393Z 19 380-399 91 91 91 156 156 156 F F F
100762024-03-06T21:02:01.393Z 20 400-419 91 91 91 156 156 156 F F F
100772024-03-06T21:02:01.393Z 21 420-439 92 92 88 159 159 153 F F F <---
100782024-03-06T21:02:01.393Z 22 440-459 91 91 91 156 156 156 F F F
100792024-03-06T21:02:01.393Z 23 460-479 78 78 78 137 137 137 F F F
100802024-03-06T21:02:01.393Z 24 480-499 84 84 84 144 144 144 F F F
100812024-03-06T21:02:01.393Z 25 500-519 89 89 89 154 154 154 F F F
100822024-03-06T21:02:01.393Z 26 520-539 91 91 91 156 156 156 T T F <---
100832024-03-06T21:02:01.393Z 27 540-559 92 92 91 158 158 156 F F F <---
100842024-03-06T21:02:01.393Z 28 560-579 91 91 91 156 156 156 F F F
100852024-03-06T21:02:01.393Z 29 580-599 87 87 87 151 151 151 F F F
100862024-03-06T21:02:01.393ZMax gen: 92, Max flush: 159
100872024-03-06T21:02:01.393ZError: Difference in extent metadata found!
100882024-03-06T21:02:01.393ZA Difference in extent metadata is expected here
100892024-03-06T21:02:01.393ZOn loop 46
100902024-03-06T21:02:01.393Z
100912024-03-06T21:02:01.398ZVerifying data now
100922024-03-06T21:02:06.031ZNo change after verify
100932024-03-06T21:02:06.031ZLoop: 46 Downstairs dump after verify (and repair):
100942024-03-06T21:02:06.048ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
100952024-03-06T21:02:06.048Z 0 000-019 91 91 91 156 156 156 F F F
100962024-03-06T21:02:06.048Z 1 020-039 87 87 87 151 151 151 F F F
100972024-03-06T21:02:06.048Z 2 040-059 92 92 92 157 157 157 F F F
100982024-03-06T21:02:06.048Z 3 060-079 89 89 89 154 154 154 F F F
100992024-03-06T21:02:06.048Z 4 080-099 61 61 61 102 102 102 F F F
101002024-03-06T21:02:06.048Z 5 100-119 92 92 92 157 157 157 F F F
101012024-03-06T21:02:06.048Z 6 120-139 92 92 92 159 159 159 F F F
101022024-03-06T21:02:06.049Z 7 140-159 92 92 92 158 158 158 F F F
101032024-03-06T21:02:06.049Z 8 160-179 87 87 87 151 151 151 F F F
101042024-03-06T21:02:06.049Z 9 180-199 91 91 91 156 156 156 F F F
101052024-03-06T21:02:06.049Z 10 200-219 92 92 92 157 157 157 F F F
101062024-03-06T21:02:06.049Z 11 220-239 92 92 92 158 158 158 F F F
101072024-03-06T21:02:06.049Z 12 240-259 74 74 74 125 125 125 F F F
101082024-03-06T21:02:06.049Z 13 260-279 92 92 92 158 158 158 F F F
101092024-03-06T21:02:06.049Z 14 280-299 91 91 91 156 156 156 F F F
101102024-03-06T21:02:06.049Z 15 300-319 92 92 92 158 158 158 F F F
101112024-03-06T21:02:06.049Z 16 320-339 92 92 92 158 158 158 F F F
101122024-03-06T21:02:06.049Z 17 340-359 90 90 90 155 155 155 F F F
101132024-03-06T21:02:06.049Z 18 360-379 89 89 89 154 154 154 F F F
101142024-03-06T21:02:06.049Z 19 380-399 91 91 91 156 156 156 F F F
101152024-03-06T21:02:06.050Z 20 400-419 91 91 91 156 156 156 F F F
101162024-03-06T21:02:06.050Z 21 420-439 92 92 92 159 159 159 F F F
101172024-03-06T21:02:06.050Z 22 440-459 91 91 91 156 156 156 F F F
101182024-03-06T21:02:06.050Z 23 460-479 78 78 78 137 137 137 F F F
101192024-03-06T21:02:06.050Z 24 480-499 84 84 84 144 144 144 F F F
101202024-03-06T21:02:06.054Z 25 500-519 89 89 89 154 154 154 F F F
101212024-03-06T21:02:06.054Z 26 520-539 93 93 93 160 160 160 F F F
101222024-03-06T21:02:06.054Z 27 540-559 92 92 92 158 158 158 F F F
101232024-03-06T21:02:06.054Z 28 560-579 91 91 91 156 156 156 F F F
101242024-03-06T21:02:06.054Z 29 580-599 87 87 87 151 151 151 F F F
101252024-03-06T21:02:06.054ZMax gen: 93, Max flush: 160
101262024-03-06T21:02:06.054Z
101272024-03-06T21:02:06.054ZBegin loop 47
101282024-03-06T21:02:06.054ZDownstairs to restart: 1
101292024-03-06T21:02:06.075Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.830863155Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515}
101302024-03-06T21:02:06.075Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"Crucible runtime is spawned
101312024-03-06T21:02:06.081Zx86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.831214581Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515}
101322024-03-06T21:02:06.081Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.831251069Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515}
101332024-03-06T21:02:06.081Z{"msg":"Crucible f69ee2c1-0afa-49d2-a0e8-6d389d23f662 has session id: 85297754-7805-4b82-b8fc-e4fde5e31a3b","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.831279907Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101342024-03-06T21:02:06.081Z{"msg":"Upstairs opts: Upstairs UUID: f69ee2c1-0afa-49d2-a0e8-6d389d23f662, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.831338233Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101352024-03-06T21:02:06.081Z{"msg":"Crucible stats registered with UUID: f69ee2c1-0afa-49d2-a0e8-6d389d23f662","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.83138187Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101362024-03-06T21:02:06.081Z{"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.831422527Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515}
101372024-03-06T21:02:06.081Z{"msg":"f69ee2c1-0afa-49d2-a0e8-6d389d23f662 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.831491402Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101382024-03-06T21:02:06.081Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.831548508Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"client":"0","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101392024-03-06T21:02:06.081Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.831584816Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"client":"0","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101402024-03-06T21:02:06.081Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.831614854Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"client":"1","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101412024-03-06T21:02:06.081Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.83166544Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"client":"1","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101422024-03-06T21:02:06.081Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.831699838Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"client":"2","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101432024-03-06T21:02:06.081Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.831760304Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"client":"2","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101442024-03-06T21:02:06.081Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.831791991Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101452024-03-06T21:02:06.082Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.831834999Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"":"io task","client":"1","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101462024-03-06T21:02:06.082Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.831872456Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"":"io task","client":"0","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101472024-03-06T21:02:06.082Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.831901314Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"":"io task","client":"1","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101482024-03-06T21:02:06.082Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.831925002Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"":"io task","client":"0","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101492024-03-06T21:02:06.082Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.83196239Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101502024-03-06T21:02:06.082Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.832716108Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"client":"2","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101512024-03-06T21:02:06.082Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.832758455Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"client":"2","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101522024-03-06T21:02:06.082Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.832775194Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"client":"0","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101532024-03-06T21:02:06.082Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.832789533Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"client":"0","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101542024-03-06T21:02:06.082Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.833427089Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"client":"2","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101552024-03-06T21:02:06.082Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.833465726Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"client":"0","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101562024-03-06T21:02:06.082Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.833801193Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"client":"1","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101572024-03-06T21:02:06.082Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.83384796Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"client":"1","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101582024-03-06T21:02:06.082Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.834061585Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"client":"2","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101592024-03-06T21:02:06.082Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.834102562Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"client":"2","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101602024-03-06T21:02:06.082Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.834119211Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101612024-03-06T21:02:06.082Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.83413519Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"client":"0","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101622024-03-06T21:02:06.082Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.834159828Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"client":"0","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101632024-03-06T21:02:06.083Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.834175317Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101642024-03-06T21:02:06.083Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.834375324Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"client":"1","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101652024-03-06T21:02:06.083Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.834767557Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"client":"1","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101662024-03-06T21:02:06.083Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.834809374Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"client":"1","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101672024-03-06T21:02:06.083Z{"msg":"[0]R flush_numbers[0..12]: [156, 151, 157, 154, 102, 157, 159, 158, 151, 156, 157, 158]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.834838472Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"":"downstairs","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101682024-03-06T21:02:06.083Z{"msg":"[0]R generation[0..12]: [91, 87, 92, 89, 61, 92, 92, 92, 87, 91, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.834870469Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"":"downstairs","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101692024-03-06T21:02:06.083Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.834904487Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"":"downstairs","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101702024-03-06T21:02:06.083Z{"msg":"[1]R flush_numbers[0..12]: [156, 151, 157, 154, 102, 157, 159, 158, 151, 156, 157, 158]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.834929815Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"":"downstairs","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101712024-03-06T21:02:06.083Z{"msg":"[1]R generation[0..12]: [91, 87, 92, 89, 61, 92, 92, 92, 87, 91, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.834957153Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"":"downstairs","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101722024-03-06T21:02:06.083Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.834982132Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"":"downstairs","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101732024-03-06T21:02:06.083ZWait for a query_work_queue command to finish before sending IO
101742024-03-06T21:02:06.083Z{"msg":"[2]R flush_numbers[0..12]: [156, 151, 157, 154, 102, 157, 159, 158, 151, 156, 157, 158]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.835015699Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"":"downstairs","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101752024-03-06T21:02:06.083Z{"msg":"[2]R generation[0..12]: [91, 87, 92, 89, 61, 92, 92, 92, 87, 91, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.835040178Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"":"downstairs","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101762024-03-06T21:02:06.083Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.835073245Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"":"downstairs","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101772024-03-06T21:02:06.083Z{"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.835096494Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"":"downstairs","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101782024-03-06T21:02:06.083Z{"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.835125262Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"":"downstairs","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101792024-03-06T21:02:06.083Z{"msg":"Next flush: 161","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.8351487Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"":"downstairs","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101802024-03-06T21:02:06.083Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.835182248Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"":"downstairs","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101812024-03-06T21:02:06.087Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.835206506Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101822024-03-06T21:02:06.087Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.835233984Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"client":"0","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101832024-03-06T21:02:06.090Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.835257593Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"client":"1","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101842024-03-06T21:02:06.091Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.835284111Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"client":"2","":"downstairs","session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101852024-03-06T21:02:06.091Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.835307409Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101862024-03-06T21:02:06.091Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.835335437Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101872024-03-06T21:02:06.091Z{"msg":"f69ee2c1-0afa-49d2-a0e8-6d389d23f662 is now active with session: 85297754-7805-4b82-b8fc-e4fde5e31a3b","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.835358066Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101882024-03-06T21:02:06.091Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.835375055Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515,"session_id":"85297754-7805-4b82-b8fc-e4fde5e31a3b"}
101892024-03-06T21:02:06.091Z{"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:04.835395453Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515}
101902024-03-06T21:02:06.091ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
101912024-03-06T21:02:06.091ZLoading write count information from file "/tmp/test_repair_verify.data"
101922024-03-06T21:02:06.091ZRun Repair workload
101932024-03-06T21:02:06.091Z01/30 Write block 588 len 32768 data: 11 11 11 10 11 7 7 6
101942024-03-06T21:02:06.091Z02/30 Flush
101952024-03-06T21:02:06.091Z03/30 Write block 1 len 8192 data: 3 6
101962024-03-06T21:02:06.092Z04/30 Read block 56 len 24576
101972024-03-06T21:02:06.092Z05/30 Read block 372 len 20480
101982024-03-06T21:02:06.095Z06/30 Read block 378 len 36864
101992024-03-06T21:02:06.096Z07/30 Read block 143 len 16384
102002024-03-06T21:02:06.096Z08/30 Read block 424 len 32768
102012024-03-06T21:02:06.096Z09/30 Read block 491 len 16384
102022024-03-06T21:02:06.096Z10/30 Write block 397 len 20480 data: 7 7 4 5 4
102032024-03-06T21:02:06.096Z11/30 Write block 591 len 24576 data: 11 12 8 8 7 5
102042024-03-06T21:02:06.096Z12/30 Flush
102052024-03-06T21:02:06.100Z13/30 Write block 183 len 32768 data: 9 9 8 8 7 7 8 9
102062024-03-06T21:02:06.100Z14/30 Read block 182 len 16384
102072024-03-06T21:02:06.105Z15/30 Read block 218 len 36864
102082024-03-06T21:02:06.105Z16/30 Read block 180 len 28672
102092024-03-06T21:02:06.105Z17/30 Write block 436 len 8192 data: 8 10
102102024-03-06T21:02:06.105Z18/30 Read block 455 len 32768
102112024-03-06T21:02:06.108Z19/30 Write block 217 len 12288 data: 12 10 7
102122024-03-06T21:02:06.108Z20/30 Write block 303 len 12288 data: 5 6 8
102132024-03-06T21:02:06.108Z21/30 Write block 264 len 32768 data: 6 7 6 6 7 6 7 8
102142024-03-06T21:02:06.108Z22/30 Read block 203 len 24576
102152024-03-06T21:02:06.112Z23/30 Write block 356 len 24576 data: 11 9 9 8 10 13
102162024-03-06T21:02:06.112Z24/30 Read block 78 len 40960
102172024-03-06T21:02:06.112Z25/30 Write block 61 len 12288 data: 16 15 18
102182024-03-06T21:02:06.112Z26/30 Read block 260 len 12288
102192024-03-06T21:02:06.117Z27/30 Read block 321 len 28672
102202024-03-06T21:02:06.117Z28/30 Read block 321 len 4096
102212024-03-06T21:02:06.117Z29/30 Read block 529 len 20480
102222024-03-06T21:02:06.117Z30/30 Write block 244 len 24576 data: 7 6 4 4 4 4
102232024-03-06T21:02:06.117Z----------------------------------------------------------------
102242024-03-06T21:02:06.117Z Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:28
102252024-03-06T21:02:06.117ZStates: Active Active Active
102262024-03-06T21:02:06.117ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
102272024-03-06T21:02:06.117Z 3 Acked 1002 Write 2 Done Done Done false
102282024-03-06T21:02:06.117Z 4 Acked 1003 Read 6 Done Done Done false
102292024-03-06T21:02:06.117Z 5 Acked 1004 Read 5 Done Done Done false
102302024-03-06T21:02:06.117Z 6 Acked 1005 Read 9 Done Done Done false
102312024-03-06T21:02:06.117Z 7 Acked 1006 Read 4 Done Sent Done false
102322024-03-06T21:02:06.117Z 8 Acked 1007 Read 8 Done Sent Done false
102332024-03-06T21:02:06.117Z 9 Acked 1008 Read 4 Done Sent Done false
102342024-03-06T21:02:06.117Z 10 Acked 1009 Write 5 Done Sent Done false
102352024-03-06T21:02:06.117Z 11 Acked 1010 Write 6 Done Sent Done false
102362024-03-06T21:02:06.117Z 12 Acked 1011 Flush 0 Done Sent Done false
102372024-03-06T21:02:06.117Z 13 Acked 1012 Write 8 Done Sent Done false
102382024-03-06T21:02:06.118Z 14 Acked 1013 Read 4 Done Sent Done false
102392024-03-06T21:02:06.118Z 15 Acked 1014 Read 9 Done Sent Done false
102402024-03-06T21:02:06.118Z 16 Acked 1015 Read 7 Done Sent Done false
102412024-03-06T21:02:06.118Z 17 Acked 1016 Write 2 Done Sent Done false
102422024-03-06T21:02:06.118Z 18 Acked 1017 Read 8 Done Sent Done false
102432024-03-06T21:02:06.118Z 19 Acked 1018 Write 3 Done Sent Done false
102442024-03-06T21:02:06.118Z 20 Acked 1019 Write 3 Done Sent Done false
102452024-03-06T21:02:06.118Z 21 Acked 1020 Write 8 Done Sent Done false
102462024-03-06T21:02:06.118Z 22 Acked 1021 Read 6 Done Sent Done false
102472024-03-06T21:02:06.118Z 23 Acked 1022 Write 6 Done Sent Done false
102482024-03-06T21:02:06.118Z 24 Acked 1023 Read 10 Done Sent Done false
102492024-03-06T21:02:06.118Z 25 Acked 1024 Write 3 Done Sent Done false
102502024-03-06T21:02:06.118Z 26 Acked 1025 Read 3 Done Sent Done false
102512024-03-06T21:02:06.118Z 27 Acked 1026 Read 7 Done Sent Done false
102522024-03-06T21:02:06.118Z 28 Acked 1027 Read 1 Done Sent Done false
102532024-03-06T21:02:06.118Z 29 Acked 1028 Read 5 Done Sent Done false
102542024-03-06T21:02:06.118Z 30 Acked 1029 Write 6 Sent Sent Sent false
102552024-03-06T21:02:06.118Z STATES DS:0 DS:1 DS:2 TOTAL
102562024-03-06T21:02:06.118Z New 0 0 0 0
102572024-03-06T21:02:06.118Z Sent 1 24 1 26
102582024-03-06T21:02:06.118Z Done 27 4 27 58
102592024-03-06T21:02:06.118Z Skipped 0 0 0 0
102602024-03-06T21:02:06.118Z Error 0 0 0 0
102612024-03-06T21:02:06.119ZLast Flush: 1011 1001 1011
102622024-03-06T21:02:06.119ZDownstairs last five completed: 1001 1000
102632024-03-06T21:02:06.119ZUpstairs last five completed: 30 29 28 27 26
102642024-03-06T21:02:06.123Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:02:04.872414572Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1515}
102652024-03-06T21:02:06.123ZWrote out file "/tmp/test_repair_verify.data"
102662024-03-06T21:02:06.123Z
102672024-03-06T21:02:06.126ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
102682024-03-06T21:02:06.145ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
102692024-03-06T21:02:06.145Z 0 000-019 94 91 94 162 156 162 F T F <---
102702024-03-06T21:02:06.145Z 1 020-039 87 87 87 151 151 151 F F F
102712024-03-06T21:02:06.145Z 2 040-059 92 92 92 157 157 157 F F F
102722024-03-06T21:02:06.145Z 3 060-079 89 89 89 154 154 154 T F T <---
102732024-03-06T21:02:06.145Z 4 080-099 61 61 61 102 102 102 F F F
102742024-03-06T21:02:06.146Z 5 100-119 92 92 92 157 157 157 F F F
102752024-03-06T21:02:06.146Z 6 120-139 92 92 92 159 159 159 F F F
102762024-03-06T21:02:06.146Z 7 140-159 92 92 92 158 158 158 F F F
102772024-03-06T21:02:06.146Z 8 160-179 87 87 87 151 151 151 F F F
102782024-03-06T21:02:06.146Z 9 180-199 91 91 91 156 156 156 T F T <---
102792024-03-06T21:02:06.146Z 10 200-219 92 92 92 157 157 157 T F T <---
102802024-03-06T21:02:06.146Z 11 220-239 92 92 92 158 158 158 F F F
102812024-03-06T21:02:06.146Z 12 240-259 74 74 74 125 125 125 T F T <---
102822024-03-06T21:02:06.146Z 13 260-279 92 92 92 158 158 158 T F T <---
102832024-03-06T21:02:06.146Z 14 280-299 91 91 91 156 156 156 F F F
102842024-03-06T21:02:06.146Z 15 300-319 92 92 92 158 158 158 T F T <---
102852024-03-06T21:02:06.146Z 16 320-339 92 92 92 158 158 158 F F F
102862024-03-06T21:02:06.146Z 17 340-359 90 90 90 155 155 155 T F T <---
102872024-03-06T21:02:06.146Z 18 360-379 89 89 89 154 154 154 T F T <---
102882024-03-06T21:02:06.146Z 19 380-399 94 91 94 162 156 162 F F F <---
102892024-03-06T21:02:06.146Z 20 400-419 94 91 94 162 156 162 F F F <---
102902024-03-06T21:02:06.146Z 21 420-439 92 92 92 159 159 159 T F T <---
102912024-03-06T21:02:06.150Z 22 440-459 91 91 91 156 156 156 F F F
102922024-03-06T21:02:06.150Z 23 460-479 78 78 78 137 137 137 F F F
102932024-03-06T21:02:06.150Z 24 480-499 84 84 84 144 144 144 F F F
102942024-03-06T21:02:06.150Z 25 500-519 89 89 89 154 154 154 F F F
102952024-03-06T21:02:06.150Z 26 520-539 93 93 93 160 160 160 F F F
102962024-03-06T21:02:06.150Z 27 540-559 92 92 92 158 158 158 F F F
102972024-03-06T21:02:06.150Z 28 560-579 91 91 91 156 156 156 F F F
102982024-03-06T21:02:06.150Z 29 580-599 94 94 94 162 161 162 F F F <---
102992024-03-06T21:02:06.150ZMax gen: 94, Max flush: 162
103002024-03-06T21:02:06.150ZError: Difference in extent metadata found!
103012024-03-06T21:02:06.150ZA Difference in extent metadata is expected here
103022024-03-06T21:02:06.150ZOn loop 47
103032024-03-06T21:02:06.150Z
103042024-03-06T21:02:06.154ZVerifying data now
103052024-03-06T21:02:10.871ZNo change after verify
103062024-03-06T21:02:10.872ZLoop: 47 Downstairs dump after verify (and repair):
103072024-03-06T21:02:10.882ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
103082024-03-06T21:02:10.883Z 0 000-019 94 94 94 162 162 162 F F F
103092024-03-06T21:02:10.883Z 1 020-039 87 87 87 151 151 151 F F F
103102024-03-06T21:02:10.883Z 2 040-059 92 92 92 157 157 157 F F F
103112024-03-06T21:02:10.883Z 3 060-079 95 95 95 163 163 163 F F F
103122024-03-06T21:02:10.883Z 4 080-099 61 61 61 102 102 102 F F F
103132024-03-06T21:02:10.883Z 5 100-119 92 92 92 157 157 157 F F F
103142024-03-06T21:02:10.883Z 6 120-139 92 92 92 159 159 159 F F F
103152024-03-06T21:02:10.883Z 7 140-159 92 92 92 158 158 158 F F F
103162024-03-06T21:02:10.883Z 8 160-179 87 87 87 151 151 151 F F F
103172024-03-06T21:02:10.883Z 9 180-199 95 95 95 163 163 163 F F F
103182024-03-06T21:02:10.883Z 10 200-219 95 95 95 163 163 163 F F F
103192024-03-06T21:02:10.883Z 11 220-239 92 92 92 158 158 158 F F F
103202024-03-06T21:02:10.883Z 12 240-259 95 95 95 163 163 163 F F F
103212024-03-06T21:02:10.883Z 13 260-279 95 95 95 163 163 163 F F F
103222024-03-06T21:02:10.883Z 14 280-299 91 91 91 156 156 156 F F F
103232024-03-06T21:02:10.883Z 15 300-319 95 95 95 163 163 163 F F F
103242024-03-06T21:02:10.883Z 16 320-339 92 92 92 158 158 158 F F F
103252024-03-06T21:02:10.883Z 17 340-359 95 95 95 163 163 163 F F F
103262024-03-06T21:02:10.883Z 18 360-379 95 95 95 163 163 163 F F F
103272024-03-06T21:02:10.883Z 19 380-399 94 94 94 162 162 162 F F F
103282024-03-06T21:02:10.883Z 20 400-419 94 94 94 162 162 162 F F F
103292024-03-06T21:02:10.883Z 21 420-439 95 95 95 163 163 163 F F F
103302024-03-06T21:02:10.883Z 22 440-459 91 91 91 156 156 156 F F F
103312024-03-06T21:02:10.884Z 23 460-479 78 78 78 137 137 137 F F F
103322024-03-06T21:02:10.884Z 24 480-499 84 84 84 144 144 144 F F F
103332024-03-06T21:02:10.884Z 25 500-519 89 89 89 154 154 154 F F F
103342024-03-06T21:02:10.884Z 26 520-539 93 93 93 160 160 160 F F F
103352024-03-06T21:02:10.884Z 27 540-559 92 92 92 158 158 158 F F F
103362024-03-06T21:02:10.884Z 28 560-579 91 91 91 156 156 156 F F F
103372024-03-06T21:02:10.884Z 29 580-599 94 94 94 162 162 162 F F F
103382024-03-06T21:02:10.884ZMax gen: 95, Max flush: 163
103392024-03-06T21:02:10.884Z
103402024-03-06T21:02:10.884ZBegin loop 48
103412024-03-06T21:02:10.884ZDownstairs to restart: 1
103422024-03-06T21:02:10.888Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.643462845Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523}
103432024-03-06T21:02:10.888Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.643733237Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523}
103442024-03-06T21:02:10.888Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.643760475Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523}
103452024-03-06T21:02:10.888Z{"msg":"Crucible runtime is spawned
103462024-03-06T21:02:10.888ZCrucible d23ae2e5-ecf3-40ba-95fc-633f4498275f has session id: ad81b23c-97b4-458d-ba7f-c4dca73dca28","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.643805522Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103472024-03-06T21:02:10.888Z{"msg":"Upstairs opts: Upstairs UUID: d23ae2e5-ecf3-40ba-95fc-633f4498275f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.643853199Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103482024-03-06T21:02:10.888Z{"msg":"Crucible stats registered with UUID: d23ae2e5-ecf3-40ba-95fc-633f4498275f","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.643890886Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103492024-03-06T21:02:10.888Z{"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.643936373Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523}
103502024-03-06T21:02:10.894Z{"msg":"d23ae2e5-ecf3-40ba-95fc-633f4498275f active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.644098892Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103512024-03-06T21:02:10.894Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.644154868Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"client":"0","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103522024-03-06T21:02:10.894Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.644193335Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"client":"0","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103532024-03-06T21:02:10.894Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.644254491Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"client":"1","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103542024-03-06T21:02:10.894Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.644300108Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"client":"1","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103552024-03-06T21:02:10.894Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.644348475Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"client":"2","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103562024-03-06T21:02:10.894Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.644427769Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"client":"2","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103572024-03-06T21:02:10.894Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.644512323Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103582024-03-06T21:02:10.894Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.64456094Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103592024-03-06T21:02:10.894Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.644589818Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103602024-03-06T21:02:10.894Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.644621166Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103612024-03-06T21:02:10.894Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.644660333Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103622024-03-06T21:02:10.895Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.644696971Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103632024-03-06T21:02:10.895Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.645338336Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"client":"0","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103642024-03-06T21:02:10.895Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.645377424Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"client":"0","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103652024-03-06T21:02:10.895Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.645395272Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"client":"2","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103662024-03-06T21:02:10.895Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.6454304Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"client":"2","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103672024-03-06T21:02:10.895Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.645993711Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"client":"2","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103682024-03-06T21:02:10.895Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.646052687Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"client":"0","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103692024-03-06T21:02:10.895Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.646491017Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"client":"1","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103702024-03-06T21:02:10.895Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.646540633Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"client":"1","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103712024-03-06T21:02:10.895Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.646701812Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"client":"2","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103722024-03-06T21:02:10.895Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.64673857Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"client":"2","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103732024-03-06T21:02:10.895Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.646755039Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103742024-03-06T21:02:10.895Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.646775567Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"client":"0","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103752024-03-06T21:02:10.895Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.646789916Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"client":"0","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103762024-03-06T21:02:10.895Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.646814375Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103772024-03-06T21:02:10.895Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.647119524Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"client":"1","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103782024-03-06T21:02:10.895Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.647521366Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"client":"1","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103792024-03-06T21:02:10.895Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.647556294Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"client":"1","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103802024-03-06T21:02:10.895Z{"msg":"[0]R flush_numbers[0..12]: [162, 151, 157, 163, 102, 157, 159, 158, 151, 163, 163, 158]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.647589091Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"":"downstairs","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103812024-03-06T21:02:10.895Z{"msg":"[0]R generation[0..12]: [94, 87, 92, 95, 61, 92, 92, 92, 87, 95, 95, 92]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.64760604Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"":"downstairs","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103822024-03-06T21:02:10.895Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.647621929Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"":"downstairs","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103832024-03-06T21:02:10.895Z{"msg":"[1]R flush_numbers[0..12]: [162, 151, 157, 163, 102, 157, 159, 158, 151, 163, 163, 158]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.647643688Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"":"downstairs","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103842024-03-06T21:02:10.895Z{"msg":"[1]R generation[0..12]: [94, 87, 92, 95, 61, 92, 92, 92, 87, 95, 95, 92]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.647658687Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"":"downstairs","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103852024-03-06T21:02:10.896Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO
103862024-03-06T21:02:10.896Zlevel":30,"time":"2024-03-06T21:02:09.647682265Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"":"downstairs","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103872024-03-06T21:02:10.896Z{"msg":"[2]R flush_numbers[0..12]: [162, 151, 157, 163, 102, 157, 159, 158, 151, 163, 163, 158]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.647696664Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"":"downstairs","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103882024-03-06T21:02:10.896Z{"msg":"[2]R generation[0..12]: [94, 87, 92, 95, 61, 92, 92, 92, 87, 95, 95, 92]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.647716792Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"":"downstairs","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103892024-03-06T21:02:10.896Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.647731251Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"":"downstairs","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103902024-03-06T21:02:10.896Z{"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.64775063Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"":"downstairs","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103912024-03-06T21:02:10.896Z{"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.647764089Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"":"downstairs","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103922024-03-06T21:02:10.896Z{"msg":"Next flush: 164","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.647778008Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"":"downstairs","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103932024-03-06T21:02:10.896Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.647791497Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"":"downstairs","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103942024-03-06T21:02:10.896Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.647817165Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103952024-03-06T21:02:10.896Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.647839034Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"client":"0","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103962024-03-06T21:02:10.896Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.647876631Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"client":"1","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103972024-03-06T21:02:10.896Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.647925258Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"client":"2","":"downstairs","session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103982024-03-06T21:02:10.896Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.647942557Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
103992024-03-06T21:02:10.896Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.647966555Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
104002024-03-06T21:02:10.896Z{"msg":"d23ae2e5-ecf3-40ba-95fc-633f4498275f is now active with session: ad81b23c-97b4-458d-ba7f-c4dca73dca28","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.647999283Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
104012024-03-06T21:02:10.896Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.648024551Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523,"session_id":"ad81b23c-97b4-458d-ba7f-c4dca73dca28"}
104022024-03-06T21:02:10.901Z{"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:09.64804631Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523}
104032024-03-06T21:02:10.901ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
104042024-03-06T21:02:10.901ZLoading write count information from file "/tmp/test_repair_verify.data"
104052024-03-06T21:02:10.901ZRun Repair workload
104062024-03-06T21:02:10.901Z01/30 Write block 112 len 4096 data: 4
104072024-03-06T21:02:10.901Z02/30 Read block 588 len 8192
104082024-03-06T21:02:10.901Z03/30 Read block 279 len 40960
104092024-03-06T21:02:10.901Z04/30 Write block 504 len 36864 data: 6 6 6 7 6 7 9 8 8
104102024-03-06T21:02:10.901Z05/30 Read block 276 len 8192
104112024-03-06T21:02:10.901Z06/30 Read block 542 len 4096
104122024-03-06T21:02:10.901Z07/30 Read block 514 len 20480
104132024-03-06T21:02:10.901Z08/30 Write block 271 len 20480 data: 9 8 8 12 10
104142024-03-06T21:02:10.905Z09/30 Write block 355 len 16384 data: 8 12 10 10
104152024-03-06T21:02:10.905Z10/30 Write block 336 len 40960 data: 6 7 10 8 9 9 8 8 9 8
104162024-03-06T21:02:10.906Z11/30 Read block 208 len 28672
104172024-03-06T21:02:10.906Z12/30 Read block 400 len 36864
104182024-03-06T21:02:10.909Z13/30 Write block 194 len 16384 data: 7 7 10 10
104192024-03-06T21:02:10.909Z14/30 Read block 173 len 4096
104202024-03-06T21:02:10.909Z15/30 Write block 309 len 36864 data: 7 8 8 8 8 7 7 6 5
104212024-03-06T21:02:10.910Z16/30 Read block 206 len 40960
104222024-03-06T21:02:10.915Z17/30 Read block 389 len 4096
104232024-03-06T21:02:10.916Z18/30 Read block 372 len 24576
104242024-03-06T21:02:10.916Z19/30 Flush
104252024-03-06T21:02:10.916Z20/30 Write block 83 len 4096 data: 4
104262024-03-06T21:02:10.916Z21/30 Flush
104272024-03-06T21:02:10.920Z22/30 Write block 161 len 32768 data: 7 7 5 5 4 3 4 4
104282024-03-06T21:02:10.920Z23/30 Write block 128 len 4096 data: 11
104292024-03-06T21:02:10.920Z24/30 Write block 587 len 32768 data: 13 12 12 12 12 13 9 9
104302024-03-06T21:02:10.920Z25/30 Read block 324 len 28672
104312024-03-06T21:02:10.926Z26/30 Flush
104322024-03-06T21:02:10.926Z27/30 Write block 345 len 24576 data: 9 7 8 11 10 10
104332024-03-06T21:02:10.926Z28/30 Read block 242 len 28672
104342024-03-06T21:02:10.930Z29/30 Write block 581 len 24576 data: 5 5 6 9 10 12
104352024-03-06T21:02:10.931Z30/30 Write block 4 len 12288 data: 8 9 8
104362024-03-06T21:02:10.931Z----------------------------------------------------------------
104372024-03-06T21:02:10.931Z Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:30
104382024-03-06T21:02:10.931ZStates: Active Active Active
104392024-03-06T21:02:10.931ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
104402024-03-06T21:02:10.931Z 1 Acked 1000 Write 1 Done Sent Done false
104412024-03-06T21:02:10.931Z 2 Acked 1001 Read 2 Done Sent Done false
104422024-03-06T21:02:10.931Z 3 Acked 1002 Read 10 Done Sent Done false
104432024-03-06T21:02:10.931Z 4 Acked 1003 Write 9 Done Sent Done false
104442024-03-06T21:02:10.931Z 5 Acked 1004 Read 2 Done Sent Done false
104452024-03-06T21:02:10.931Z 6 Acked 1005 Read 1 Done Sent Done false
104462024-03-06T21:02:10.931Z 7 Acked 1006 Read 5 Done Sent Done false
104472024-03-06T21:02:10.932Z 8 Acked 1007 Write 5 Done Sent Done false
104482024-03-06T21:02:10.932Z 9 Acked 1008 Write 4 Done Sent Done false
104492024-03-06T21:02:10.932Z 10 Acked 1009 Write 10 Done Sent Done false
104502024-03-06T21:02:10.932Z 11 Acked 1010 Read 7 Done Sent Done false
104512024-03-06T21:02:10.932Z 12 Acked 1011 Read 9 Done Sent Done false
104522024-03-06T21:02:10.932Z 13 Acked 1012 Write 4 Done Sent Done false
104532024-03-06T21:02:10.932Z 14 Acked 1013 Read 1 Done Sent Done false
104542024-03-06T21:02:10.932Z 15 Acked 1014 Write 9 Done Sent Done false
104552024-03-06T21:02:10.932Z 16 Acked 1015 Read 10 Done Sent Done false
104562024-03-06T21:02:10.932Z 17 Acked 1016 Read 1 Done Sent Done false
104572024-03-06T21:02:10.932Z 18 Acked 1017 Read 6 Done Sent Done false
104582024-03-06T21:02:10.932Z 19 Acked 1018 Flush 0 Done Sent Done false
104592024-03-06T21:02:10.933Z 20 Acked 1019 Write 1 Done Sent Done false
104602024-03-06T21:02:10.933Z 21 Acked 1020 Flush 0 Done Sent Done false
104612024-03-06T21:02:10.933Z 22 Acked 1021 Write 8 Done Sent Done false
104622024-03-06T21:02:10.933Z 23 Acked 1022 Write 1 Done Sent Done false
104632024-03-06T21:02:10.933Z 24 Acked 1023 Write 8 Done Sent Done false
104642024-03-06T21:02:10.933Z 25 Acked 1024 Read 7 Done Sent Done false
104652024-03-06T21:02:10.933Z 26 Acked 1025 Flush 0 Done Sent Done false
104662024-03-06T21:02:10.933Z 27 Acked 1026 Write 6 Done Sent Done false
104672024-03-06T21:02:10.933Z 28 Acked 1027 Read 7 Done Sent Done false
104682024-03-06T21:02:10.933Z 29 Acked 1028 Write 6 Sent Sent Done false
104692024-03-06T21:02:10.933Z 30 Acked 1029 Write 3 Sent Sent Sent false
104702024-03-06T21:02:10.933Z STATES DS:0 DS:1 DS:2 TOTAL
104712024-03-06T21:02:10.933Z New 0 0 0 0
104722024-03-06T21:02:10.934Z Sent 2 30 1 33
104732024-03-06T21:02:10.934Z Done 28 0 29 57
104742024-03-06T21:02:10.934Z Skipped 0 0 0 0
104752024-03-06T21:02:10.934Z Error 0 0 0 0
104762024-03-06T21:02:10.934ZLast Flush: 1025 0 1025
104772024-03-06T21:02:10.934ZDownstairs last five completed:
104782024-03-06T21:02:10.934ZUpstairs last five completed: 30 29 28 27 26
104792024-03-06T21:02:10.934Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:02:09.683367695Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1523}
104802024-03-06T21:02:10.934ZWrote out file "/tmp/test_repair_verify.data"
104812024-03-06T21:02:10.950Z
104822024-03-06T21:02:10.950ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
104832024-03-06T21:02:10.955ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
104842024-03-06T21:02:10.955Z 0 000-019 94 94 94 162 162 162 T F T <---
104852024-03-06T21:02:10.955Z 1 020-039 87 87 87 151 151 151 F F F
104862024-03-06T21:02:10.955Z 2 040-059 92 92 92 157 157 157 F F F
104872024-03-06T21:02:10.955Z 3 060-079 95 95 95 163 163 163 F F F
104882024-03-06T21:02:10.955Z 4 080-099 96 61 96 165 102 165 F F F <---
104892024-03-06T21:02:10.955Z 5 100-119 96 92 96 164 157 164 F F F <---
104902024-03-06T21:02:10.955Z 6 120-139 96 92 96 166 159 166 F F F <---
104912024-03-06T21:02:10.955Z 7 140-159 92 92 92 158 158 158 F F F
104922024-03-06T21:02:10.956Z 8 160-179 96 87 96 166 151 166 F F F <---
104932024-03-06T21:02:10.956Z 9 180-199 96 95 96 164 163 164 F F F <---
104942024-03-06T21:02:10.956Z 10 200-219 95 95 95 163 163 163 F F F
104952024-03-06T21:02:10.956Z 11 220-239 92 92 92 158 158 158 F F F
104962024-03-06T21:02:10.956Z 12 240-259 95 95 95 163 163 163 F F F
104972024-03-06T21:02:10.956Z 13 260-279 96 95 96 164 163 164 F F F <---
104982024-03-06T21:02:10.956Z 14 280-299 91 91 91 156 156 156 F F F
104992024-03-06T21:02:10.956Z 15 300-319 96 95 96 164 163 164 F F F <---
105002024-03-06T21:02:10.956Z 16 320-339 96 92 96 164 158 164 F F F <---
105012024-03-06T21:02:10.956Z 17 340-359 96 95 96 164 163 164 T F T <---
105022024-03-06T21:02:10.956Z 18 360-379 95 95 95 163 163 163 F F F
105032024-03-06T21:02:10.956Z 19 380-399 94 94 94 162 162 162 F F F
105042024-03-06T21:02:10.956Z 20 400-419 94 94 94 162 162 162 F F F
105052024-03-06T21:02:10.956Z 21 420-439 95 95 95 163 163 163 F F F
105062024-03-06T21:02:10.956Z 22 440-459 91 91 91 156 156 156 F F F
105072024-03-06T21:02:10.956Z 23 460-479 78 78 78 137 137 137 F F F
105082024-03-06T21:02:10.956Z 24 480-499 84 84 84 144 144 144 F F F
105092024-03-06T21:02:10.956Z 25 500-519 96 89 96 164 154 164 F F F <---
105102024-03-06T21:02:10.956Z 26 520-539 93 93 93 160 160 160 F F F
105112024-03-06T21:02:10.956Z 27 540-559 92 92 92 158 158 158 F F F
105122024-03-06T21:02:10.956Z 28 560-579 91 91 91 156 156 156 F F F
105132024-03-06T21:02:10.957Z 29 580-599 96 94 96 166 162 166 T F T <---
105142024-03-06T21:02:10.957ZMax gen: 96, Max flush: 166
105152024-03-06T21:02:10.966ZError: Difference in extent metadata found!
105162024-03-06T21:02:10.966ZA Difference in extent metadata is expected here
105172024-03-06T21:02:10.966ZOn loop 48
105182024-03-06T21:02:10.966Z
105192024-03-06T21:02:10.971ZVerifying data now
105202024-03-06T21:02:15.608ZNo change after verify
105212024-03-06T21:02:15.608ZLoop: 48 Downstairs dump after verify (and repair):
105222024-03-06T21:02:15.614ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
105232024-03-06T21:02:15.615Z 0 000-019 97 97 97 167 167 167 F F F
105242024-03-06T21:02:15.615Z 1 020-039 87 87 87 151 151 151 F F F
105252024-03-06T21:02:15.615Z 2 040-059 92 92 92 157 157 157 F F F
105262024-03-06T21:02:15.615Z 3 060-079 95 95 95 163 163 163 F F F
105272024-03-06T21:02:15.616Z 4 080-099 96 96 96 165 165 165 F F F
105282024-03-06T21:02:15.616Z 5 100-119 96 96 96 164 164 164 F F F
105292024-03-06T21:02:15.616Z 6 120-139 96 96 96 166 166 166 F F F
105302024-03-06T21:02:15.616Z 7 140-159 92 92 92 158 158 158 F F F
105312024-03-06T21:02:15.616Z 8 160-179 96 96 96 166 166 166 F F F
105322024-03-06T21:02:15.616Z 9 180-199 96 96 96 164 164 164 F F F
105332024-03-06T21:02:15.617Z 10 200-219 95 95 95 163 163 163 F F F
105342024-03-06T21:02:15.617Z 11 220-239 92 92 92 158 158 158 F F F
105352024-03-06T21:02:15.617Z 12 240-259 95 95 95 163 163 163 F F F
105362024-03-06T21:02:15.617Z 13 260-279 96 96 96 164 164 164 F F F
105372024-03-06T21:02:15.633Z 14 280-299 91 91 91 156 156 156 F F F
105382024-03-06T21:02:15.633Z 15 300-319 96 96 96 164 164 164 F F F
105392024-03-06T21:02:15.634Z 16 320-339 96 96 96 164 164 164 F F F
105402024-03-06T21:02:15.634Z 17 340-359 97 97 97 167 167 167 F F F
105412024-03-06T21:02:15.634Z 18 360-379 95 95 95 163 163 163 F F F
105422024-03-06T21:02:15.634Z 19 380-399 94 94 94 162 162 162 F F F
105432024-03-06T21:02:15.634Z 20 400-419 94 94 94 162 162 162 F F F
105442024-03-06T21:02:15.634Z 21 420-439 95 95 95 163 163 163 F F F
105452024-03-06T21:02:15.635Z 22 440-459 91 91 91 156 156 156 F F F
105462024-03-06T21:02:15.635Z 23 460-479 78 78 78 137 137 137 F F F
105472024-03-06T21:02:15.635Z 24 480-499 84 84 84 144 144 144 F F F
105482024-03-06T21:02:15.635Z 25 500-519 96 96 96 164 164 164 F F F
105492024-03-06T21:02:15.635Z 26 520-539 93 93 93 160 160 160 F F F
105502024-03-06T21:02:15.635Z 27 540-559 92 92 92 158 158 158 F F F
105512024-03-06T21:02:15.635Z 28 560-579 91 91 91 156 156 156 F F F
105522024-03-06T21:02:15.636Z 29 580-599 97 97 97 167 167 167 F F F
105532024-03-06T21:02:15.636ZMax gen: 97, Max flush: 167
105542024-03-06T21:02:15.636Z
105552024-03-06T21:02:15.636ZBegin loop 49
105562024-03-06T21:02:15.636ZDownstairs to restart: 0
105572024-03-06T21:02:16.052Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.395117246Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531}
105582024-03-06T21:02:16.052Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.395410186Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531}
105592024-03-06T21:02:16.052Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.395428035Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531}
105602024-03-06T21:02:16.052Z{"msg":"Crucible 9a0495cc-ab1a-4d9e-964f-c227341e3b7b has session id: c5a67edd-444f-45cd-9846-85241127c207","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.395442874Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105612024-03-06T21:02:16.052Z{"msg":"Upstairs opts: Upstairs UUID: 9a0495cc-ab1a-4d9e-964f-c227341e3b7b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","levelCrucible runtime is spawned
105622024-03-06T21:02:16.052Z":30,"time":"2024-03-06T21:02:14.39549804Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105632024-03-06T21:02:16.052Z{"msg":"Crucible stats registered with UUID: 9a0495cc-ab1a-4d9e-964f-c227341e3b7b","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.395538687Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105642024-03-06T21:02:16.052Z{"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.395598223Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531}
105652024-03-06T21:02:16.052Z{"msg":"9a0495cc-ab1a-4d9e-964f-c227341e3b7b active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.395745563Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105662024-03-06T21:02:16.052Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.39578387Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"client":"0","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105672024-03-06T21:02:16.052Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.395802119Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"client":"0","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105682024-03-06T21:02:16.052Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.395816778Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"client":"1","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105692024-03-06T21:02:16.052Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.395834277Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"client":"1","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105702024-03-06T21:02:16.052Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.395870024Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"client":"2","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105712024-03-06T21:02:16.052Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.395888163Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"client":"2","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105722024-03-06T21:02:16.052Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.395909461Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"":"io task","client":"0","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105732024-03-06T21:02:16.053Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.395946369Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"":"io task","client":"1","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105742024-03-06T21:02:16.053Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.39607897Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"":"io task","client":"2","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105752024-03-06T21:02:16.053Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.396212671Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"":"io task","client":"0","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105762024-03-06T21:02:16.053Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.396280996Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"":"io task","client":"2","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105772024-03-06T21:02:16.053Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.396328783Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"":"io task","client":"1","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105782024-03-06T21:02:16.053Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.39708692Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"client":"2","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105792024-03-06T21:02:16.053Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.397122138Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"client":"2","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105802024-03-06T21:02:16.053Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.397161435Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"client":"1","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105812024-03-06T21:02:16.053Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.397215511Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"client":"1","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105822024-03-06T21:02:16.053Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.397836099Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"client":"2","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105832024-03-06T21:02:16.053Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.397899784Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"client":"1","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105842024-03-06T21:02:16.053Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.398368472Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"client":"0","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105852024-03-06T21:02:16.053Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.398414539Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"client":"0","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105862024-03-06T21:02:16.053Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.398580197Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"client":"2","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105872024-03-06T21:02:16.053Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.398612095Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"client":"2","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105882024-03-06T21:02:16.053Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.398627984Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105892024-03-06T21:02:16.053Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.398641383Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"client":"1","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105902024-03-06T21:02:16.053Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.398670601Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"client":"1","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105912024-03-06T21:02:16.053Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.39869344Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105922024-03-06T21:02:16.053Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.399000799Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"client":"0","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105932024-03-06T21:02:16.053Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.399443538Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"client":"0","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105942024-03-06T21:02:16.053Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.399481775Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"client":"0","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105952024-03-06T21:02:16.053Z{"msg":"[0]R flush_numbers[0..12]: [167, 151, 157, 163, 165, 164, 166, 158, 166, 164, 163, 158]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.399496934Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"":"downstairs","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105962024-03-06T21:02:16.053Z{"msg":"[0]R generation[0..12]: [97, 87, 92, 95, 96, 96, 96, 92, 96, 96, 95, 92]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.399596107Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"":"downstairs","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105972024-03-06T21:02:16.054Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.399615256Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"":"downstairs","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105982024-03-06T21:02:16.054Z{"msg":"[1]R flush_numbers[0..12]: [167, 151, 157, 163, 165, 164, 166, 158, 166, 164, 163, 158]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.399630485Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"":"downstairs","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
105992024-03-06T21:02:16.054Z{"msg":"[1]R generation[0..12]: [97, 87, 92, 95, 96, 96, 96, 92, 96, 96, 95, 92]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
106002024-03-06T21:02:16.054Z,"time":"2024-03-06T21:02:14.399653023Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"":"downstairs","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
106012024-03-06T21:02:16.054Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.399670462Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"":"downstairs","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
106022024-03-06T21:02:16.054Z{"msg":"[2]R flush_numbers[0..12]: [167, 151, 157, 163, 165, 164, 166, 158, 166, 164, 163, 158]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.399685301Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"":"downstairs","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
106032024-03-06T21:02:16.054Z{"msg":"[2]R generation[0..12]: [97, 87, 92, 95, 96, 96, 96, 92, 96, 96, 95, 92]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.3997065Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"":"downstairs","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
106042024-03-06T21:02:16.054Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.399736768Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"":"downstairs","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
106052024-03-06T21:02:16.054Z{"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.399752217Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"":"downstairs","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
106062024-03-06T21:02:16.054Z{"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.399773155Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"":"downstairs","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
106072024-03-06T21:02:16.054Z{"msg":"Next flush: 168","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.399787014Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"":"downstairs","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
106082024-03-06T21:02:16.054Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.399806963Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"":"downstairs","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
106092024-03-06T21:02:16.054Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.399827922Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
106102024-03-06T21:02:16.054Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.39984143Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"client":"0","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
106112024-03-06T21:02:16.054Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.399862509Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"client":"1","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
106122024-03-06T21:02:16.054Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.399876238Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"client":"2","":"downstairs","session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
106132024-03-06T21:02:16.054Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.399890587Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
106142024-03-06T21:02:16.054Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.399918315Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
106152024-03-06T21:02:16.054Z{"msg":"9a0495cc-ab1a-4d9e-964f-c227341e3b7b is now active with session: c5a67edd-444f-45cd-9846-85241127c207","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.399936344Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
106162024-03-06T21:02:16.054Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.399957393Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531,"session_id":"c5a67edd-444f-45cd-9846-85241127c207"}
106172024-03-06T21:02:16.054Z{"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:14.399971672Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531}
106182024-03-06T21:02:16.054ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
106192024-03-06T21:02:16.054ZLoading write count information from file "/tmp/test_repair_verify.data"
106202024-03-06T21:02:16.055ZRun Repair workload
106212024-03-06T21:02:16.055Z01/30 Write block 182 len 16384 data: 11 10 10 9
106222024-03-06T21:02:16.055Z02/30 Read block 93 len 36864
106232024-03-06T21:02:16.055Z03/30 Read block 487 len 36864
106242024-03-06T21:02:16.055Z04/30 Write block 269 len 8192 data: 7 8
106252024-03-06T21:02:16.055Z05/30 Read block 482 len 24576
106262024-03-06T21:02:16.055Z06/30 Write block 26 len 16384 data: 10 9 7 7
106272024-03-06T21:02:16.055Z07/30 Flush
106282024-03-06T21:02:16.055Z08/30 Write block 221 len 4096 data: 5
106292024-03-06T21:02:16.055Z09/30 Read block 451 len 8192
106302024-03-06T21:02:16.055Z10/30 Write block 148 len 20480 data: 7 8 10 9 7
106312024-03-06T21:02:16.055Z11/30 Read block 504 len 40960
106322024-03-06T21:02:16.055Z12/30 Write block 206 len 12288 data: 6 3 6
106332024-03-06T21:02:16.055Z13/30 Write block 540 len 12288 data: 10 8 11
106342024-03-06T21:02:16.055Z14/30 Flush
106352024-03-06T21:02:16.055Z15/30 Write block 591 len 8192 data: 13 14
106362024-03-06T21:02:16.055Z16/30 Write block 427 len 8192 data: 10 10
106372024-03-06T21:02:16.055Z17/30 Write block 199 len 32768 data: 9 9 8 6 5 5 5 7
106382024-03-06T21:02:16.055Z18/30 Write block 427 len 4096 data: 11
106392024-03-06T21:02:16.055Z19/30 Write block 528 len 8192 data: 8 9
106402024-03-06T21:02:16.055Z20/30 Write block 38 len 28672 data: 6 7 7 9 10 9 8
106412024-03-06T21:02:16.055Z21/30 Read block 322 len 20480
106422024-03-06T21:02:16.055Z22/30 Read block 356 len 4096
106432024-03-06T21:02:16.055Z23/30 Write block 560 len 20480 data: 8 8 8 7 8
106442024-03-06T21:02:16.056Z24/30 Read block 296 len 32768
106452024-03-06T21:02:16.056Z25/30 Read block 508 len 28672
106462024-03-06T21:02:16.056Z26/30 Read block 195 len 20480
106472024-03-06T21:02:16.056Z27/30 Write block 590 len 8192 data: 13 14
106482024-03-06T21:02:16.056Z28/30 Write block 386 len 28672 data: 10 8 8 8 10 9 8
106492024-03-06T21:02:16.056Z29/30 Write block 175 len 28672 data: 8 7 7 6 9 10 12
106502024-03-06T21:02:16.056Z30/30 Write block 442 len 36864 data: 6 5 7 6 7 5 9 8 9
106512024-03-06T21:02:16.056Z----------------------------------------------------------------
106522024-03-06T21:02:16.056Z Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:23
106532024-03-06T21:02:16.056ZStates: Active Active Active
106542024-03-06T21:02:16.056ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
106552024-03-06T21:02:16.056Z 8 Acked 1007 Write 1 Done Done Done false
106562024-03-06T21:02:16.056Z 9 Acked 1008 Read 2 Done Done Done false
106572024-03-06T21:02:16.087Z 10 Acked 1009 Write 5 Sent Done Done false
106582024-03-06T21:02:16.088Z 11 Acked 1010 Read 10 Sent Done Done false
106592024-03-06T21:02:16.088Z 12 Acked 1011 Write 3 Sent Done Done false
106602024-03-06T21:02:16.088Z 13 Acked 1012 Write 3 Sent Done Done false
106612024-03-06T21:02:16.088Z 14 Acked 1013 Flush 0 Sent Done Done false
106622024-03-06T21:02:16.089Z 15 Acked 1014 Write 2 Sent Done Done false
106632024-03-06T21:02:16.089Z 16 Acked 1015 Write 2 Sent Done Done false
106642024-03-06T21:02:16.089Z 17 Acked 1016 Write 8 Sent Done Done false
106652024-03-06T21:02:16.089Z 18 Acked 1017 Write 1 Sent Done Done false
106662024-03-06T21:02:16.089Z 19 Acked 1018 Write 2 Sent Done Done false
106672024-03-06T21:02:16.089Z 20 Acked 1019 Write 7 Sent Done Done false
106682024-03-06T21:02:16.089Z 21 Acked 1020 Read 5 Sent Done Done false
106692024-03-06T21:02:16.089Z 22 Acked 1021 Read 1 Sent Done Done false
106702024-03-06T21:02:16.089Z 23 Acked 1022 Write 5 Sent Done Done false
106712024-03-06T21:02:16.089Z 24 Acked 1023 Read 8 Sent Done Done false
106722024-03-06T21:02:16.090Z 25 Acked 1024 Read 7 Sent Done Done false
106732024-03-06T21:02:16.090Z 26 Acked 1025 Read 5 Sent Done Done false
106742024-03-06T21:02:16.090Z 27 Acked 1026 Write 2 Sent Done Done false
106752024-03-06T21:02:16.090Z 28 Acked 1027 Write 7 Sent Done Done false
106762024-03-06T21:02:16.090Z 29 Acked 1028 Write 7 Sent Done Sent false
106772024-03-06T21:02:16.090Z 30 Acked 1029 Write 9 Sent Sent Sent false
106782024-03-06T21:02:16.090Z STATES DS:0 DS:1 DS:2 TOTAL
106792024-03-06T21:02:16.090Z New 0 0 0 0
106802024-03-06T21:02:16.090Z Sent 21 1 2 24
106812024-03-06T21:02:16.090Z Done 2 22 21 45
106822024-03-06T21:02:16.090Z Skipped 0 0 0 0
106832024-03-06T21:02:16.090Z Error 0 0 0 0
106842024-03-06T21:02:16.090ZLast Flush: 1006 1013 1013
106852024-03-06T21:02:16.091ZDownstairs last five completed: 1006 1005 1004 1003 1002
106862024-03-06T21:02:16.091ZUpstairs last five completed: 30 29 28 27 26
106872024-03-06T21:02:16.091Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:02:14.43250457Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1531}
106882024-03-06T21:02:16.091ZWrote out file "/tmp/test_repair_verify.data"
106892024-03-06T21:02:16.091Z
106902024-03-06T21:02:16.091ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
106912024-03-06T21:02:16.091ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
106922024-03-06T21:02:16.091Z 0 000-019 97 97 97 167 167 167 F F F
106932024-03-06T21:02:16.091Z 1 020-039 98 98 98 168 168 168 F T T <---
106942024-03-06T21:02:16.091Z 2 040-059 92 92 92 157 157 157 F T T <---
106952024-03-06T21:02:16.091Z 3 060-079 95 95 95 163 163 163 F F F
106962024-03-06T21:02:16.092Z 4 080-099 96 96 96 165 165 165 F F F
106972024-03-06T21:02:16.092Z 5 100-119 96 96 96 164 164 164 F F F
106982024-03-06T21:02:16.092Z 6 120-139 96 96 96 166 166 166 F F F
106992024-03-06T21:02:16.092Z 7 140-159 92 98 98 158 169 169 F F F <---
107002024-03-06T21:02:16.092Z 8 160-179 96 96 96 166 166 166 F T T <---
107012024-03-06T21:02:16.092Z 9 180-199 98 98 98 168 168 168 F T T <---
107022024-03-06T21:02:16.092Z 10 200-219 95 98 98 163 169 169 F T T <---
107032024-03-06T21:02:16.093Z 11 220-239 92 98 98 158 169 169 T F F <---
107042024-03-06T21:02:16.093Z 12 240-259 95 95 95 163 163 163 F F F
107052024-03-06T21:02:16.093Z 13 260-279 98 98 98 168 168 168 F F F
107062024-03-06T21:02:16.093Z 14 280-299 91 91 91 156 156 156 F F F
107072024-03-06T21:02:16.093Z 15 300-319 96 96 96 164 164 164 F F F
107082024-03-06T21:02:16.093Z 16 320-339 96 96 96 164 164 164 F F F
107092024-03-06T21:02:16.093Z 17 340-359 97 97 97 167 167 167 F F F
107102024-03-06T21:02:16.093Z 18 360-379 95 95 95 163 163 163 F F F
107112024-03-06T21:02:16.094Z 19 380-399 94 94 94 162 162 162 F T T <---
107122024-03-06T21:02:16.094Z 20 400-419 94 94 94 162 162 162 F F F
107132024-03-06T21:02:16.094Z 21 420-439 95 95 95 163 163 163 F T T <---
107142024-03-06T21:02:16.094Z 22 440-459 91 91 91 156 156 156 F T F <---
107152024-03-06T21:02:16.094Z 23 460-479 78 78 78 137 137 137 F F F
107162024-03-06T21:02:16.094Z 24 480-499 84 84 84 144 144 144 F F F
107172024-03-06T21:02:16.094Z 25 500-519 96 96 96 164 164 164 F F F
107182024-03-06T21:02:16.094Z 26 520-539 93 93 93 160 160 160 F T T <---
107192024-03-06T21:02:16.094Z 27 540-559 92 98 98 158 169 169 F F F <---
107202024-03-06T21:02:16.094Z 28 560-579 91 91 91 156 156 156 F T T <---
107212024-03-06T21:02:16.094Z 29 580-599 97 97 97 167 167 167 F T T <---
107222024-03-06T21:02:16.094ZMax gen: 98, Max flush: 169
107232024-03-06T21:02:16.094ZError: Difference in extent metadata found!
107242024-03-06T21:02:16.094ZA Difference in extent metadata is expected here
107252024-03-06T21:02:16.094ZOn loop 49
107262024-03-06T21:02:16.095Z
107272024-03-06T21:02:16.095ZVerifying data now
107282024-03-06T21:02:20.631ZNo change after verify
107292024-03-06T21:02:20.631ZLoop: 49 Downstairs dump after verify (and repair):
107302024-03-06T21:02:20.671ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
107312024-03-06T21:02:20.671Z 0 000-019 97 97 97 167 167 167 F F F
107322024-03-06T21:02:20.671Z 1 020-039 99 99 99 170 170 170 F F F
107332024-03-06T21:02:20.671Z 2 040-059 99 99 99 170 170 170 F F F
107342024-03-06T21:02:20.671Z 3 060-079 95 95 95 163 163 163 F F F
107352024-03-06T21:02:20.671Z 4 080-099 96 96 96 165 165 165 F F F
107362024-03-06T21:02:20.671Z 5 100-119 96 96 96 164 164 164 F F F
107372024-03-06T21:02:20.671Z 6 120-139 96 96 96 166 166 166 F F F
107382024-03-06T21:02:20.672Z 7 140-159 98 98 98 169 169 169 F F F
107392024-03-06T21:02:20.672Z 8 160-179 99 99 99 170 170 170 F F F
107402024-03-06T21:02:20.672Z 9 180-199 99 99 99 170 170 170 F F F
107412024-03-06T21:02:20.672Z 10 200-219 99 99 99 170 170 170 F F F
107422024-03-06T21:02:20.672Z 11 220-239 98 98 98 169 169 169 F F F
107432024-03-06T21:02:20.672Z 12 240-259 95 95 95 163 163 163 F F F
107442024-03-06T21:02:20.672Z 13 260-279 98 98 98 168 168 168 F F F
107452024-03-06T21:02:20.672Z 14 280-299 91 91 91 156 156 156 F F F
107462024-03-06T21:02:20.672Z 15 300-319 96 96 96 164 164 164 F F F
107472024-03-06T21:02:20.672Z 16 320-339 96 96 96 164 164 164 F F F
107482024-03-06T21:02:20.672Z 17 340-359 97 97 97 167 167 167 F F F
107492024-03-06T21:02:20.672Z 18 360-379 95 95 95 163 163 163 F F F
107502024-03-06T21:02:20.672Z 19 380-399 99 99 99 170 170 170 F F F
107512024-03-06T21:02:20.672Z 20 400-419 94 94 94 162 162 162 F F F
107522024-03-06T21:02:20.672Z 21 420-439 99 99 99 170 170 170 F F F
107532024-03-06T21:02:20.672Z 22 440-459 99 99 99 170 170 170 F F F
107542024-03-06T21:02:20.672Z 23 460-479 78 78 78 137 137 137 F F F
107552024-03-06T21:02:20.672Z 24 480-499 84 84 84 144 144 144 F F F
107562024-03-06T21:02:20.672Z 25 500-519 96 96 96 164 164 164 F F F
107572024-03-06T21:02:20.672Z 26 520-539 99 99 99 170 170 170 F F F
107582024-03-06T21:02:20.672Z 27 540-559 98 98 98 169 169 169 F F F
107592024-03-06T21:02:20.672Z 28 560-579 99 99 99 170 170 170 F F F
107602024-03-06T21:02:20.673Z 29 580-599 99 99 99 170 170 170 F F F
107612024-03-06T21:02:20.673ZMax gen: 99, Max flush: 170
107622024-03-06T21:02:20.673Z
107632024-03-06T21:02:20.673ZBegin loop 50
107642024-03-06T21:02:20.673ZDownstairs to restart: 2
107652024-03-06T21:02:20.673Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.253399429Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539}
107662024-03-06T21:02:20.673Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.25367424Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539}
107672024-03-06T21:02:20.673Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.253691949Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539}
107682024-03-06T21:02:20.673Z{"msg":"Crucible 641ddd42-be2c-49a1-b59f-eac69bb34644 has session id: 9dc465f4-b2da-4e81-839d-88b47b6619c0","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.253707118Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
107692024-03-06T21:02:20.673Z{"msg":"Upstairs opts: Upstairs UUID: 641ddd42-be2c-49a1-b59f-eac69bb34644, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.253754605Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
107702024-03-06T21:02:20.673Z{"msg":"Crucible stats registered with UUID: 641ddd42-be2c-49a1-b59f-eac69bb34644","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.253780773Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
107712024-03-06T21:02:20.673ZCrucible runtime is spawned
107722024-03-06T21:02:20.673Z{"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.253903764Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539}
107732024-03-06T21:02:20.673Z{"msg":"641ddd42-be2c-49a1-b59f-eac69bb34644 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.254005617Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
107742024-03-06T21:02:20.673Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.254031315Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"client":"0","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
107752024-03-06T21:02:20.673Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.254046924Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"client":"0","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
107762024-03-06T21:02:20.673Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.254062333Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"client":"1","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
107772024-03-06T21:02:20.673Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.254096811Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"client":"1","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
107782024-03-06T21:02:20.673Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.254144918Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"client":"2","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
107792024-03-06T21:02:20.673Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.254184755Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"client":"2","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
107802024-03-06T21:02:20.673Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.254237741Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"":"io task","client":"2","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
107812024-03-06T21:02:20.673Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.254271309Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"":"io task","client":"1","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
107822024-03-06T21:02:20.673Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.254296077Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"":"io task","client":"0","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
107832024-03-06T21:02:20.674Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.254416759Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"":"io task","client":"0","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
107842024-03-06T21:02:20.674Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.254536931Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"":"io task","client":"2","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
107852024-03-06T21:02:20.674Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.254584217Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"":"io task","client":"1","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
107862024-03-06T21:02:20.674Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.255300348Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"client":"0","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
107872024-03-06T21:02:20.674Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.255337205Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"client":"0","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
107882024-03-06T21:02:20.674Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.255429689Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"client":"1","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
107892024-03-06T21:02:20.674Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.255507064Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"client":"1","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
107902024-03-06T21:02:20.674Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.255950293Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"client":"0","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
107912024-03-06T21:02:20.674Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.256114352Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"client":"1","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
107922024-03-06T21:02:20.674Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.256240913Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"client":"2","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
107932024-03-06T21:02:20.674Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.25628086Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"client":"2","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
107942024-03-06T21:02:20.674Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.25643415Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"client":"0","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
107952024-03-06T21:02:20.674Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.256470647Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"client":"0","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
107962024-03-06T21:02:20.674Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.256486256Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
107972024-03-06T21:02:20.674Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.256600078Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"client":"1","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
107982024-03-06T21:02:20.674Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.256633116Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"client":"1","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
107992024-03-06T21:02:20.674Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.256648355Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
108002024-03-06T21:02:20.674Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.256855331Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"client":"2","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
108012024-03-06T21:02:20.674Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.257255373Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"client":"2","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
108022024-03-06T21:02:20.674Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.257283701Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"client":"2","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
108032024-03-06T21:02:20.674Z{"msg":"[0]R flush_numbers[0..12]: [167, 170, 170, 163, 165, 164, 166, 169, 170, 170, 170, 169]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.25729952Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"":"downstairs","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
108042024-03-06T21:02:20.674Z{"msg":"[0]R generation[0..12]: [97, 99, 99, 95, 96, 96, 96, 98, 99, 99, 99, 98]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.257314209Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"":"downstairs","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
108052024-03-06T21:02:20.674Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.257328788Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"":"downstairs","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
108062024-03-06T21:02:20.674Z{"msg":"[1]R flush_numbers[0..12]: [167, 170, 170, 163, 165, 164, 166, 169, 170, 170, 170, 169]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.257349647Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"":"downstairs","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
108072024-03-06T21:02:20.675Z{"msg":"[1]R generation[0..12]: [97, 99, 99, 95, 96, 96, 96, 98, 99, 99, 99, 98]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.257363836Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"":"downstairs","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
108082024-03-06T21:02:20.675Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.257377925Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"":"downstairs","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
108092024-03-06T21:02:20.675Z{"msg":"[2]R flush_numbers[0..12]: [167, 170, 170, 163, 165, 164, 166, 169, 170, 170, 170, 169]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.257398583Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"":"downstairs","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
108102024-03-06T21:02:20.675Z{"msg":"[2]R generation[0..12]: [97, 99, 99, 95, 96, 96, 96, 98, 99, 99, 99, 98]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.257412213Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"":"downstairs","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
108112024-03-06T21:02:20.675Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.257427061Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"":"downstairs","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
108122024-03-06T21:02:20.675Z{"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.25744699Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"":"downstairs","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
108132024-03-06T21:02:20.675Z{"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO
108142024-03-06T21:02:20.675Z30,"time":"2024-03-06T21:02:19.257463189Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"":"downstairs","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
108152024-03-06T21:02:20.675Z{"msg":"Next flush: 171","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.257476668Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"":"downstairs","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
108162024-03-06T21:02:20.675Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.257490467Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"":"downstairs","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
108172024-03-06T21:02:20.675Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.257503676Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
108182024-03-06T21:02:20.675Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.257515465Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"client":"0","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
108192024-03-06T21:02:20.675Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.257535594Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"client":"1","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
108202024-03-06T21:02:20.675Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.257549033Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"client":"2","":"downstairs","session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
108212024-03-06T21:02:20.675Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.257562282Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
108222024-03-06T21:02:20.675Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.257581251Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
108232024-03-06T21:02:20.675Z{"msg":"641ddd42-be2c-49a1-b59f-eac69bb34644 is now active with session: 9dc465f4-b2da-4e81-839d-88b47b6619c0","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.257594Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
108242024-03-06T21:02:20.675Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.257606109Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539,"session_id":"9dc465f4-b2da-4e81-839d-88b47b6619c0"}
108252024-03-06T21:02:20.675Z{"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:19.257630247Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539}
108262024-03-06T21:02:20.675ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
108272024-03-06T21:02:20.675ZLoading write count information from file "/tmp/test_repair_verify.data"
108282024-03-06T21:02:20.675ZRun Repair workload
108292024-03-06T21:02:20.675Z01/30 Write block 42 len 12288 data: 11 10 9
108302024-03-06T21:02:20.686Z02/30 Write block 441 len 40960 data: 7 7 6 8 7 8 6 10 9 10
108312024-03-06T21:02:20.686Z03/30 Write block 514 len 20480 data: 7 9 9 9 7
108322024-03-06T21:02:20.686Z04/30 Read block 359 len 8192
108332024-03-06T21:02:20.686Z05/30 Read block 399 len 32768
108342024-03-06T21:02:20.686Z06/30 Read block 69 len 12288
108352024-03-06T21:02:20.686Z07/30 Write block 378 len 36864 data: 6 7 8 7 9 8 10 10 11
108362024-03-06T21:02:20.686Z08/30 Write block 341 len 36864 data: 10 9 9 10 10 8 9 12 11
108372024-03-06T21:02:20.686Z09/30 Read block 471 len 32768
108382024-03-06T21:02:20.686Z10/30 Write block 353 len 36864 data: 9 10 9 13 11 11 9 11 14
108392024-03-06T21:02:20.686Z11/30 Write block 79 len 32768 data: 5 5 5 5 5 3 2 3
108402024-03-06T21:02:20.686Z12/30 Write block 324 len 28672 data: 7 6 7 5 5 6 5
108412024-03-06T21:02:20.686Z13/30 Read block 317 len 8192
108422024-03-06T21:02:20.686Z14/30 Write block 407 len 20480 data: 9 10 8 7 5
108432024-03-06T21:02:20.686Z15/30 Write block 20 len 24576 data: 10 9 11 12 10 9
108442024-03-06T21:02:20.687Z16/30 Flush
108452024-03-06T21:02:20.687Z17/30 Write block 483 len 16384 data: 7 7 9 7
108462024-03-06T21:02:20.687Z18/30 Write block 157 len 40960 data: 11 10 8 8 8 8 6 6 5 4
108472024-03-06T21:02:20.687Z19/30 Read block 412 len 24576
108482024-03-06T21:02:20.687Z20/30 Write block 15 len 24576 data: 8 6 7 9 9 11
108492024-03-06T21:02:20.687Z21/30 Write block 493 len 32768 data: 10 8 9 9 7 8 8 7
108502024-03-06T21:02:20.687Z22/30 Read block 567 len 24576
108512024-03-06T21:02:20.687Z23/30 Read block 488 len 16384
108522024-03-06T21:02:20.687Z24/30 Write block 423 len 4096 data: 11
108532024-03-06T21:02:20.687Z25/30 Write block 314 len 12288 data: 8 8 7
108542024-03-06T21:02:20.687Z26/30 Read block 382 len 28672
108552024-03-06T21:02:20.688Z27/30 Read block 251 len 12288
108562024-03-06T21:02:20.688Z28/30 Read block 478 len 16384
108572024-03-06T21:02:20.688Z29/30 Read block 594 len 8192
108582024-03-06T21:02:20.688Z30/30 Read block 45 len 24576
108592024-03-06T21:02:20.688Z----------------------------------------------------------------
108602024-03-06T21:02:20.688Z Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:30
108612024-03-06T21:02:20.688ZStates: Active Active Active
108622024-03-06T21:02:20.688ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
108632024-03-06T21:02:20.688Z 1 Acked 1000 Write 3 Done Done Done false
108642024-03-06T21:02:20.689Z 2 Acked 1001 Write 10 Done Done Done false
108652024-03-06T21:02:20.689Z 3 Acked 1002 Write 5 Done Done Done false
108662024-03-06T21:02:20.689Z 4 Acked 1003 Read 2 Done Done Done false
108672024-03-06T21:02:20.689Z 5 Acked 1004 Read 8 Done Done Done false
108682024-03-06T21:02:20.689Z 6 Acked 1005 Read 3 Done Done Done false
108692024-03-06T21:02:20.689Z 7 Acked 1006 Write 9 Done Done Done false
108702024-03-06T21:02:20.689Z 8 Acked 1007 Write 9 Done Done Done false
108712024-03-06T21:02:20.689Z 9 Acked 1008 Read 8 Done Done Done false
108722024-03-06T21:02:20.689Z 10 Acked 1009 Write 9 Done Done Sent false
108732024-03-06T21:02:20.689Z 11 Acked 1010 Write 8 Done Done Sent false
108742024-03-06T21:02:20.689Z 12 Acked 1011 Write 7 Done Done Sent false
108752024-03-06T21:02:20.689Z 13 Acked 1012 Read 2 Done Done Sent false
108762024-03-06T21:02:20.689Z 14 Acked 1013 Write 5 Done Done Sent false
108772024-03-06T21:02:20.689Z 15 Acked 1014 Write 6 Done Done Sent false
108782024-03-06T21:02:20.690Z 16 Acked 1015 Flush 0 Done Done Sent false
108792024-03-06T21:02:20.690Z 17 Acked 1016 Write 4 Done Done Sent false
108802024-03-06T21:02:20.690Z 18 Acked 1017 Write 10 Done Done Sent false
108812024-03-06T21:02:20.690Z 19 Acked 1018 Read 6 Done Done Sent false
108822024-03-06T21:02:20.690Z 20 Acked 1019 Write 6 Done Done Sent false
108832024-03-06T21:02:20.690Z 21 Acked 1020 Write 8 Done Done Sent false
108842024-03-06T21:02:20.690Z 22 Acked 1021 Read 6 Done Done Sent false
108852024-03-06T21:02:20.690Z 23 Acked 1022 Read 4 Done Done Sent false
108862024-03-06T21:02:20.690Z 24 Acked 1023 Write 1 Done Done Sent false
108872024-03-06T21:02:20.690Z 25 Acked 1024 Write 3 Done Done Sent false
108882024-03-06T21:02:20.690Z 26 Acked 1025 Read 7 Done Done Sent false
108892024-03-06T21:02:20.690Z 27 Acked 1026 Read 3 Done Done Sent false
108902024-03-06T21:02:20.690Z 28 Acked 1027 Read 4 Done Done Sent false
108912024-03-06T21:02:20.690Z 29 Acked 1028 Read 2 Done Done Sent false
108922024-03-06T21:02:20.690Z 30 Acked 1029 Read 6 Done Sent Sent false
108932024-03-06T21:02:20.690Z STATES DS:0 DS:1 DS:2 TOTAL
108942024-03-06T21:02:20.691Z New 0 0 0 0
108952024-03-06T21:02:20.691Z Sent 0 1 21 22
108962024-03-06T21:02:20.691Z Done 30 29 9 68
108972024-03-06T21:02:20.691Z Skipped 0 0 0 0
108982024-03-06T21:02:20.691Z Error 0 0 0 0
108992024-03-06T21:02:20.691ZLast Flush: 1015 1015 0
109002024-03-06T21:02:20.691ZDownstairs last five completed:
109012024-03-06T21:02:20.691ZUpstairs last five completed: 30 29 28 27 26
109022024-03-06T21:02:20.691Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:02:19.29647718Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1539}
109032024-03-06T21:02:20.691ZWrote out file "/tmp/test_repair_verify.data"
109042024-03-06T21:02:20.691Z
109052024-03-06T21:02:20.691ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
109062024-03-06T21:02:20.691ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
109072024-03-06T21:02:20.692Z 0 000-019 97 97 97 167 167 167 T T F <---
109082024-03-06T21:02:20.692Z 1 020-039 100 100 99 171 171 170 T T F <---
109092024-03-06T21:02:20.692Z 2 040-059 100 100 99 171 171 170 F F T <---
109102024-03-06T21:02:20.692Z 3 060-079 100 100 95 171 171 163 F F F <---
109112024-03-06T21:02:20.692Z 4 080-099 100 100 96 171 171 165 F F F <---
109122024-03-06T21:02:20.692Z 5 100-119 96 96 96 164 164 164 F F F
109132024-03-06T21:02:20.692Z 6 120-139 96 96 96 166 166 166 F F F
109142024-03-06T21:02:20.692Z 7 140-159 98 98 98 169 169 169 T T F <---
109152024-03-06T21:02:20.692Z 8 160-179 99 99 99 170 170 170 T T F <---
109162024-03-06T21:02:20.692Z 9 180-199 99 99 99 170 170 170 F F F
109172024-03-06T21:02:20.692Z 10 200-219 99 99 99 170 170 170 F F F
109182024-03-06T21:02:20.692Z 11 220-239 98 98 98 169 169 169 F F F
109192024-03-06T21:02:20.693Z 12 240-259 95 95 95 163 163 163 F F F
109202024-03-06T21:02:20.693Z 13 260-279 98 98 98 168 168 168 F F F
109212024-03-06T21:02:20.693Z 14 280-299 91 91 91 156 156 156 F F F
109222024-03-06T21:02:20.693Z 15 300-319 96 96 96 164 164 164 T T F <---
109232024-03-06T21:02:20.693Z 16 320-339 100 100 96 171 171 164 F F F <---
109242024-03-06T21:02:20.693Z 17 340-359 100 100 97 171 171 167 F F T <---
109252024-03-06T21:02:20.693Z 18 360-379 100 100 95 171 171 163 F F T <---
109262024-03-06T21:02:20.693Z 19 380-399 100 100 99 171 171 170 F F T <---
109272024-03-06T21:02:20.693Z 20 400-419 100 100 94 171 171 162 F F F <---
109282024-03-06T21:02:20.693Z 21 420-439 99 99 99 170 170 170 T T F <---
109292024-03-06T21:02:20.693Z 22 440-459 100 100 99 171 171 170 F F T <---
109302024-03-06T21:02:21.148Z 23 460-479 78 78 78 137 137 137 F F F
109312024-03-06T21:02:21.148Z 24 480-499 84 84 84 144 144 144 T T F <---
109322024-03-06T21:02:21.149Z 25 500-519 100 100 96 171 171 164 T T T <---
109332024-03-06T21:02:21.149Z 26 520-539 99 99 99 170 170 170 F F F
109342024-03-06T21:02:21.149Z 27 540-559 98 98 98 169 169 169 F F F
109352024-03-06T21:02:21.149Z 28 560-579 99 99 99 170 170 170 F F F
109362024-03-06T21:02:21.149Z 29 580-599 99 99 99 170 170 170 F F F
109372024-03-06T21:02:21.149ZMax gen: 100, Max flush: 171
109382024-03-06T21:02:21.149ZError: Difference in extent metadata found!
109392024-03-06T21:02:21.149ZA Difference in extent metadata is expected here
109402024-03-06T21:02:21.149ZOn loop 50
109412024-03-06T21:02:21.149Z
109422024-03-06T21:02:21.149ZVerifying data now
109432024-03-06T21:02:26.448ZNo change after verify
109442024-03-06T21:02:26.449ZLoop: 50 Downstairs dump after verify (and repair):
109452024-03-06T21:02:26.465ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
109462024-03-06T21:02:26.465Z 0 000-019 101 101 101 172 172 172 F F F
109472024-03-06T21:02:26.465Z 1 020-039 101 101 101 172 172 172 F F F
109482024-03-06T21:02:26.465Z 2 040-059 100 100 100 171 171 171 F F F
109492024-03-06T21:02:26.465Z 3 060-079 100 100 100 171 171 171 F F F
109502024-03-06T21:02:26.465Z 4 080-099 100 100 100 171 171 171 F F F
109512024-03-06T21:02:26.465Z 5 100-119 96 96 96 164 164 164 F F F
109522024-03-06T21:02:26.465Z 6 120-139 96 96 96 166 166 166 F F F
109532024-03-06T21:02:26.465Z 7 140-159 101 101 101 172 172 172 F F F
109542024-03-06T21:02:26.465Z 8 160-179 101 101 101 172 172 172 F F F
109552024-03-06T21:02:26.465Z 9 180-199 99 99 99 170 170 170 F F F
109562024-03-06T21:02:26.465Z 10 200-219 99 99 99 170 170 170 F F F
109572024-03-06T21:02:26.465Z 11 220-239 98 98 98 169 169 169 F F F
109582024-03-06T21:02:26.465Z 12 240-259 95 95 95 163 163 163 F F F
109592024-03-06T21:02:26.465Z 13 260-279 98 98 98 168 168 168 F F F
109602024-03-06T21:02:26.465Z 14 280-299 91 91 91 156 156 156 F F F
109612024-03-06T21:02:26.465Z 15 300-319 101 101 101 172 172 172 F F F
109622024-03-06T21:02:26.466Z 16 320-339 100 100 100 171 171 171 F F F
109632024-03-06T21:02:26.466Z 17 340-359 100 100 100 171 171 171 F F F
109642024-03-06T21:02:26.466Z 18 360-379 100 100 100 171 171 171 F F F
109652024-03-06T21:02:26.470Z 19 380-399 100 100 100 171 171 171 F F F
109662024-03-06T21:02:26.470Z 20 400-419 100 100 100 171 171 171 F F F
109672024-03-06T21:02:26.470Z 21 420-439 101 101 101 172 172 172 F F F
109682024-03-06T21:02:26.470Z 22 440-459 100 100 100 171 171 171 F F F
109692024-03-06T21:02:26.470Z 23 460-479 78 78 78 137 137 137 F F F
109702024-03-06T21:02:26.470Z 24 480-499 101 101 101 172 172 172 F F F
109712024-03-06T21:02:26.470Z 25 500-519 101 101 101 172 172 172 F F F
109722024-03-06T21:02:26.470Z 26 520-539 99 99 99 170 170 170 F F F
109732024-03-06T21:02:26.470Z 27 540-559 98 98 98 169 169 169 F F F
109742024-03-06T21:02:26.470Z 28 560-579 99 99 99 170 170 170 F F F
109752024-03-06T21:02:26.470Z 29 580-599 99 99 99 170 170 170 F F F
109762024-03-06T21:02:26.471ZMax gen: 101, Max flush: 172
109772024-03-06T21:02:26.471Z
109782024-03-06T21:02:26.471ZBegin loop 51
109792024-03-06T21:02:26.471ZDownstairs to restart: 2
109802024-03-06T21:02:26.492Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.245994157Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547}
109812024-03-06T21:02:26.492Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","Crucible runtime is spawned
109822024-03-06T21:02:26.492Zv":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.246315355Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547}
109832024-03-06T21:02:26.502Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.246364732Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547}
109842024-03-06T21:02:26.502Z{"msg":"Crucible 8bc20283-3f34-4272-8249-bd4bd24addc8 has session id: 37985551-3d70-4e71-a903-5b5199b190a6","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.246427787Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
109852024-03-06T21:02:26.502Z{"msg":"Upstairs opts: Upstairs UUID: 8bc20283-3f34-4272-8249-bd4bd24addc8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.246483673Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
109862024-03-06T21:02:26.502Z{"msg":"Crucible stats registered with UUID: 8bc20283-3f34-4272-8249-bd4bd24addc8","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.24652736Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
109872024-03-06T21:02:26.502Z{"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.246566458Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547}
109882024-03-06T21:02:26.502Z{"msg":"8bc20283-3f34-4272-8249-bd4bd24addc8 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.246618694Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
109892024-03-06T21:02:26.502Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.24668026Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"client":"0","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
109902024-03-06T21:02:26.502Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.246700569Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"client":"0","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
109912024-03-06T21:02:26.502Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.246717187Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"client":"1","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
109922024-03-06T21:02:26.503Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.246739246Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"client":"1","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
109932024-03-06T21:02:26.503Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.246781063Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"client":"2","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
109942024-03-06T21:02:26.503Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.246807571Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"client":"2","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
109952024-03-06T21:02:26.503Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.246842759Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"":"io task","client":"0","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
109962024-03-06T21:02:26.503Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.246876666Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"":"io task","client":"2","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
109972024-03-06T21:02:26.503Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.246929143Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"":"io task","client":"1","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
109982024-03-06T21:02:26.503Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.246958151Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"":"io task","client":"2","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
109992024-03-06T21:02:26.503Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.246984629Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"":"io task","client":"0","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110002024-03-06T21:02:26.503Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.247009077Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"":"io task","client":"1","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110012024-03-06T21:02:26.503Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.247708529Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"client":"0","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110022024-03-06T21:02:26.503Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.247749856Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"client":"0","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110032024-03-06T21:02:26.503Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.247811742Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"client":"1","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110042024-03-06T21:02:26.503Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.247904575Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"client":"1","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110052024-03-06T21:02:26.503Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.248481096Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"client":"1","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110062024-03-06T21:02:26.503Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.248539302Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"client":"0","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110072024-03-06T21:02:26.503Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.248701471Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"client":"2","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110082024-03-06T21:02:26.503Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.248735958Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"client":"2","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110092024-03-06T21:02:26.503Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.249119652Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"client":"0","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110102024-03-06T21:02:26.503Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.249156309Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"client":"0","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110112024-03-06T21:02:26.504Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.249185497Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110122024-03-06T21:02:26.504Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.249199166Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"client":"1","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110132024-03-06T21:02:26.504Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.249215235Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"client":"1","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110142024-03-06T21:02:26.504Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.249234364Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110152024-03-06T21:02:26.504Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.249348216Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"client":"2","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110162024-03-06T21:02:26.504Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.249744539Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"client":"2","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110172024-03-06T21:02:26.504Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.249771647Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"client":"2","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110182024-03-06T21:02:26.504Z{"msg":"[0]R flush_numbers[0..12]: [172, 172, 171, 171, 171, 164, 166, 172, 172, 170, 170, 169]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.249786826Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"":"downstairs","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110192024-03-06T21:02:26.504Z{"msg":"[0]R generation[0..12]: [101, 101, 100, 100, 100, 96, 96, 101, 101, 99, 99, 98]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.249801155Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"":"downstairs","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110202024-03-06T21:02:26.504Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.249816074Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"":"downstairs","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110212024-03-06T21:02:26.504Z{"msg":"[1]R flush_numbers[0..12]: [172, 172, 171, 171, 171, 164, 166, 172, 172, 170, 170, 169]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.249830193Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"":"downstairs","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110222024-03-06T21:02:26.504Z{"msg":"[1]R generation[0..12]: [101, 101, 100, 100, 100, 96, 96, 101, 101, 99, 99, 98]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.249852061Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"":"downstairs","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110232024-03-06T21:02:26.504Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.24986657Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"":"downstairs","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110242024-03-06T21:02:26.504Z{"msg":"[2]R flush_numbers[0..12]: [172, 172, 171, 171, 171, 164, 166, 172, 172, 170, 170, 169]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.249880639Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"":"downstairs","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110252024-03-06T21:02:26.504Z{"msg":"[2]R generation[0..12]: [101, 101, 100, 100, 100, 96, 96, 101, 101, 99, 99, 98]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.249895358Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"":"downstairs","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110262024-03-06T21:02:26.504Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
110272024-03-06T21:02:26.504Z,"time":"2024-03-06T21:02:25.249909887Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"":"downstairs","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110282024-03-06T21:02:26.504Z{"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.249924336Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"":"downstairs","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110292024-03-06T21:02:26.504Z{"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.249937685Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"":"downstairs","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110302024-03-06T21:02:26.504Z{"msg":"Next flush: 173","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.249951484Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"":"downstairs","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110312024-03-06T21:02:26.505Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.249964744Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"":"downstairs","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110322024-03-06T21:02:26.505Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.249985092Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110332024-03-06T21:02:26.505Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.249997191Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"client":"0","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110342024-03-06T21:02:26.505Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.25001054Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"client":"1","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110352024-03-06T21:02:26.505Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.250030689Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"client":"2","":"downstairs","session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110362024-03-06T21:02:26.505Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.250044718Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110372024-03-06T21:02:26.505Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.250057657Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110382024-03-06T21:02:26.505Z{"msg":"8bc20283-3f34-4272-8249-bd4bd24addc8 is now active with session: 37985551-3d70-4e71-a903-5b5199b190a6","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.250070436Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110392024-03-06T21:02:26.505Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.250083095Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547,"session_id":"37985551-3d70-4e71-a903-5b5199b190a6"}
110402024-03-06T21:02:26.505Z{"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:25.250095175Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547}
110412024-03-06T21:02:26.505ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
110422024-03-06T21:02:26.505ZLoading write count information from file "/tmp/test_repair_verify.data"
110432024-03-06T21:02:26.505ZRun Repair workload
110442024-03-06T21:02:26.505Z01/30 Write block 434 len 40960 data: 11 9 9 11 11 12 10 8 8 7
110452024-03-06T21:02:26.505Z02/30 Write block 226 len 8192 data: 7 7
110462024-03-06T21:02:26.505Z03/30 Read block 140 len 16384
110472024-03-06T21:02:26.517Z04/30 Read block 82 len 12288
110482024-03-06T21:02:26.517Z05/30 Write block 161 len 32768 data: 9 9 7 7 6 5 5 5
110492024-03-06T21:02:26.517Z06/30 Write block 176 len 40960 data: 8 8 7 10 11 13 12 11 11 10
110502024-03-06T21:02:26.517Z07/30 Read block 462 len 20480
110512024-03-06T21:02:26.517Z08/30 Read block 164 len 32768
110522024-03-06T21:02:26.517Z09/30 Read block 92 len 32768
110532024-03-06T21:02:26.517Z10/30 Read block 304 len 28672
110542024-03-06T21:02:26.517Z11/30 Read block 318 len 12288
110552024-03-06T21:02:26.517Z12/30 Write block 400 len 4096 data: 6
110562024-03-06T21:02:26.517Z13/30 Read block 79 len 36864
110572024-03-06T21:02:26.517Z14/30 Read block 390 len 36864
110582024-03-06T21:02:26.517Z15/30 Read block 28 len 12288
110592024-03-06T21:02:26.517Z16/30 Write block 96 len 24576 data: 7 6 7 7 5 7
110602024-03-06T21:02:26.517Z17/30 Flush
110612024-03-06T21:02:26.521Z18/30 Write block 528 len 32768 data: 9 10 11 10 9 8 6 8
110622024-03-06T21:02:26.521Z19/30 Flush
110632024-03-06T21:02:26.521Z20/30 Write block 171 len 20480 data: 3 5 9 8 9
110642024-03-06T21:02:26.521Z21/30 Read block 526 len 40960
110652024-03-06T21:02:26.525Z22/30 Read block 397 len 36864
110662024-03-06T21:02:26.525Z23/30 Read block 21 len 20480
110672024-03-06T21:02:26.525Z24/30 Read block 463 len 12288
110682024-03-06T21:02:26.537Z25/30 Read block 339 len 36864
110692024-03-06T21:02:26.538Z26/30 Read block 85 len 12288
110702024-03-06T21:02:26.538Z27/30 Read block 332 len 28672
110712024-03-06T21:02:26.538Z28/30 Read block 294 len 40960
110722024-03-06T21:02:26.538Z29/30 Read block 456 len 4096
110732024-03-06T21:02:26.539Z30/30 Write block 19 len 16384 data: 10 12 10 12
110742024-03-06T21:02:26.539Z----------------------------------------------------------------
110752024-03-06T21:02:26.539Z Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:30
110762024-03-06T21:02:26.539ZStates: Active Active Active
110772024-03-06T21:02:26.540ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
110782024-03-06T21:02:26.540Z 1 Acked 1000 Write 10 Done Done Done false
110792024-03-06T21:02:26.540Z 2 Acked 1001 Write 2 Done Done Done false
110802024-03-06T21:02:26.540Z 3 Acked 1002 Read 4 Done Done Done false
110812024-03-06T21:02:26.540Z 4 Acked 1003 Read 3 Done Done Done false
110822024-03-06T21:02:26.540Z 5 Acked 1004 Write 8 Done Done Done false
110832024-03-06T21:02:26.540Z 6 Acked 1005 Write 10 Done Done Sent false
110842024-03-06T21:02:26.540Z 7 Acked 1006 Read 5 Done Done Sent false
110852024-03-06T21:02:26.540Z 8 Acked 1007 Read 8 Done Done Sent false
110862024-03-06T21:02:26.540Z 9 Acked 1008 Read 8 Done Done Sent false
110872024-03-06T21:02:26.540Z 10 Acked 1009 Read 7 Done Done Sent false
110882024-03-06T21:02:26.541Z 11 Acked 1010 Read 3 Done Done Sent false
110892024-03-06T21:02:26.541Z 12 Acked 1011 Write 1 Done Done Sent false
110902024-03-06T21:02:26.541Z 13 Acked 1012 Read 9 Done Done Sent false
110912024-03-06T21:02:26.541Z 14 Acked 1013 Read 9 Done Done Sent false
110922024-03-06T21:02:26.541Z 15 Acked 1014 Read 3 Done Done Sent false
110932024-03-06T21:02:26.541Z 16 Acked 1015 Write 6 Done Done Sent false
110942024-03-06T21:02:26.541Z 17 Acked 1016 Flush 0 Done Done Sent false
110952024-03-06T21:02:26.541Z 18 Acked 1017 Write 8 Done Done Sent false
110962024-03-06T21:02:26.541Z 19 Acked 1018 Flush 0 Done Done Sent false
110972024-03-06T21:02:26.541Z 20 Acked 1019 Write 5 Done Done Sent false
110982024-03-06T21:02:26.541Z 21 Acked 1020 Read 10 Done Done Sent false
110992024-03-06T21:02:26.541Z 22 Acked 1021 Read 9 Done Done Sent false
111002024-03-06T21:02:26.541Z 23 Acked 1022 Read 5 Done Done Sent false
111012024-03-06T21:02:26.541Z 24 Acked 1023 Read 3 Done Done Sent false
111022024-03-06T21:02:26.541Z 25 Acked 1024 Read 9 Done Done Sent false
111032024-03-06T21:02:26.541Z 26 Acked 1025 Read 3 Done Done Sent false
111042024-03-06T21:02:26.541Z 27 Acked 1026 Read 7 Done Done Sent false
111052024-03-06T21:02:26.542Z 28 Acked 1027 Read 10 Done Done Sent false
111062024-03-06T21:02:26.542Z 29 Acked 1028 Read 1 Done Done Sent false
111072024-03-06T21:02:26.542Z 30 Acked 1029 Write 4 Sent Sent Sent false
111082024-03-06T21:02:26.542Z STATES DS:0 DS:1 DS:2 TOTAL
111092024-03-06T21:02:26.542Z New 0 0 0 0
111102024-03-06T21:02:26.542Z Sent 1 1 25 27
111112024-03-06T21:02:26.542Z Done 29 29 5 63
111122024-03-06T21:02:26.542Z Skipped 0 0 0 0
111132024-03-06T21:02:26.542Z Error 0 0 0 0
111142024-03-06T21:02:26.543ZLast Flush: 1018 1018 0
111152024-03-06T21:02:26.543ZDownstairs last five completed:
111162024-03-06T21:02:26.543ZUpstairs last five completed: 30 29 28 27 26
111172024-03-06T21:02:26.543Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:02:25.285848051Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1547}
111182024-03-06T21:02:26.550ZWrote out file "/tmp/test_repair_verify.data"
111192024-03-06T21:02:26.550Z
111202024-03-06T21:02:26.550ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
111212024-03-06T21:02:26.563ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
111222024-03-06T21:02:26.563Z 0 000-019 101 101 101 172 172 172 T T F <---
111232024-03-06T21:02:26.563Z 1 020-039 101 101 101 172 172 172 T T F <---
111242024-03-06T21:02:26.563Z 2 040-059 100 100 100 171 171 171 F F F
111252024-03-06T21:02:26.563Z 3 060-079 100 100 100 171 171 171 F F F
111262024-03-06T21:02:26.563Z 4 080-099 102 102 100 173 173 171 F F F <---
111272024-03-06T21:02:26.563Z 5 100-119 102 102 96 173 173 164 F F F <---
111282024-03-06T21:02:26.563Z 6 120-139 96 96 96 166 166 166 F F F
111292024-03-06T21:02:26.563Z 7 140-159 101 101 101 172 172 172 F F F
111302024-03-06T21:02:26.563Z 8 160-179 102 102 101 173 173 172 T T T <---
111312024-03-06T21:02:26.563Z 9 180-199 102 102 99 173 173 170 F F F <---
111322024-03-06T21:02:26.563Z 10 200-219 99 99 99 170 170 170 F F F
111332024-03-06T21:02:26.563Z 11 220-239 102 102 98 173 173 169 F F T <---
111342024-03-06T21:02:26.564Z 12 240-259 95 95 95 163 163 163 F F F
111352024-03-06T21:02:26.564Z 13 260-279 98 98 98 168 168 168 F F F
111362024-03-06T21:02:26.564Z 14 280-299 91 91 91 156 156 156 F F F
111372024-03-06T21:02:26.564Z 15 300-319 101 101 101 172 172 172 F F F
111382024-03-06T21:02:26.564Z 16 320-339 100 100 100 171 171 171 F F F
111392024-03-06T21:02:26.568Z 17 340-359 100 100 100 171 171 171 F F F
111402024-03-06T21:02:26.568Z 18 360-379 100 100 100 171 171 171 F F F
111412024-03-06T21:02:26.568Z 19 380-399 100 100 100 171 171 171 F F F
111422024-03-06T21:02:26.568Z 20 400-419 102 102 100 173 173 171 F F F <---
111432024-03-06T21:02:26.568Z 21 420-439 102 102 101 173 173 172 F F T <---
111442024-03-06T21:02:26.568Z 22 440-459 102 102 100 173 173 171 F F T <---
111452024-03-06T21:02:26.568Z 23 460-479 78 78 78 137 137 137 F F F
111462024-03-06T21:02:26.568Z 24 480-499 101 101 101 172 172 172 F F F
111472024-03-06T21:02:26.568Z 25 500-519 101 101 101 172 172 172 F F F
111482024-03-06T21:02:26.568Z 26 520-539 102 102 99 174 174 170 F F F <---
111492024-03-06T21:02:26.568Z 27 540-559 98 98 98 169 169 169 F F F
111502024-03-06T21:02:26.568Z 28 560-579 99 99 99 170 170 170 F F F
111512024-03-06T21:02:26.568Z 29 580-599 99 99 99 170 170 170 F F F
111522024-03-06T21:02:26.568ZMax gen: 102, Max flush: 174
111532024-03-06T21:02:26.568ZError: Difference in extent metadata found!
111542024-03-06T21:02:26.568ZA Difference in extent metadata is expected here
111552024-03-06T21:02:26.568ZOn loop 51
111562024-03-06T21:02:26.568Z
111572024-03-06T21:02:26.568ZVerifying data now
111582024-03-06T21:02:32.169ZNo change after verify
111592024-03-06T21:02:32.169ZLoop: 51 Downstairs dump after verify (and repair):
111602024-03-06T21:02:32.185ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
111612024-03-06T21:02:32.185Z 0 000-019 103 103 103 175 175 175 F F F
111622024-03-06T21:02:32.185Z 1 020-039 103 103 103 175 175 175 F F F
111632024-03-06T21:02:32.185Z 2 040-059 100 100 100 171 171 171 F F F
111642024-03-06T21:02:32.185Z 3 060-079 100 100 100 171 171 171 F F F
111652024-03-06T21:02:32.185Z 4 080-099 102 102 102 173 173 173 F F F
111662024-03-06T21:02:32.185Z 5 100-119 102 102 102 173 173 173 F F F
111672024-03-06T21:02:32.185Z 6 120-139 96 96 96 166 166 166 F F F
111682024-03-06T21:02:32.185Z 7 140-159 101 101 101 172 172 172 F F F
111692024-03-06T21:02:32.185Z 8 160-179 103 103 103 175 175 175 F F F
111702024-03-06T21:02:32.185Z 9 180-199 102 102 102 173 173 173 F F F
111712024-03-06T21:02:32.185Z 10 200-219 99 99 99 170 170 170 F F F
111722024-03-06T21:02:32.185Z 11 220-239 102 102 102 173 173 173 F F F
111732024-03-06T21:02:32.185Z 12 240-259 95 95 95 163 163 163 F F F
111742024-03-06T21:02:32.185Z 13 260-279 98 98 98 168 168 168 F F F
111752024-03-06T21:02:32.186Z 14 280-299 91 91 91 156 156 156 F F F
111762024-03-06T21:02:32.186Z 15 300-319 101 101 101 172 172 172 F F F
111772024-03-06T21:02:32.186Z 16 320-339 100 100 100 171 171 171 F F F
111782024-03-06T21:02:32.186Z 17 340-359 100 100 100 171 171 171 F F F
111792024-03-06T21:02:32.186Z 18 360-379 100 100 100 171 171 171 F F F
111802024-03-06T21:02:32.186Z 19 380-399 100 100 100 171 171 171 F F F
111812024-03-06T21:02:32.186Z 20 400-419 102 102 102 173 173 173 F F F
111822024-03-06T21:02:32.186Z 21 420-439 102 102 102 173 173 173 F F F
111832024-03-06T21:02:32.186Z 22 440-459 102 102 102 173 173 173 F F F
111842024-03-06T21:02:32.189Z 23 460-479 78 78 78 137 137 137 F F F
111852024-03-06T21:02:32.189Z 24 480-499 101 101 101 172 172 172 F F F
111862024-03-06T21:02:32.190Z 25 500-519 101 101 101 172 172 172 F F F
111872024-03-06T21:02:32.190Z 26 520-539 102 102 102 174 174 174 F F F
111882024-03-06T21:02:32.190Z 27 540-559 98 98 98 169 169 169 F F F
111892024-03-06T21:02:32.190Z 28 560-579 99 99 99 170 170 170 F F F
111902024-03-06T21:02:32.190Z 29 580-599 99 99 99 170 170 170 F F F
111912024-03-06T21:02:32.190ZMax gen: 103, Max flush: 175
111922024-03-06T21:02:32.190Z
111932024-03-06T21:02:32.190ZBegin loop 52
111942024-03-06T21:02:32.190ZDownstairs to restart: 1
111952024-03-06T21:02:32.212Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.966075306Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555}
111962024-03-06T21:02:32.212Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.966351917Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555}
111972024-03-06T21:02:32.212Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.966373616Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555}
111982024-03-06T21:02:32.212Z{"msg":"Crucible 4eec313a-584d-49ad-b59b-e92215b9f7b4 has session id: c74e7f1f-8a36-46fa-afb1-8764a67f1316","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.966405944Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
111992024-03-06T21:02:32.212Z{Crucible runtime is spawned
112002024-03-06T21:02:32.212Z"msg":"Upstairs opts: Upstairs UUID: 4eec313a-584d-49ad-b59b-e92215b9f7b4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.96645526Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112012024-03-06T21:02:32.218Z{"msg":"Crucible stats registered with UUID: 4eec313a-584d-49ad-b59b-e92215b9f7b4","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.966480778Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112022024-03-06T21:02:32.218Z{"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.966505737Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555}
112032024-03-06T21:02:32.218Z{"msg":"4eec313a-584d-49ad-b59b-e92215b9f7b4 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.966654466Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112042024-03-06T21:02:32.218Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.966720162Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"client":"0","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112052024-03-06T21:02:32.218Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.966768048Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"client":"0","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112062024-03-06T21:02:32.218Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.966796806Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"client":"1","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112072024-03-06T21:02:32.219Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.966831654Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"client":"1","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112082024-03-06T21:02:32.219Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.966854793Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"client":"2","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112092024-03-06T21:02:32.219Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.966880461Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"client":"2","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112102024-03-06T21:02:32.219Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.966911439Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112112024-03-06T21:02:32.220Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.966941587Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112122024-03-06T21:02:32.220Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.966967045Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112132024-03-06T21:02:32.220Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.966990393Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112142024-03-06T21:02:32.221Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.967065118Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112152024-03-06T21:02:32.221Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.967095976Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112162024-03-06T21:02:32.221Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.967876862Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"client":"0","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112172024-03-06T21:02:32.222Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.967916459Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"client":"0","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112182024-03-06T21:02:32.222Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.967932808Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"client":"2","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112192024-03-06T21:02:32.222Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.967965296Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"client":"2","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112202024-03-06T21:02:32.222Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.968555035Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"client":"0","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112212024-03-06T21:02:32.222Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.968608732Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"client":"2","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112222024-03-06T21:02:32.223Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.969194231Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"client":"0","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112232024-03-06T21:02:32.223Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.969256547Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"client":"0","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112242024-03-06T21:02:32.223Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.969314343Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112252024-03-06T21:02:32.223Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.969343981Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"client":"1","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112262024-03-06T21:02:32.223Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.969382158Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"client":"1","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112272024-03-06T21:02:32.223Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.969468153Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"client":"2","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112282024-03-06T21:02:32.223Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.969495731Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"client":"2","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112292024-03-06T21:02:32.224Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.969510479Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112302024-03-06T21:02:32.224Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.969857666Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"client":"1","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112312024-03-06T21:02:32.232Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.970315604Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"client":"1","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112322024-03-06T21:02:32.232Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.970345902Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"client":"1","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112332024-03-06T21:02:32.232Z{"msg":"[0]R flush_numbers[0..12]: [175, 175, 171, 171, 173, 173, 166, 172, 175, 173, 170, 173]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.970361621Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"":"downstairs","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112342024-03-06T21:02:32.232Z{"msg":"[0]R generation[0..12]: [103, 103, 100, 100, 102, 102, 96, 101, 103, 102, 99, 102]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.97037608Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"":"downstairs","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112352024-03-06T21:02:32.232Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.970390629Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"":"downstairs","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112362024-03-06T21:02:32.232Z{"msg":"[1]R flush_numbers[0..12]: [175, 175, 171, 171, 173, 173, 166, 172, 175, 173, 170, 173]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.970411897Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"":"downstairs","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112372024-03-06T21:02:32.232Z{"msg":"[1]R generation[0..12]: [103, 103, 100, 100, 102, 102, 96, 101, 103, 102, 99, 102]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.970426107Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"":"downstairs","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112382024-03-06T21:02:32.232Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.970446955Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"":"downstairs","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112392024-03-06T21:02:32.233Z{"msg":"[2]R flush_numbers[0..12]: [175, 175, 171, 171, 173, 173, 166, 172, 175, 173, 170, 173]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.970461294Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"":"downstairs","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112402024-03-06T21:02:32.233Z{"msg":"[2]R generation[0..12]: [103, 103, 100, 100, 102, 102, 96, 101, 103, 102, 99, 102]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.970481563Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"":"downstairs","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112412024-03-06T21:02:32.233Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.970495672Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"":"downstairs","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112422024-03-06T21:02:32.233Z{"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.97051579Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"":"downstairs","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112432024-03-06T21:02:32.233ZWait for a query_work_queue command to finish before sending IO
112442024-03-06T21:02:32.233Z{"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.970534729Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"":"downstairs","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112452024-03-06T21:02:32.233Z{"msg":"Next flush: 176","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.970549108Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"":"downstairs","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112462024-03-06T21:02:32.233Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.970562657Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"":"downstairs","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112472024-03-06T21:02:32.234Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.970576186Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112482024-03-06T21:02:32.234Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.970595255Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"client":"0","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112492024-03-06T21:02:32.234Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.970610494Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"client":"1","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112502024-03-06T21:02:32.234Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.970642832Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"client":"2","":"downstairs","session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112512024-03-06T21:02:32.234Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.970677269Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112522024-03-06T21:02:32.234Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.970710727Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112532024-03-06T21:02:32.234Z{"msg":"4eec313a-584d-49ad-b59b-e92215b9f7b4 is now active with session: c74e7f1f-8a36-46fa-afb1-8764a67f1316","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.970748044Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112542024-03-06T21:02:32.234Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.970772023Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555,"session_id":"c74e7f1f-8a36-46fa-afb1-8764a67f1316"}
112552024-03-06T21:02:32.234Z{"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:30.97080556Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555}
112562024-03-06T21:02:32.234ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
112572024-03-06T21:02:32.235ZLoading write count information from file "/tmp/test_repair_verify.data"
112582024-03-06T21:02:32.235ZRun Repair workload
112592024-03-06T21:02:32.235Z01/30 Write block 564 len 40960 data: 9 7 8 6 4 4 6 6 6 6
112602024-03-06T21:02:32.235Z02/30 Read block 475 len 32768
112612024-03-06T21:02:32.235Z03/30 Flush
112622024-03-06T21:02:32.236Z04/30 Write block 421 len 32768 data: 14 12 12 11 11 11 12 11
112632024-03-06T21:02:32.236Z05/30 Read block 419 len 20480
112642024-03-06T21:02:32.236Z06/30 Write block 379 len 40960 data: 8 9 8 10 9 11 11 12 9 9
112652024-03-06T21:02:32.236Z07/30 Read block 558 len 28672
112662024-03-06T21:02:32.241Z08/30 Read block 21 len 24576
112672024-03-06T21:02:32.241Z09/30 Write block 189 len 12288 data: 9 10 8
112682024-03-06T21:02:32.242Z10/30 Read block 47 len 28672
112692024-03-06T21:02:32.242Z11/30 Read block 133 len 16384
112702024-03-06T21:02:32.242Z12/30 Read block 232 len 12288
112712024-03-06T21:02:32.242Z13/30 Read block 482 len 40960
112722024-03-06T21:02:32.242Z14/30 Write block 355 len 8192 data: 10 14
112732024-03-06T21:02:32.242Z15/30 Read block 328 len 32768
112742024-03-06T21:02:32.242Z16/30 Write block 129 len 4096 data: 11
112752024-03-06T21:02:32.242Z17/30 Read block 230 len 28672
112762024-03-06T21:02:32.242Z18/30 Read block 491 len 32768
112772024-03-06T21:02:32.243Z19/30 Read block 112 len 36864
112782024-03-06T21:02:32.243Z20/30 Read block 152 len 12288
112792024-03-06T21:02:32.250Z21/30 Read block 6 len 20480
112802024-03-06T21:02:32.250Z22/30 Flush
112812024-03-06T21:02:32.250Z23/30 Write block 208 len 40960 data: 7 6 8 7 7 6 6 9 11 13
112822024-03-06T21:02:32.250Z24/30 Write block 32 len 4096 data: 7
112832024-03-06T21:02:32.250Z25/30 Write block 361 len 4096 data: 15
112842024-03-06T21:02:32.254Z26/30 Write block 513 len 16384 data: 8 8 10 10
112852024-03-06T21:02:32.254Z27/30 Read block 91 len 36864
112862024-03-06T21:02:32.254Z28/30 Write block 82 len 32768 data: 6 6 4 3 4 3 4 4
112872024-03-06T21:02:32.254Z29/30 Read block 437 len 24576
112882024-03-06T21:02:32.254Z30/30 Read block 319 len 12288
112892024-03-06T21:02:32.254Z----------------------------------------------------------------
112902024-03-06T21:02:32.254Z Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:27
112912024-03-06T21:02:32.254ZStates: Active Active Active
112922024-03-06T21:02:32.254ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
112932024-03-06T21:02:32.254Z 4 Acked 1003 Write 8 Done Done Done false
112942024-03-06T21:02:32.254Z 5 Acked 1004 Read 5 Done Done Done false
112952024-03-06T21:02:32.254Z 6 Acked 1005 Write 10 Done Sent Done false
112962024-03-06T21:02:32.254Z 7 Acked 1006 Read 7 Done Sent Done false
112972024-03-06T21:02:32.254Z 8 Acked 1007 Read 6 Done Sent Done false
112982024-03-06T21:02:32.254Z 9 Acked 1008 Write 3 Done Sent Done false
112992024-03-06T21:02:32.254Z 10 Acked 1009 Read 7 Done Sent Done false
113002024-03-06T21:02:32.254Z 11 Acked 1010 Read 4 Done Sent Done false
113012024-03-06T21:02:32.254Z 12 Acked 1011 Read 3 Done Sent Done false
113022024-03-06T21:02:32.254Z 13 Acked 1012 Read 10 Done Sent Done false
113032024-03-06T21:02:32.254Z 14 Acked 1013 Write 2 Done Sent Done false
113042024-03-06T21:02:32.255Z 15 Acked 1014 Read 8 Done Sent Done false
113052024-03-06T21:02:32.255Z 16 Acked 1015 Write 1 Done Sent Done false
113062024-03-06T21:02:32.255Z 17 Acked 1016 Read 7 Done Sent Done false
113072024-03-06T21:02:32.255Z 18 Acked 1017 Read 8 Done Sent Done false
113082024-03-06T21:02:32.255Z 19 Acked 1018 Read 9 Done Sent Done false
113092024-03-06T21:02:32.255Z 20 Acked 1019 Read 3 Done Sent Done false
113102024-03-06T21:02:32.255Z 21 Acked 1020 Read 5 Done Sent Done false
113112024-03-06T21:02:32.255Z 22 Acked 1021 Flush 0 Done Sent Done false
113122024-03-06T21:02:32.255Z 23 Acked 1022 Write 10 Done Sent Done false
113132024-03-06T21:02:32.255Z 24 Acked 1023 Write 1 Done Sent Done false
113142024-03-06T21:02:32.255Z 25 Acked 1024 Write 1 Done Sent Done false
113152024-03-06T21:02:32.255Z 26 Acked 1025 Write 4 Done Sent Done false
113162024-03-06T21:02:32.255Z 27 Acked 1026 Read 9 Done Sent Done false
113172024-03-06T21:02:32.255Z 28 Acked 1027 Write 8 Done Sent Done false
113182024-03-06T21:02:32.255Z 29 Acked 1028 Read 6 Done Sent Done false
113192024-03-06T21:02:32.255Z 30 Acked 1029 Read 3 Done Sent Sent false
113202024-03-06T21:02:32.255Z STATES DS:0 DS:1 DS:2 TOTAL
113212024-03-06T21:02:32.255Z New 0 0 0 0
113222024-03-06T21:02:32.255Z Sent 0 25 1 26
113232024-03-06T21:02:32.255Z Done 27 2 26 55
113242024-03-06T21:02:32.255Z Skipped 0 0 0 0
113252024-03-06T21:02:32.255Z Error 0 0 0 0
113262024-03-06T21:02:32.255ZLast Flush: 1021 1002 1021
113272024-03-06T21:02:32.255ZDownstairs last five completed: 1002 1001 1000
113282024-03-06T21:02:32.256ZUpstairs last five completed: 30 29 28 27 26
113292024-03-06T21:02:32.256Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:02:31.007390069Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1555}
113302024-03-06T21:02:32.259ZWrote out file "/tmp/test_repair_verify.data"
113312024-03-06T21:02:32.259Z
113322024-03-06T21:02:32.263ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
113332024-03-06T21:02:32.285ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
113342024-03-06T21:02:32.285Z 0 000-019 103 103 103 175 175 175 F F F
113352024-03-06T21:02:32.285Z 1 020-039 103 103 103 175 175 175 T F T <---
113362024-03-06T21:02:32.285Z 2 040-059 100 100 100 171 171 171 F F F
113372024-03-06T21:02:32.285Z 3 060-079 100 100 100 171 171 171 F F F
113382024-03-06T21:02:32.286Z 4 080-099 102 102 102 173 173 173 T F T <---
113392024-03-06T21:02:32.286Z 5 100-119 102 102 102 173 173 173 F F F
113402024-03-06T21:02:32.286Z 6 120-139 104 96 104 177 166 177 F F F <---
113412024-03-06T21:02:32.286Z 7 140-159 101 101 101 172 172 172 F F F
113422024-03-06T21:02:32.286Z 8 160-179 103 103 103 175 175 175 F F F
113432024-03-06T21:02:32.286Z 9 180-199 104 102 104 177 173 177 F F F <---
113442024-03-06T21:02:32.286Z 10 200-219 99 99 99 170 170 170 T F T <---
113452024-03-06T21:02:32.286Z 11 220-239 102 102 102 173 173 173 F F F
113462024-03-06T21:02:32.286Z 12 240-259 95 95 95 163 163 163 F F F
113472024-03-06T21:02:32.286Z 13 260-279 98 98 98 168 168 168 F F F
113482024-03-06T21:02:32.286Z 14 280-299 91 91 91 156 156 156 F F F
113492024-03-06T21:02:32.286Z 15 300-319 101 101 101 172 172 172 F F F
113502024-03-06T21:02:32.286Z 16 320-339 100 100 100 171 171 171 F F F
113512024-03-06T21:02:32.287Z 17 340-359 104 100 104 177 171 177 F F F <---
113522024-03-06T21:02:32.287Z 18 360-379 104 100 104 177 171 177 T F T <---
113532024-03-06T21:02:32.287Z 19 380-399 104 100 104 177 171 177 F F F <---
113542024-03-06T21:02:32.287Z 20 400-419 102 102 102 173 173 173 F F F
113552024-03-06T21:02:32.287Z 21 420-439 104 102 104 177 173 177 F T F <---
113562024-03-06T21:02:32.287Z 22 440-459 102 102 102 173 173 173 F F F
113572024-03-06T21:02:32.287Z 23 460-479 78 78 78 137 137 137 F F F
113582024-03-06T21:02:32.287Z 24 480-499 101 101 101 172 172 172 F F F
113592024-03-06T21:02:32.287Z 25 500-519 101 101 101 172 172 172 T F T <---
113602024-03-06T21:02:32.287Z 26 520-539 102 102 102 174 174 174 F F F
113612024-03-06T21:02:32.292Z 27 540-559 98 98 98 169 169 169 F F F
113622024-03-06T21:02:32.292Z 28 560-579 104 104 104 176 176 176 F F F
113632024-03-06T21:02:32.293Z 29 580-599 99 99 99 170 170 170 F F F
113642024-03-06T21:02:32.293ZMax gen: 104, Max flush: 177
113652024-03-06T21:02:32.293ZError: Difference in extent metadata found!
113662024-03-06T21:02:32.293ZA Difference in extent metadata is expected here
113672024-03-06T21:02:32.293ZOn loop 52
113682024-03-06T21:02:32.293Z
113692024-03-06T21:02:32.293ZVerifying data now
113702024-03-06T21:02:36.841ZNo change after verify
113712024-03-06T21:02:36.841ZLoop: 52 Downstairs dump after verify (and repair):
113722024-03-06T21:02:36.859ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
113732024-03-06T21:02:36.859Z 0 000-019 103 103 103 175 175 175 F F F
113742024-03-06T21:02:36.859Z 1 020-039 105 105 105 178 178 178 F F F
113752024-03-06T21:02:36.859Z 2 040-059 100 100 100 171 171 171 F F F
113762024-03-06T21:02:36.859Z 3 060-079 100 100 100 171 171 171 F F F
113772024-03-06T21:02:36.859Z 4 080-099 105 105 105 178 178 178 F F F
113782024-03-06T21:02:36.859Z 5 100-119 102 102 102 173 173 173 F F F
113792024-03-06T21:02:36.859Z 6 120-139 104 104 104 177 177 177 F F F
113802024-03-06T21:02:36.859Z 7 140-159 101 101 101 172 172 172 F F F
113812024-03-06T21:02:36.859Z 8 160-179 103 103 103 175 175 175 F F F
113822024-03-06T21:02:36.860Z 9 180-199 104 104 104 177 177 177 F F F
113832024-03-06T21:02:36.860Z 10 200-219 105 105 105 178 178 178 F F F
113842024-03-06T21:02:36.860Z 11 220-239 102 102 102 173 173 173 F F F
113852024-03-06T21:02:36.860Z 12 240-259 95 95 95 163 163 163 F F F
113862024-03-06T21:02:36.860Z 13 260-279 98 98 98 168 168 168 F F F
113872024-03-06T21:02:36.860Z 14 280-299 91 91 91 156 156 156 F F F
113882024-03-06T21:02:36.860Z 15 300-319 101 101 101 172 172 172 F F F
113892024-03-06T21:02:36.860Z 16 320-339 100 100 100 171 171 171 F F F
113902024-03-06T21:02:36.860Z 17 340-359 104 104 104 177 177 177 F F F
113912024-03-06T21:02:36.860Z 18 360-379 105 105 105 178 178 178 F F F
113922024-03-06T21:02:36.860Z 19 380-399 104 104 104 177 177 177 F F F
113932024-03-06T21:02:36.860Z 20 400-419 102 102 102 173 173 173 F F F
113942024-03-06T21:02:36.864Z 21 420-439 104 104 104 177 177 177 F F F
113952024-03-06T21:02:36.864Z 22 440-459 102 102 102 173 173 173 F F F
113962024-03-06T21:02:36.864Z 23 460-479 78 78 78 137 137 137 F F F
113972024-03-06T21:02:36.864Z 24 480-499 101 101 101 172 172 172 F F F
113982024-03-06T21:02:36.864Z 25 500-519 105 105 105 178 178 178 F F F
113992024-03-06T21:02:36.864Z 26 520-539 102 102 102 174 174 174 F F F
114002024-03-06T21:02:36.864Z 27 540-559 98 98 98 169 169 169 F F F
114012024-03-06T21:02:36.864Z 28 560-579 104 104 104 176 176 176 F F F
114022024-03-06T21:02:36.864Z 29 580-599 99 99 99 170 170 170 F F F
114032024-03-06T21:02:36.864ZMax gen: 105, Max flush: 178
114042024-03-06T21:02:36.864Z
114052024-03-06T21:02:36.864ZBegin loop 53
114062024-03-06T21:02:36.864ZDownstairs to restart: 2
114072024-03-06T21:02:36.884Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.637687982Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563}
114082024-03-06T21:02:36.884Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.637991522Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563}
114092024-03-06T21:02:36.885Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.63800944Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563}
114102024-03-06T21:02:36.885Z{"msg":"Crucible a630ac9a-e93d-4bde-b028-559875596ce5 has session id: f5b34b5e-7112-4769-a608-10b515b54cb7","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.638028609Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114112024-03-06T21:02:36.892Z{"msg":"Upstairs opts: Upstairs UUID: a630ac9a-e93d-4bde-b028-559875596ce5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.638094485Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114122024-03-06T21:02:36.892Z{"msg":"Crucible stats registered with UUID: a630ac9a-e93d-4bde-b028-559875596ce5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.638120213Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114132024-03-06T21:02:36.892ZCrucible runtime is spawned
114142024-03-06T21:02:36.892Z{"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.638245654Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563}
114152024-03-06T21:02:36.892Z{"msg":"a630ac9a-e93d-4bde-b028-559875596ce5 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.638350837Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114162024-03-06T21:02:36.892Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.638378745Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"client":"0","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114172024-03-06T21:02:36.892Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.638394974Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"client":"0","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114182024-03-06T21:02:36.892Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.638455949Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"client":"1","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114192024-03-06T21:02:36.892Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.638477538Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"client":"1","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114202024-03-06T21:02:36.892Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.638511036Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"client":"2","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114212024-03-06T21:02:36.893Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.638527825Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"client":"2","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114222024-03-06T21:02:36.893Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.638549323Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"":"io task","client":"2","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114232024-03-06T21:02:36.893Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.638569952Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"":"io task","client":"0","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114242024-03-06T21:02:36.893Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.638584811Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"":"io task","client":"1","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114252024-03-06T21:02:36.893Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.638721681Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"":"io task","client":"0","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114262024-03-06T21:02:36.893Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.638827514Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"":"io task","client":"1","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114272024-03-06T21:02:36.893Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.638893999Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"":"io task","client":"2","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114282024-03-06T21:02:36.893Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.639457791Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"client":"1","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114292024-03-06T21:02:36.893Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.639496138Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"client":"1","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114302024-03-06T21:02:36.893Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.639520456Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"client":"0","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114312024-03-06T21:02:36.893Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.639561153Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"client":"0","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114322024-03-06T21:02:36.893Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.640143693Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"client":"1","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114332024-03-06T21:02:36.893Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.64019013Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"client":"0","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114342024-03-06T21:02:36.893Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.64048094Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"client":"2","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114352024-03-06T21:02:36.893Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.640517927Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"client":"2","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114362024-03-06T21:02:36.893Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.640726653Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"client":"1","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114372024-03-06T21:02:36.893Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.64077295Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"client":"1","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114382024-03-06T21:02:36.893Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.640789659Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114392024-03-06T21:02:36.893Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.640840885Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"client":"0","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114402024-03-06T21:02:36.893Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.640884362Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"client":"0","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114412024-03-06T21:02:36.893Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.640901621Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114422024-03-06T21:02:36.893Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.641141155Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"client":"2","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114432024-03-06T21:02:36.893Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.641575765Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"client":"2","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114442024-03-06T21:02:36.894Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.641615262Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"client":"2","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114452024-03-06T21:02:36.894Z{"msg":"[0]R flush_numbers[0..12]: [175, 178, 171, 171, 178, 173, 177, 172, 175, 177, 178, 173]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.641633051Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"":"downstairs","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114462024-03-06T21:02:36.894Z{"msg":"[0]R generation[0..12]: [103, 105, 100, 100, 105, 102, 104, 101, 103, 104, 105, 102]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.641661289Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"":"downstairs","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114472024-03-06T21:02:36.894Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.641681367Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"":"downstairs","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114482024-03-06T21:02:36.894Z{"msg":"[1]R flush_numbers[0..12]: [175, 178, 171, 171, 178, 173, 177, 172, 175, 177, 178, 173]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.641712715Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"":"downstairs","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114492024-03-06T21:02:36.894Z{"msg":"[1]R generation[0..12]: [103, 105, 100, 100, 105, 102, 104, 101, 103, 104, 105, 102]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
114502024-03-06T21:02:36.894Z,"time":"2024-03-06T21:02:35.641733124Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"":"downstairs","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114512024-03-06T21:02:36.894Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.641760082Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"":"downstairs","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114522024-03-06T21:02:36.894Z{"msg":"[2]R flush_numbers[0..12]: [175, 178, 171, 171, 178, 173, 177, 172, 175, 177, 178, 173]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.641805699Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"":"downstairs","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114532024-03-06T21:02:36.894Z{"msg":"[2]R generation[0..12]: [103, 105, 100, 100, 105, 102, 104, 101, 103, 104, 105, 102]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.641825037Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"":"downstairs","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114542024-03-06T21:02:36.894Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.641859285Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"":"downstairs","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114552024-03-06T21:02:36.894Z{"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.641894383Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"":"downstairs","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114562024-03-06T21:02:36.894Z{"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.6419394Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"":"downstairs","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114572024-03-06T21:02:36.894Z{"msg":"Next flush: 179","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.641974107Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"":"downstairs","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114582024-03-06T21:02:36.894Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.642007855Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"":"downstairs","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114592024-03-06T21:02:36.894Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.642027874Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114602024-03-06T21:02:36.894Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.642057791Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"client":"0","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114612024-03-06T21:02:36.894Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.642093759Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"client":"1","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114622024-03-06T21:02:36.894Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.642133776Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"client":"2","":"downstairs","session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114632024-03-06T21:02:36.894Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.642171644Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114642024-03-06T21:02:36.894Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.642205541Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114652024-03-06T21:02:36.894Z{"msg":"a630ac9a-e93d-4bde-b028-559875596ce5 is now active with session: f5b34b5e-7112-4769-a608-10b515b54cb7","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.642231449Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114662024-03-06T21:02:36.894ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
114672024-03-06T21:02:36.895Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.642270717Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563,"session_id":"f5b34b5e-7112-4769-a608-10b515b54cb7"}
114682024-03-06T21:02:36.895Z{"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:35.642292065Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563}
114692024-03-06T21:02:36.898ZLoading write count information from file "/tmp/test_repair_verify.data"
114702024-03-06T21:02:36.899ZRun Repair workload
114712024-03-06T21:02:36.899Z01/30 Write block 114 len 24576 data: 4 4 3 5 5 5
114722024-03-06T21:02:36.899Z02/30 Read block 590 len 40960
114732024-03-06T21:02:36.899Z03/30 Flush
114742024-03-06T21:02:36.899Z04/30 Write block 133 len 40960 data: 7 7 7 8 6 5 6 6 5 5
114752024-03-06T21:02:36.899Z05/30 Write block 19 len 20480 data: 11 13 11 13 13
114762024-03-06T21:02:36.899Z06/30 Read block 274 len 16384
114772024-03-06T21:02:36.899Z07/30 Read block 397 len 28672
114782024-03-06T21:02:36.905Z08/30 Read block 16 len 4096
114792024-03-06T21:02:36.905Z09/30 Read block 503 len 28672
114802024-03-06T21:02:36.905Z10/30 Flush
114812024-03-06T21:02:36.908Z11/30 Write block 156 len 8192 data: 11 12
114822024-03-06T21:02:36.908Z12/30 Read block 58 len 40960
114832024-03-06T21:02:36.908Z13/30 Read block 450 len 16384
114842024-03-06T21:02:36.908Z14/30 Write block 6 len 28672 data: 9 9 8 6 6 9 7
114852024-03-06T21:02:36.908Z15/30 Write block 300 len 40960 data: 6 5 6 6 7 9 10 10 9 8
114862024-03-06T21:02:36.908Z16/30 Read block 483 len 28672
114872024-03-06T21:02:36.912Z17/30 Read block 544 len 32768
114882024-03-06T21:02:36.912Z18/30 Read block 330 len 40960
114892024-03-06T21:02:36.912Z19/30 Read block 205 len 40960
114902024-03-06T21:02:36.915Z20/30 Read block 3 len 24576
114912024-03-06T21:02:36.915Z21/30 Read block 318 len 12288
114922024-03-06T21:02:36.915Z22/30 Write block 79 len 32768 data: 6 6 6 7 7 5 4 5
114932024-03-06T21:02:36.918Z23/30 Read block 159 len 40960
114942024-03-06T21:02:36.918Z24/30 Read block 147 len 16384
114952024-03-06T21:02:36.918Z25/30 Write block 406 len 32768 data: 8 10 11 9 8 6 4 6
114962024-03-06T21:02:36.921Z26/30 Flush
114972024-03-06T21:02:36.924Z27/30 Write block 67 len 28672 data: 13 11 11 8 7 9 8
114982024-03-06T21:02:36.924Z28/30 Read block 203 len 20480
114992024-03-06T21:02:36.924Z29/30 Write block 200 len 32768 data: 10 9 7 6 6 6 8 4
115002024-03-06T21:02:36.928Z30/30 Read block 287 len 32768
115012024-03-06T21:02:36.928Z----------------------------------------------------------------
115022024-03-06T21:02:36.928Z Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:30
115032024-03-06T21:02:36.928ZStates: Active Active Active
115042024-03-06T21:02:36.928ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
115052024-03-06T21:02:36.928Z 1 Acked 1000 Write 6 Done Done Sent false
115062024-03-06T21:02:36.928Z 2 Acked 1001 Read 10 Done Done Sent false
115072024-03-06T21:02:36.928Z 3 Acked 1002 Flush 0 Done Done Sent false
115082024-03-06T21:02:36.928Z 4 Acked 1003 Write 10 Done Done Sent false
115092024-03-06T21:02:36.928Z 5 Acked 1004 Write 5 Done Done Sent false
115102024-03-06T21:02:36.928Z 6 Acked 1005 Read 4 Done Done Sent false
115112024-03-06T21:02:36.929Z 7 Acked 1006 Read 7 Done Done Sent false
115122024-03-06T21:02:36.929Z 8 Acked 1007 Read 1 Done Done Sent false
115132024-03-06T21:02:36.929Z 9 Acked 1008 Read 7 Done Done Sent false
115142024-03-06T21:02:36.929Z 10 Acked 1009 Flush 0 Done Done Sent false
115152024-03-06T21:02:36.929Z 11 Acked 1010 Write 2 Done Done Sent false
115162024-03-06T21:02:36.929Z 12 Acked 1011 Read 10 Done Done Sent false
115172024-03-06T21:02:36.929Z 13 Acked 1012 Read 4 Done Done Sent false
115182024-03-06T21:02:36.929Z 14 Acked 1013 Write 7 Done Done Sent false
115192024-03-06T21:02:36.929Z 15 Acked 1014 Write 10 Done Done Sent false
115202024-03-06T21:02:36.929Z 16 Acked 1015 Read 7 Done Done Sent false
115212024-03-06T21:02:36.929Z 17 Acked 1016 Read 8 Done Done Sent false
115222024-03-06T21:02:36.929Z 18 Acked 1017 Read 10 Done Done Sent false
115232024-03-06T21:02:36.929Z 19 Acked 1018 Read 10 Done Done Sent false
115242024-03-06T21:02:36.929Z 20 Acked 1019 Read 6 Done Done Sent false
115252024-03-06T21:02:36.929Z 21 Acked 1020 Read 3 Done Done Sent false
115262024-03-06T21:02:36.929Z 22 Acked 1021 Write 8 Done Done Sent false
115272024-03-06T21:02:36.929Z 23 Acked 1022 Read 10 Done Done Sent false
115282024-03-06T21:02:36.929Z 24 Acked 1023 Read 4 Done Done Sent false
115292024-03-06T21:02:36.929Z 25 Acked 1024 Write 8 Done Done Sent false
115302024-03-06T21:02:36.929Z 26 Acked 1025 Flush 0 Done Done Sent false
115312024-03-06T21:02:36.929Z 27 Acked 1026 Write 7 Done Done Sent false
115322024-03-06T21:02:36.929Z 28 Acked 1027 Read 5 Done Done Sent false
115332024-03-06T21:02:36.929Z 29 Acked 1028 Write 8 Done Done Sent false
115342024-03-06T21:02:36.929Z 30 Acked 1029 Read 8 Done Sent Sent false
115352024-03-06T21:02:36.930Z STATES DS:0 DS:1 DS:2 TOTAL
115362024-03-06T21:02:36.930Z New 0 0 0 0
115372024-03-06T21:02:36.930Z Sent 0 1 30 31
115382024-03-06T21:02:36.930Z Done 30 29 0 59
115392024-03-06T21:02:36.930Z Skipped 0 0 0 0
115402024-03-06T21:02:36.930Z Error 0 0 0 0
115412024-03-06T21:02:36.930ZLast Flush: 1025 1025 0
115422024-03-06T21:02:36.930ZDownstairs last five completed:
115432024-03-06T21:02:36.930ZUpstairs last five completed: 30 29 28 27 26
115442024-03-06T21:02:36.930Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:02:35.680875546Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1563}
115452024-03-06T21:02:36.930ZWrote out file "/tmp/test_repair_verify.data"
115462024-03-06T21:02:36.933Z
115472024-03-06T21:02:36.936ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
115482024-03-06T21:02:36.956ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
115492024-03-06T21:02:36.957Z 0 000-019 106 106 103 181 181 175 F F F <---
115502024-03-06T21:02:36.957Z 1 020-039 106 106 105 180 180 178 F F F <---
115512024-03-06T21:02:36.957Z 2 040-059 100 100 100 171 171 171 F F F
115522024-03-06T21:02:36.957Z 3 060-079 106 106 100 181 181 171 T T F <---
115532024-03-06T21:02:36.957Z 4 080-099 106 106 105 181 181 178 F F F <---
115542024-03-06T21:02:36.957Z 5 100-119 106 106 102 179 179 173 F F F <---
115552024-03-06T21:02:36.957Z 6 120-139 106 106 104 180 180 177 F F F <---
115562024-03-06T21:02:36.957Z 7 140-159 106 106 101 181 181 172 F F F <---
115572024-03-06T21:02:36.957Z 8 160-179 103 103 103 175 175 175 F F F
115582024-03-06T21:02:36.957Z 9 180-199 104 104 104 177 177 177 F F F
115592024-03-06T21:02:36.957Z 10 200-219 105 105 105 178 178 178 T T F <---
115602024-03-06T21:02:36.957Z 11 220-239 102 102 102 173 173 173 F F F
115612024-03-06T21:02:36.957Z 12 240-259 95 95 95 163 163 163 F F F
115622024-03-06T21:02:36.958Z 13 260-279 98 98 98 168 168 168 F F F
115632024-03-06T21:02:36.958Z 14 280-299 91 91 91 156 156 156 F F F
115642024-03-06T21:02:36.958Z 15 300-319 106 106 101 181 181 172 F F F <---
115652024-03-06T21:02:36.958Z 16 320-339 100 100 100 171 171 171 F F F
115662024-03-06T21:02:36.958Z 17 340-359 104 104 104 177 177 177 F F F
115672024-03-06T21:02:36.958Z 18 360-379 105 105 105 178 178 178 F F F
115682024-03-06T21:02:36.958Z 19 380-399 104 104 104 177 177 177 F F F
115692024-03-06T21:02:36.958Z 20 400-419 106 106 102 181 181 173 F F F <---
115702024-03-06T21:02:36.958Z 21 420-439 104 104 104 177 177 177 F F F
115712024-03-06T21:02:36.958Z 22 440-459 102 102 102 173 173 173 F F F
115722024-03-06T21:02:36.958Z 23 460-479 78 78 78 137 137 137 F F F
115732024-03-06T21:02:36.958Z 24 480-499 101 101 101 172 172 172 F F F
115742024-03-06T21:02:36.958Z 25 500-519 105 105 105 178 178 178 F F F
115752024-03-06T21:02:36.958Z 26 520-539 102 102 102 174 174 174 F F F
115762024-03-06T21:02:36.958Z 27 540-559 98 98 98 169 169 169 F F F
115772024-03-06T21:02:36.958Z 28 560-579 104 104 104 176 176 176 F F F
115782024-03-06T21:02:36.958Z 29 580-599 99 99 99 170 170 170 F F F
115792024-03-06T21:02:36.970ZMax gen: 106, Max flush: 181
115802024-03-06T21:02:36.970ZError: Difference in extent metadata found!
115812024-03-06T21:02:36.970ZA Difference in extent metadata is expected here
115822024-03-06T21:02:36.970ZOn loop 53
115832024-03-06T21:02:36.970Z
115842024-03-06T21:02:36.971ZVerifying data now
115852024-03-06T21:02:41.516ZNo change after verify
115862024-03-06T21:02:41.516ZLoop: 53 Downstairs dump after verify (and repair):
115872024-03-06T21:02:41.538ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
115882024-03-06T21:02:41.538Z 0 000-019 106 106 106 181 181 181 F F F
115892024-03-06T21:02:41.538Z 1 020-039 106 106 106 180 180 180 F F F
115902024-03-06T21:02:41.538Z 2 040-059 100 100 100 171 171 171 F F F
115912024-03-06T21:02:41.538Z 3 060-079 107 107 107 182 182 182 F F F
115922024-03-06T21:02:41.538Z 4 080-099 106 106 106 181 181 181 F F F
115932024-03-06T21:02:41.539Z 5 100-119 106 106 106 179 179 179 F F F
115942024-03-06T21:02:41.539Z 6 120-139 106 106 106 180 180 180 F F F
115952024-03-06T21:02:41.539Z 7 140-159 106 106 106 181 181 181 F F F
115962024-03-06T21:02:41.539Z 8 160-179 103 103 103 175 175 175 F F F
115972024-03-06T21:02:41.539Z 9 180-199 104 104 104 177 177 177 F F F
115982024-03-06T21:02:41.539Z 10 200-219 107 107 107 182 182 182 F F F
115992024-03-06T21:02:41.539Z 11 220-239 102 102 102 173 173 173 F F F
116002024-03-06T21:02:41.539Z 12 240-259 95 95 95 163 163 163 F F F
116012024-03-06T21:02:41.539Z 13 260-279 98 98 98 168 168 168 F F F
116022024-03-06T21:02:41.539Z 14 280-299 91 91 91 156 156 156 F F F
116032024-03-06T21:02:41.539Z 15 300-319 106 106 106 181 181 181 F F F
116042024-03-06T21:02:41.539Z 16 320-339 100 100 100 171 171 171 F F F
116052024-03-06T21:02:41.539Z 17 340-359 104 104 104 177 177 177 F F F
116062024-03-06T21:02:41.539Z 18 360-379 105 105 105 178 178 178 F F F
116072024-03-06T21:02:41.544Z 19 380-399 104 104 104 177 177 177 F F F
116082024-03-06T21:02:41.544Z 20 400-419 106 106 106 181 181 181 F F F
116092024-03-06T21:02:41.544Z 21 420-439 104 104 104 177 177 177 F F F
116102024-03-06T21:02:41.544Z 22 440-459 102 102 102 173 173 173 F F F
116112024-03-06T21:02:41.544Z 23 460-479 78 78 78 137 137 137 F F F
116122024-03-06T21:02:41.544Z 24 480-499 101 101 101 172 172 172 F F F
116132024-03-06T21:02:41.545Z 25 500-519 105 105 105 178 178 178 F F F
116142024-03-06T21:02:41.545Z 26 520-539 102 102 102 174 174 174 F F F
116152024-03-06T21:02:41.545Z 27 540-559 98 98 98 169 169 169 F F F
116162024-03-06T21:02:41.545Z 28 560-579 104 104 104 176 176 176 F F F
116172024-03-06T21:02:41.545Z 29 580-599 99 99 99 170 170 170 F F F
116182024-03-06T21:02:41.545ZMax gen: 107, Max flush: 182
116192024-03-06T21:02:41.545Z
116202024-03-06T21:02:41.545ZBegin loop 54
116212024-03-06T21:02:41.545ZDownstairs to restart: 1
116222024-03-06T21:02:41.561Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.313404477Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571}
116232024-03-06T21:02:41.561Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.313660339Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571}
116242024-03-06T21:02:41.561Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.313685047Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571}
116252024-03-06T21:02:41.570Z{"msg":"Crucible 78173230-737a-4c6e-89c6-c2b59d7ec2d2 has session id: ba25a5d5-8c9a-4fce-a044-4492f919c03f","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.313709536Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116262024-03-06T21:02:41.570ZCrucible runtime is spawned
116272024-03-06T21:02:41.570Z{"msg":"Upstairs opts: Upstairs UUID: 78173230-737a-4c6e-89c6-c2b59d7ec2d2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.313859175Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116282024-03-06T21:02:41.570Z{"msg":"Crucible stats registered with UUID: 78173230-737a-4c6e-89c6-c2b59d7ec2d2","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.313923411Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116292024-03-06T21:02:41.571Z{"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.313973267Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571}
116302024-03-06T21:02:41.571Z{"msg":"78173230-737a-4c6e-89c6-c2b59d7ec2d2 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.314067481Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116312024-03-06T21:02:41.571Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.314097219Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"client":"0","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116322024-03-06T21:02:41.571Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.314114028Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"client":"0","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116332024-03-06T21:02:41.571Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.314128656Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"client":"1","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116342024-03-06T21:02:41.571Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.314144355Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"client":"1","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116352024-03-06T21:02:41.571Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.314161864Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"client":"2","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116362024-03-06T21:02:41.572Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.314186252Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"client":"2","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116372024-03-06T21:02:41.572Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.314212361Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116382024-03-06T21:02:41.572Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.314270897Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116392024-03-06T21:02:41.572Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.314355951Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116402024-03-06T21:02:41.572Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.314407907Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116412024-03-06T21:02:41.572Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.314437935Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116422024-03-06T21:02:41.573Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.314674539Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116432024-03-06T21:02:41.573Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.315165735Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"client":"2","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116442024-03-06T21:02:41.573Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.315219391Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"client":"2","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116452024-03-06T21:02:41.573Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.315248389Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"client":"0","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116462024-03-06T21:02:41.573Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.315276437Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"client":"0","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116472024-03-06T21:02:41.573Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.315601145Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"client":"1","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116482024-03-06T21:02:41.574Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.315665371Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"client":"1","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116492024-03-06T21:02:41.574Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.315884495Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"client":"0","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116502024-03-06T21:02:41.574Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.315922983Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"client":"2","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116512024-03-06T21:02:41.574Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.316184775Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"client":"1","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116522024-03-06T21:02:41.574Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.316538521Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"client":"2","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116532024-03-06T21:02:41.574Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.316571548Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"client":"2","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116542024-03-06T21:02:41.574Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.316586857Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116552024-03-06T21:02:41.575Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.316599956Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"client":"0","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116562024-03-06T21:02:41.575Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.316624164Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"client":"0","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116572024-03-06T21:02:41.575Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.316638104Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116582024-03-06T21:02:41.575Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.316723178Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"client":"1","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116592024-03-06T21:02:41.575Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.316751216Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"client":"1","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116602024-03-06T21:02:41.575Z{"msg":"[0]R flush_numbers[0..12]: [181, 180, 171, 182, 181, 179, 180, 181, 175, 177, 182, 173]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.316765955Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"":"downstairs","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116612024-03-06T21:02:41.576Z{"msg":"[0]R generation[0..12]: [106, 106, 100, 107, 106, 106, 106, 106, 103, 104, 107, 102]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.316780274Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"":"downstairs","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116622024-03-06T21:02:41.576Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.316794553Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"":"downstairs","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116632024-03-06T21:02:41.576Z{"msg":"[1]R flush_numbers[0..12]: [181, 180, 171, 182, 181, 179, 180, 181, 175, 177, 182, 173]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.316809412Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"":"downstairs","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116642024-03-06T21:02:41.576Z{"msg":"[1]R generation[0..12]: [106, 106, 100, 107, 106, 106, 106, 106, 103, 104, 107, 102]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.31683137Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"":"downstairs","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116652024-03-06T21:02:41.576Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.316847149Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"":"downstairs","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116662024-03-06T21:02:41.576Z{"msg":"[2]R flush_numbers[0..12]: [181, 180, 171, 182, 181, 179, 180, 181, 175, 177, 182, 173]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.316861828Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"":"downstairs","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116672024-03-06T21:02:41.576Z{"msg":"[2]R generation[0..12]: [106, 106, 100, 107, 106, 106, 106, 106, 103, 104, 107, 102]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.316875657Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"":"downstairs","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116682024-03-06T21:02:41.577Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.316898096Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"":"downstairs","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116692024-03-06T21:02:41.577Z{"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.316912115Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"":"downstairs","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116702024-03-06T21:02:41.577Z{"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.316926144Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"":"downstairs","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116712024-03-06T21:02:41.577Z{"msg":"Next flush: 183","v":0Wait for a query_work_queue command to finish before sending IO
116722024-03-06T21:02:41.577Z,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.316941833Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"":"downstairs","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116732024-03-06T21:02:41.577Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.316955602Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"":"downstairs","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116742024-03-06T21:02:41.577Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.316968931Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116752024-03-06T21:02:41.578Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.3169815Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"client":"0","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116762024-03-06T21:02:41.578Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.317001698Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"client":"1","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116772024-03-06T21:02:41.578Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.317024137Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"client":"2","":"downstairs","session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116782024-03-06T21:02:41.578Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.317038376Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116792024-03-06T21:02:41.578Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.317050915Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116802024-03-06T21:02:41.578Z{"msg":"78173230-737a-4c6e-89c6-c2b59d7ec2d2 is now active with session: ba25a5d5-8c9a-4fce-a044-4492f919c03f","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.317076883Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116812024-03-06T21:02:41.579Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.317089303Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571,"session_id":"ba25a5d5-8c9a-4fce-a044-4492f919c03f"}
116822024-03-06T21:02:41.579Z{"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:40.317103271Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571}
116832024-03-06T21:02:41.579ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
116842024-03-06T21:02:41.579ZLoading write count information from file "/tmp/test_repair_verify.data"
116852024-03-06T21:02:41.579ZRun Repair workload
116862024-03-06T21:02:41.579Z01/30 Write block 57 len 16384 data: 8 11 13 14
116872024-03-06T21:02:41.586Z02/30 Write block 189 len 16384 data: 10 11 9 10
116882024-03-06T21:02:41.586Z03/30 Read block 117 len 24576
116892024-03-06T21:02:41.586Z04/30 Flush
116902024-03-06T21:02:41.586Z05/30 Write block 436 len 8192 data: 10 12
116912024-03-06T21:02:41.586Z06/30 Write block 474 len 20480 data: 4 4 5 5 6
116922024-03-06T21:02:41.587Z07/30 Write block 308 len 32768 data: 10 9 9 9 9 9 9 9
116932024-03-06T21:02:41.587Z08/30 Flush
116942024-03-06T21:02:41.587Z09/30 Write block 456 len 40960 data: 6 9 8 8 9 12 11 13 13 13
116952024-03-06T21:02:41.587Z10/30 Read block 336 len 32768
116962024-03-06T21:02:41.587Z11/30 Write block 510 len 28672 data: 10 9 9 9 9 11 11
116972024-03-06T21:02:41.587Z12/30 Read block 129 len 36864
116982024-03-06T21:02:41.587Z13/30 Read block 369 len 4096
116992024-03-06T21:02:41.587Z14/30 Read block 190 len 4096
117002024-03-06T21:02:41.587Z15/30 Read block 330 len 8192
117012024-03-06T21:02:41.588Z16/30 Write block 302 len 32768 data: 7 7 8 10 11 11 11 10
117022024-03-06T21:02:41.588Z17/30 Read block 122 len 24576
117032024-03-06T21:02:41.592Z18/30 Write block 534 len 8192 data: 7 9
117042024-03-06T21:02:41.594Z19/30 Read block 577 len 12288
117052024-03-06T21:02:41.594Z20/30 Write block 152 len 20480 data: 8 8 8 9 12
117062024-03-06T21:02:41.594Z21/30 Flush
117072024-03-06T21:02:41.594Z22/30 Write block 245 len 40960 data: 7 5 5 5 5 4 4 4 4 4
117082024-03-06T21:02:41.602Z23/30 Read block 446 len 4096
117092024-03-06T21:02:41.602Z24/30 Read block 398 len 36864
117102024-03-06T21:02:41.603Z25/30 Read block 173 len 24576
117112024-03-06T21:02:41.603Z26/30 Write block 594 len 8192 data: 10 8
117122024-03-06T21:02:41.603Z27/30 Read block 188 len 32768
117132024-03-06T21:02:41.603Z28/30 Write block 383 len 32768 data: 10 12 12 13 10 10 9 11
117142024-03-06T21:02:41.603Z29/30 Write block 155 len 12288 data: 10 13 13
117152024-03-06T21:02:41.604Z30/30 Write block 528 len 4096 data: 10
117162024-03-06T21:02:41.611Z----------------------------------------------------------------
117172024-03-06T21:02:41.611Z Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:26
117182024-03-06T21:02:41.612ZStates: Active Active Active
117192024-03-06T21:02:41.612ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
117202024-03-06T21:02:41.612Z 5 Acked 1004 Write 2 Done Sent Done false
117212024-03-06T21:02:41.612Z 6 Acked 1005 Write 5 Done Sent Done false
117222024-03-06T21:02:41.612Z 7 Acked 1006 Write 8 Done Sent Done false
117232024-03-06T21:02:41.612Z 8 Acked 1007 Flush 0 Done Sent Done false
117242024-03-06T21:02:41.613Z 9 Acked 1008 Write 10 Done Sent Done false
117252024-03-06T21:02:41.613Z 10 Acked 1009 Read 8 Done Sent Done false
117262024-03-06T21:02:41.613Z 11 Acked 1010 Write 7 Done Sent Done false
117272024-03-06T21:02:41.613Z 12 Acked 1011 Read 9 Done Sent Done false
117282024-03-06T21:02:41.613Z 13 Acked 1012 Read 1 Done Sent Done false
117292024-03-06T21:02:41.613Z 14 Acked 1013 Read 1 Done Sent Done false
117302024-03-06T21:02:41.614Z 15 Acked 1014 Read 2 Done Sent Done false
117312024-03-06T21:02:41.614Z 16 Acked 1015 Write 8 Done Sent Done false
117322024-03-06T21:02:41.614Z 17 Acked 1016 Read 6 Done Sent Done false
117332024-03-06T21:02:41.614Z 18 Acked 1017 Write 2 Done Sent Done false
117342024-03-06T21:02:41.614Z 19 Acked 1018 Read 3 Done Sent Done false
117352024-03-06T21:02:41.614Z 20 Acked 1019 Write 5 Done Sent Done false
117362024-03-06T21:02:41.614Z 21 Acked 1020 Flush 0 Done Sent Done false
117372024-03-06T21:02:41.615Z 22 Acked 1021 Write 10 Done Sent Done false
117382024-03-06T21:02:41.615Z 23 Acked 1022 Read 1 Done Sent Done false
117392024-03-06T21:02:41.615Z 24 Acked 1023 Read 9 Done Sent Done false
117402024-03-06T21:02:41.615Z 25 Acked 1024 Read 6 Done Sent Done false
117412024-03-06T21:02:41.615Z 26 Acked 1025 Write 2 Done Sent Done false
117422024-03-06T21:02:41.615Z 27 Acked 1026 Read 8 Done Sent Done false
117432024-03-06T21:02:41.615Z 28 Acked 1027 Write 8 Done Sent Done false
117442024-03-06T21:02:41.615Z 29 Acked 1028 Write 3 Sent Sent Sent false
117452024-03-06T21:02:41.615Z 30 Acked 1029 Write 1 Sent Sent Sent false
117462024-03-06T21:02:41.615Z STATES DS:0 DS:1 DS:2 TOTAL
117472024-03-06T21:02:41.616Z New 0 0 0 0
117482024-03-06T21:02:41.616Z Sent 2 26 2 30
117492024-03-06T21:02:41.616Z Done 24 0 24 48
117502024-03-06T21:02:41.616Z Skipped 0 0 0 0
117512024-03-06T21:02:41.616Z Error 0 0 0 0
117522024-03-06T21:02:41.616ZLast Flush: 1020 1003 1020
117532024-03-06T21:02:41.616ZDownstairs last five completed: 1003 1002 1001 1000
117542024-03-06T21:02:41.617ZUpstairs last five completed: 30 29 28 27 26
117552024-03-06T21:02:41.617Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:02:40.354309087Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1571}
117562024-03-06T21:02:41.617ZWrote out file "/tmp/test_repair_verify.data"
117572024-03-06T21:02:41.617Z
117582024-03-06T21:02:41.622ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
117592024-03-06T21:02:41.631ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
117602024-03-06T21:02:41.637Z 0 000-019 106 106 106 181 181 181 F F F
117612024-03-06T21:02:41.637Z 1 020-039 106 106 106 180 180 180 F F F
117622024-03-06T21:02:41.637Z 2 040-059 108 108 108 183 183 183 F F F
117632024-03-06T21:02:41.637Z 3 060-079 108 108 108 183 183 183 F F F
117642024-03-06T21:02:41.637Z 4 080-099 106 106 106 181 181 181 F F F
117652024-03-06T21:02:41.637Z 5 100-119 106 106 106 179 179 179 F F F
117662024-03-06T21:02:41.637Z 6 120-139 106 106 106 180 180 180 F F F
117672024-03-06T21:02:41.637Z 7 140-159 108 106 108 185 181 185 T F T <---
117682024-03-06T21:02:41.638Z 8 160-179 103 103 103 175 175 175 F F F
117692024-03-06T21:02:41.638Z 9 180-199 108 108 108 183 183 183 F F F
117702024-03-06T21:02:41.638Z 10 200-219 107 107 107 182 182 182 F F F
117712024-03-06T21:02:41.638Z 11 220-239 102 102 102 173 173 173 F F F
117722024-03-06T21:02:41.638Z 12 240-259 95 95 95 163 163 163 T F T <---
117732024-03-06T21:02:41.638Z 13 260-279 98 98 98 168 168 168 F F F
117742024-03-06T21:02:41.638Z 14 280-299 91 91 91 156 156 156 F F F
117752024-03-06T21:02:41.638Z 15 300-319 108 106 108 185 181 185 F F F <---
117762024-03-06T21:02:41.638Z 16 320-339 100 100 100 171 171 171 F F F
117772024-03-06T21:02:41.638Z 17 340-359 104 104 104 177 177 177 F F F
117782024-03-06T21:02:41.638Z 18 360-379 105 105 105 178 178 178 F F F
117792024-03-06T21:02:41.638Z 19 380-399 104 104 104 177 177 177 T F T <---
117802024-03-06T21:02:41.639Z 20 400-419 106 106 106 181 181 181 F F F
117812024-03-06T21:02:41.639Z 21 420-439 108 104 108 184 177 184 F F F <---
117822024-03-06T21:02:41.639Z 22 440-459 108 102 108 185 173 185 F F F <---
117832024-03-06T21:02:41.639Z 23 460-479 108 78 108 185 137 185 F F F <---
117842024-03-06T21:02:41.639Z 24 480-499 101 101 101 172 172 172 F F F
117852024-03-06T21:02:41.639Z 25 500-519 108 105 108 185 178 185 F F F <---
117862024-03-06T21:02:41.639Z 26 520-539 108 102 108 185 174 185 T F T <---
117872024-03-06T21:02:41.639Z 27 540-559 98 98 98 169 169 169 F F F
117882024-03-06T21:02:41.639Z 28 560-579 104 104 104 176 176 176 F F F
117892024-03-06T21:02:41.639Z 29 580-599 99 99 99 170 170 170 T F T <---
117902024-03-06T21:02:41.639ZMax gen: 108, Max flush: 185
117912024-03-06T21:02:41.639ZError: Difference in extent metadata found!
117922024-03-06T21:02:41.639ZA Difference in extent metadata is expected here
117932024-03-06T21:02:41.640ZOn loop 54
117942024-03-06T21:02:41.640Z
117952024-03-06T21:02:41.648ZVerifying data now
117962024-03-06T21:02:46.205ZNo change after verify
117972024-03-06T21:02:46.205ZLoop: 54 Downstairs dump after verify (and repair):
117982024-03-06T21:02:46.221ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
117992024-03-06T21:02:46.221Z 0 000-019 106 106 106 181 181 181 F F F
118002024-03-06T21:02:46.221Z 1 020-039 106 106 106 180 180 180 F F F
118012024-03-06T21:02:46.221Z 2 040-059 108 108 108 183 183 183 F F F
118022024-03-06T21:02:46.221Z 3 060-079 108 108 108 183 183 183 F F F
118032024-03-06T21:02:46.221Z 4 080-099 106 106 106 181 181 181 F F F
118042024-03-06T21:02:46.221Z 5 100-119 106 106 106 179 179 179 F F F
118052024-03-06T21:02:46.221Z 6 120-139 106 106 106 180 180 180 F F F
118062024-03-06T21:02:46.221Z 7 140-159 109 109 109 186 186 186 F F F
118072024-03-06T21:02:46.221Z 8 160-179 103 103 103 175 175 175 F F F
118082024-03-06T21:02:46.221Z 9 180-199 108 108 108 183 183 183 F F F
118092024-03-06T21:02:46.221Z 10 200-219 107 107 107 182 182 182 F F F
118102024-03-06T21:02:46.221Z 11 220-239 102 102 102 173 173 173 F F F
118112024-03-06T21:02:46.221Z 12 240-259 109 109 109 186 186 186 F F F
118122024-03-06T21:02:46.221Z 13 260-279 98 98 98 168 168 168 F F F
118132024-03-06T21:02:46.221Z 14 280-299 91 91 91 156 156 156 F F F
118142024-03-06T21:02:46.221Z 15 300-319 108 108 108 185 185 185 F F F
118152024-03-06T21:02:46.221Z 16 320-339 100 100 100 171 171 171 F F F
118162024-03-06T21:02:46.221Z 17 340-359 104 104 104 177 177 177 F F F
118172024-03-06T21:02:46.221Z 18 360-379 105 105 105 178 178 178 F F F
118182024-03-06T21:02:46.221Z 19 380-399 109 109 109 186 186 186 F F F
118192024-03-06T21:02:46.222Z 20 400-419 106 106 106 181 181 181 F F F
118202024-03-06T21:02:46.222Z 21 420-439 108 108 108 184 184 184 F F F
118212024-03-06T21:02:46.222Z 22 440-459 108 108 108 185 185 185 F F F
118222024-03-06T21:02:46.222Z 23 460-479 108 108 108 185 185 185 F F F
118232024-03-06T21:02:46.222Z 24 480-499 101 101 101 172 172 172 F F F
118242024-03-06T21:02:46.222Z 25 500-519 108 108 108 185 185 185 F F F
118252024-03-06T21:02:46.225Z 26 520-539 109 109 109 186 186 186 F F F
118262024-03-06T21:02:46.225Z 27 540-559 98 98 98 169 169 169 F F F
118272024-03-06T21:02:46.225Z 28 560-579 104 104 104 176 176 176 F F F
118282024-03-06T21:02:46.225Z 29 580-599 109 109 109 186 186 186 F F F
118292024-03-06T21:02:46.225ZMax gen: 109, Max flush: 186
118302024-03-06T21:02:46.228Z
118312024-03-06T21:02:46.228ZBegin loop 55
118322024-03-06T21:02:46.228ZDownstairs to restart: 1
118332024-03-06T21:02:46.248Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.000958534Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579}
118342024-03-06T21:02:46.249Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.001268533Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579}
118352024-03-06T21:02:46.249Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.001295681Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579}
118362024-03-06T21:02:46.249Z{"msg":"Crucible runtime is spawned
118372024-03-06T21:02:46.256ZCrucible 75a98626-6058-4f3c-9fda-d098fe5f2438 has session id: e74e558b-72aa-40aa-9695-14ca1343b18b","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.001341228Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118382024-03-06T21:02:46.256Z{"msg":"Upstairs opts: Upstairs UUID: 75a98626-6058-4f3c-9fda-d098fe5f2438, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.001394744Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118392024-03-06T21:02:46.256Z{"msg":"Crucible stats registered with UUID: 75a98626-6058-4f3c-9fda-d098fe5f2438","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.001429002Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118402024-03-06T21:02:46.256Z{"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.00145266Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579}
118412024-03-06T21:02:46.256Z{"msg":"75a98626-6058-4f3c-9fda-d098fe5f2438 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.001536274Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118422024-03-06T21:02:46.256Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.001579251Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"client":"0","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118432024-03-06T21:02:46.256Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.00160099Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"client":"0","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118442024-03-06T21:02:46.256Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.001638087Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"client":"1","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118452024-03-06T21:02:46.256Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.001669805Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"client":"1","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118462024-03-06T21:02:46.256Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.001711852Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"client":"2","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118472024-03-06T21:02:46.256Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.00173654Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"client":"2","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118482024-03-06T21:02:46.256Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.001767538Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"":"io task","client":"0","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118492024-03-06T21:02:46.256Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.001799046Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"":"io task","client":"1","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118502024-03-06T21:02:46.256Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.001852212Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"":"io task","client":"2","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118512024-03-06T21:02:46.256Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.00188862Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"":"io task","client":"0","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118522024-03-06T21:02:46.256Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.002105005Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"":"io task","client":"2","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118532024-03-06T21:02:46.257Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.002148682Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"":"io task","client":"1","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118542024-03-06T21:02:46.257Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.002831545Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"client":"0","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118552024-03-06T21:02:46.257Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.002897711Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"client":"0","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118562024-03-06T21:02:46.257Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.002982065Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"client":"2","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118572024-03-06T21:02:46.257Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.003026502Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"client":"2","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118582024-03-06T21:02:46.257Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.003564874Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"client":"2","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118592024-03-06T21:02:46.257Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.003619661Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"client":"0","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118602024-03-06T21:02:46.257Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.003764081Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"client":"1","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118612024-03-06T21:02:46.257Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.003805008Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"client":"1","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118622024-03-06T21:02:46.257Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.004236688Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"client":"2","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118632024-03-06T21:02:46.257Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.004279885Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"client":"2","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118642024-03-06T21:02:46.257Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.004312123Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118652024-03-06T21:02:46.257Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.004333552Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"client":"0","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118662024-03-06T21:02:46.257Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.004348091Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"client":"0","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118672024-03-06T21:02:46.257Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.004368739Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118682024-03-06T21:02:46.257Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.004555726Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"client":"1","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118692024-03-06T21:02:46.257Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.004959938Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"client":"1","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118702024-03-06T21:02:46.257Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.005014254Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"client":"1","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118712024-03-06T21:02:46.257Z{"msg":"[0]R flush_numbers[0..12]: [181, 180, 183, 183, 181, 179, 180, 186, 175, 183, 182, 173]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.005043583Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"":"downstairs","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118722024-03-06T21:02:46.257Z{"msg":"[0]R generation[0..12]: [106, 106, 108, 108, 106, 106, 106, 109, 103, 108, 107, 102]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.005068341Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"":"downstairs","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118732024-03-06T21:02:46.257Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.00508434Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"":"downstairs","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118742024-03-06T21:02:46.257Z{"msg":"[1]R flush_numbers[0..12]: [181, 180, 183, 183, 181, 179, 180, 186, 175, 183, 182, 173]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.005105728Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"":"downstairs","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118752024-03-06T21:02:46.257Z{"msg":"[1]R generation[0..12]: [106, 106, 108, 108, 106, 106, 106, 109, 103, 108, 107, 102]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.005120397Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"":"downstairs","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118762024-03-06T21:02:46.257Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.005141776Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"":"downstairs","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118772024-03-06T21:02:46.258Z{"msg":"[2]R flush_numbers[0..12]: [181, 180, 183, 183, 181, 179, 180, 186, 175, 183, 182, 173]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.005156495Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"":"downstairs","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118782024-03-06T21:02:46.258Z{"msg":"[2]R generation[0..12]: [106, 106, 108, 108, 106, 106, 106, 109, 103, 108, 107, 102]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
118792024-03-06T21:02:46.258Z,"time":"2024-03-06T21:02:45.005176973Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"":"downstairs","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118802024-03-06T21:02:46.258Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.005191972Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"":"downstairs","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118812024-03-06T21:02:46.258Z{"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.005211751Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"":"downstairs","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118822024-03-06T21:02:46.258Z{"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.00522576Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"":"downstairs","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118832024-03-06T21:02:46.258Z{"msg":"Next flush: 187","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.005245349Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"":"downstairs","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118842024-03-06T21:02:46.258Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.005258938Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"":"downstairs","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118852024-03-06T21:02:46.258Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.005272477Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118862024-03-06T21:02:46.258Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.005293925Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"client":"0","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118872024-03-06T21:02:46.258Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.005307924Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"client":"1","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118882024-03-06T21:02:46.258Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.005339932Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"client":"2","":"downstairs","session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118892024-03-06T21:02:46.258Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.0053659Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118902024-03-06T21:02:46.258Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.005395238Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118912024-03-06T21:02:46.258Z{"msg":"75a98626-6058-4f3c-9fda-d098fe5f2438 is now active with session: e74e558b-72aa-40aa-9695-14ca1343b18b","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.005429586Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118922024-03-06T21:02:46.258Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.005457404Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579,"session_id":"e74e558b-72aa-40aa-9695-14ca1343b18b"}
118932024-03-06T21:02:46.258Z{"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:45.005487062Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579}
118942024-03-06T21:02:46.258ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
118952024-03-06T21:02:46.262ZLoading write count information from file "/tmp/test_repair_verify.data"
118962024-03-06T21:02:46.262ZRun Repair workload
118972024-03-06T21:02:46.262Z01/30 Flush
118982024-03-06T21:02:46.262Z02/30 Write block 497 len 4096 data: 8
118992024-03-06T21:02:46.262Z03/30 Read block 356 len 20480
119002024-03-06T21:02:46.262Z04/30 Write block 404 len 36864 data: 5 8 9 11 12 10 9 7 5
119012024-03-06T21:02:46.262Z05/30 Write block 78 len 28672 data: 6 7 7 7 8 8 6
119022024-03-06T21:02:46.262Z06/30 Read block 192 len 40960
119032024-03-06T21:02:46.262Z07/30 Flush
119042024-03-06T21:02:46.266Z08/30 Write block 119 len 28672 data: 6 5 5 6 5 8 12
119052024-03-06T21:02:46.266Z09/30 Read block 389 len 12288
119062024-03-06T21:02:46.266Z10/30 Read block 237 len 20480
119072024-03-06T21:02:46.269Z11/30 Write block 29 len 20480 data: 8 8 7 8 8
119082024-03-06T21:02:46.269Z12/30 Flush
119092024-03-06T21:02:46.272Z13/30 Write block 53 len 28672 data: 7 5 8 6 9 12 14
119102024-03-06T21:02:46.272Z14/30 Write block 458 len 8192 data: 9 9
119112024-03-06T21:02:46.272Z15/30 Read block 40 len 36864
119122024-03-06T21:02:46.275Z16/30 Read block 258 len 16384
119132024-03-06T21:02:46.275Z17/30 Write block 189 len 8192 data: 11 12
119142024-03-06T21:02:46.275Z18/30 Read block 234 len 28672
119152024-03-06T21:02:46.275Z19/30 Flush
119162024-03-06T21:02:46.278Z20/30 Write block 12 len 36864 data: 8 8 7 9 7 8 10 12 14
119172024-03-06T21:02:46.278Z21/30 Read block 577 len 24576
119182024-03-06T21:02:46.282Z22/30 Write block 95 len 20480 data: 6 8 7 8 8
119192024-03-06T21:02:46.282Z23/30 Write block 484 len 16384 data: 8 10 8 8
119202024-03-06T21:02:46.282Z24/30 Flush
119212024-03-06T21:02:46.288Z25/30 Flush
119222024-03-06T21:02:46.288Z26/30 Write block 45 len 4096 data: 6
119232024-03-06T21:02:46.292Z27/30 Write block 237 len 40960 data: 13 14 14 10 11 10 8 8 8 6
119242024-03-06T21:02:46.292Z28/30 Write block 320 len 36864 data: 8 8 8 7 8 7 8 6 6
119252024-03-06T21:02:46.292Z29/30 Write block 343 len 8192 data: 10 11
119262024-03-06T21:02:46.292Z30/30 Read block 403 len 32768
119272024-03-06T21:02:46.292Z----------------------------------------------------------------
119282024-03-06T21:02:46.292Z Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:29
119292024-03-06T21:02:46.292ZStates: Active Active Active
119302024-03-06T21:02:46.292ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
119312024-03-06T21:02:46.292Z 2 Acked 1001 Write 1 Done Done Done false
119322024-03-06T21:02:46.292Z 3 Acked 1002 Read 5 Done Done Done false
119332024-03-06T21:02:46.292Z 4 Acked 1003 Write 9 Done Done Done false
119342024-03-06T21:02:46.292Z 5 Acked 1004 Write 7 Done Sent Done false
119352024-03-06T21:02:46.292Z 6 Acked 1005 Read 10 Done Sent Done false
119362024-03-06T21:02:46.292Z 7 Acked 1006 Flush 0 Done Sent Done false
119372024-03-06T21:02:46.292Z 8 Acked 1007 Write 7 Done Sent Done false
119382024-03-06T21:02:46.292Z 9 Acked 1008 Read 3 Done Sent Done false
119392024-03-06T21:02:46.292Z 10 Acked 1009 Read 5 Done Sent Done false
119402024-03-06T21:02:46.292Z 11 Acked 1010 Write 5 Done Sent Done false
119412024-03-06T21:02:46.292Z 12 Acked 1011 Flush 0 Done Sent Done false
119422024-03-06T21:02:46.292Z 13 Acked 1012 Write 7 Done Sent Done false
119432024-03-06T21:02:46.292Z 14 Acked 1013 Write 2 Done Sent Done false
119442024-03-06T21:02:46.293Z 15 Acked 1014 Read 9 Done Sent Done false
119452024-03-06T21:02:46.293Z 16 Acked 1015 Read 4 Done Sent Done false
119462024-03-06T21:02:46.293Z 17 Acked 1016 Write 2 Done Sent Done false
119472024-03-06T21:02:46.293Z 18 Acked 1017 Read 7 Done Sent Done false
119482024-03-06T21:02:46.293Z 19 Acked 1018 Flush 0 Done Sent Done false
119492024-03-06T21:02:46.293Z 20 Acked 1019 Write 9 Done Sent Done false
119502024-03-06T21:02:46.293Z 21 Acked 1020 Read 6 Done Sent Done false
119512024-03-06T21:02:46.293Z 22 Acked 1021 Write 5 Done Sent Done false
119522024-03-06T21:02:46.293Z 23 Acked 1022 Write 4 Done Sent Done false
119532024-03-06T21:02:46.293Z 24 Acked 1023 Flush 0 Done Sent Done false
119542024-03-06T21:02:46.293Z 25 Acked 1024 Flush 0 Done Sent Done false
119552024-03-06T21:02:46.293Z 26 Acked 1025 Write 1 Done Sent Done false
119562024-03-06T21:02:46.293Z 27 Acked 1026 Write 10 Done Sent Done false
119572024-03-06T21:02:46.293Z 28 Acked 1027 Write 9 Done Sent Done false
119582024-03-06T21:02:46.293Z 29 Acked 1028 Write 2 Done Sent Done false
119592024-03-06T21:02:46.293Z 30 Acked 1029 Read 8 Done Sent Done false
119602024-03-06T21:02:46.293Z STATES DS:0 DS:1 DS:2 TOTAL
119612024-03-06T21:02:46.293Z New 0 0 0 0
119622024-03-06T21:02:46.293Z Sent 0 26 0 26
119632024-03-06T21:02:46.293Z Done 29 3 29 61
119642024-03-06T21:02:46.293Z Skipped 0 0 0 0
119652024-03-06T21:02:46.293Z Error 0 0 0 0
119662024-03-06T21:02:46.293ZLast Flush: 1024 1000 1024
119672024-03-06T21:02:46.293ZDownstairs last five completed: 1000
119682024-03-06T21:02:46.294ZUpstairs last five completed: 30 29 28 27 26
119692024-03-06T21:02:46.294Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:02:45.043549119Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1579}
119702024-03-06T21:02:46.294ZWrote out file "/tmp/test_repair_verify.data"
119712024-03-06T21:02:46.300Z
119722024-03-06T21:02:46.311ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
119732024-03-06T21:02:46.323ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
119742024-03-06T21:02:46.323Z 0 000-019 110 106 110 191 181 191 F F F <---
119752024-03-06T21:02:46.323Z 1 020-039 110 106 110 191 180 191 F F F <---
119762024-03-06T21:02:46.323Z 2 040-059 110 108 110 190 183 190 T F T <---
119772024-03-06T21:02:46.323Z 3 060-079 110 108 110 188 183 188 F F F <---
119782024-03-06T21:02:46.323Z 4 080-099 110 106 110 191 181 191 F F F <---
119792024-03-06T21:02:46.324Z 5 100-119 110 106 110 189 179 189 F F F <---
119802024-03-06T21:02:46.324Z 6 120-139 110 106 110 189 180 189 F F F <---
119812024-03-06T21:02:46.324Z 7 140-159 109 109 109 186 186 186 F F F
119822024-03-06T21:02:46.324Z 8 160-179 103 103 103 175 175 175 F F F
119832024-03-06T21:02:46.324Z 9 180-199 110 108 110 190 183 190 F F F <---
119842024-03-06T21:02:46.324Z 10 200-219 107 107 107 182 182 182 F F F
119852024-03-06T21:02:46.330Z 11 220-239 102 102 102 173 173 173 T F T <---
119862024-03-06T21:02:46.330Z 12 240-259 109 109 109 186 186 186 T F T <---
119872024-03-06T21:02:46.330Z 13 260-279 98 98 98 168 168 168 F F F
119882024-03-06T21:02:46.330Z 14 280-299 91 91 91 156 156 156 F F F
119892024-03-06T21:02:46.330Z 15 300-319 108 108 108 185 185 185 F F F
119902024-03-06T21:02:46.330Z 16 320-339 100 100 100 171 171 171 T F T <---
119912024-03-06T21:02:46.330Z 17 340-359 104 104 104 177 177 177 T F T <---
119922024-03-06T21:02:46.330Z 18 360-379 105 105 105 178 178 178 F F F
119932024-03-06T21:02:46.330Z 19 380-399 109 109 109 186 186 186 F F F
119942024-03-06T21:02:46.330Z 20 400-419 110 106 110 188 181 188 F T F <---
119952024-03-06T21:02:46.330Z 21 420-439 108 108 108 184 184 184 F F F
119962024-03-06T21:02:46.330Z 22 440-459 110 108 110 190 185 190 F F F <---
119972024-03-06T21:02:46.331Z 23 460-479 108 108 108 185 185 185 F F F
119982024-03-06T21:02:46.331Z 24 480-499 110 101 110 191 172 191 F T F <---
119992024-03-06T21:02:46.331Z 25 500-519 108 108 108 185 185 185 F F F
120002024-03-06T21:02:46.331Z 26 520-539 109 109 109 186 186 186 F F F
120012024-03-06T21:02:46.331Z 27 540-559 98 98 98 169 169 169 F F F
120022024-03-06T21:02:46.331Z 28 560-579 104 104 104 176 176 176 F F F
120032024-03-06T21:02:46.331Z 29 580-599 109 109 109 186 186 186 F F F
120042024-03-06T21:02:46.331ZMax gen: 110, Max flush: 191
120052024-03-06T21:02:46.331ZError: Difference in extent metadata found!
120062024-03-06T21:02:46.331ZA Difference in extent metadata is expected here
120072024-03-06T21:02:46.331ZOn loop 55
120082024-03-06T21:02:46.331Z
120092024-03-06T21:02:46.331ZVerifying data now
120102024-03-06T21:02:52.095ZNo change after verify
120112024-03-06T21:02:52.095ZLoop: 55 Downstairs dump after verify (and repair):
120122024-03-06T21:02:52.110ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
120132024-03-06T21:02:52.110Z 0 000-019 110 110 110 191 191 191 F F F
120142024-03-06T21:02:52.110Z 1 020-039 110 110 110 191 191 191 F F F
120152024-03-06T21:02:52.110Z 2 040-059 111 111 111 192 192 192 F F F
120162024-03-06T21:02:52.110Z 3 060-079 110 110 110 188 188 188 F F F
120172024-03-06T21:02:52.110Z 4 080-099 110 110 110 191 191 191 F F F
120182024-03-06T21:02:52.110Z 5 100-119 110 110 110 189 189 189 F F F
120192024-03-06T21:02:52.110Z 6 120-139 110 110 110 189 189 189 F F F
120202024-03-06T21:02:52.111Z 7 140-159 109 109 109 186 186 186 F F F
120212024-03-06T21:02:52.111Z 8 160-179 103 103 103 175 175 175 F F F
120222024-03-06T21:02:52.111Z 9 180-199 110 110 110 190 190 190 F F F
120232024-03-06T21:02:52.111Z 10 200-219 107 107 107 182 182 182 F F F
120242024-03-06T21:02:52.111Z 11 220-239 111 111 111 192 192 192 F F F
120252024-03-06T21:02:52.111Z 12 240-259 111 111 111 192 192 192 F F F
120262024-03-06T21:02:52.111Z 13 260-279 98 98 98 168 168 168 F F F
120272024-03-06T21:02:52.111Z 14 280-299 91 91 91 156 156 156 F F F
120282024-03-06T21:02:52.111Z 15 300-319 108 108 108 185 185 185 F F F
120292024-03-06T21:02:52.111Z 16 320-339 111 111 111 192 192 192 F F F
120302024-03-06T21:02:52.111Z 17 340-359 111 111 111 192 192 192 F F F
120312024-03-06T21:02:52.118Z 18 360-379 105 105 105 178 178 178 F F F
120322024-03-06T21:02:52.118Z 19 380-399 109 109 109 186 186 186 F F F
120332024-03-06T21:02:52.118Z 20 400-419 110 110 110 188 188 188 F F F
120342024-03-06T21:02:52.119Z 21 420-439 108 108 108 184 184 184 F F F
120352024-03-06T21:02:52.119Z 22 440-459 110 110 110 190 190 190 F F F
120362024-03-06T21:02:52.119Z 23 460-479 108 108 108 185 185 185 F F F
120372024-03-06T21:02:52.119Z 24 480-499 110 110 110 191 191 191 F F F
120382024-03-06T21:02:52.119Z 25 500-519 108 108 108 185 185 185 F F F
120392024-03-06T21:02:52.119Z 26 520-539 109 109 109 186 186 186 F F F
120402024-03-06T21:02:52.119Z 27 540-559 98 98 98 169 169 169 F F F
120412024-03-06T21:02:52.119Z 28 560-579 104 104 104 176 176 176 F F F
120422024-03-06T21:02:52.119Z 29 580-599 109 109 109 186 186 186 F F F
120432024-03-06T21:02:52.119ZMax gen: 111, Max flush: 192
120442024-03-06T21:02:52.119Z
120452024-03-06T21:02:52.119ZBegin loop 56
120462024-03-06T21:02:52.119ZDownstairs to restart: 2
120472024-03-06T21:02:52.137Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.887992547Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587}
120482024-03-06T21:02:52.137Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.888253109Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587}
120492024-03-06T21:02:52.138Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.888280368Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587}
120502024-03-06T21:02:52.138Z{"msg":"Crucible cd34f64a-c683-48e4-bc6b-418d8db37677 has session id: ebf58839-75ee-4cc3-b12c-84f6b09b5c95","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.888305206Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120512024-03-06T21:02:52.138ZCrucible runtime is spawned
120522024-03-06T21:02:52.150Z{"msg":"Upstairs opts: Upstairs UUID: cd34f64a-c683-48e4-bc6b-418d8db37677, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.888367682Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120532024-03-06T21:02:52.150Z{"msg":"Crucible stats registered with UUID: cd34f64a-c683-48e4-bc6b-418d8db37677","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.888419778Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120542024-03-06T21:02:52.150Z{"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.888453476Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587}
120552024-03-06T21:02:52.150Z{"msg":"cd34f64a-c683-48e4-bc6b-418d8db37677 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.888578737Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120562024-03-06T21:02:52.150Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.888621234Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"client":"0","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120572024-03-06T21:02:52.150Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.888650962Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"client":"0","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120582024-03-06T21:02:52.150Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.88867734Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"client":"1","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120592024-03-06T21:02:52.150Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.888713988Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"client":"1","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120602024-03-06T21:02:52.150Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.888751195Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"client":"2","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120612024-03-06T21:02:52.150Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.888775123Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"client":"2","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120622024-03-06T21:02:52.150Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.888807401Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"":"io task","client":"0","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120632024-03-06T21:02:52.151Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.888845829Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"":"io task","client":"1","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120642024-03-06T21:02:52.151Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.888953911Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"":"io task","client":"2","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120652024-03-06T21:02:52.151Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.889019986Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"":"io task","client":"0","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120662024-03-06T21:02:52.151Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.889060714Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"":"io task","client":"1","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120672024-03-06T21:02:52.151Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.889097961Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"":"io task","client":"2","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120682024-03-06T21:02:52.151Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.889936013Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"client":"0","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120692024-03-06T21:02:52.151Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.88998816Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"client":"0","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120702024-03-06T21:02:52.151Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.890044786Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"client":"1","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120712024-03-06T21:02:52.151Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.890090543Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"client":"1","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120722024-03-06T21:02:52.151Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.890579789Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"client":"2","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120732024-03-06T21:02:52.151Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.890634555Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"client":"2","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120742024-03-06T21:02:52.151Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.890745548Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"client":"1","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120752024-03-06T21:02:52.151Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.890786045Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"client":"0","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120762024-03-06T21:02:52.152Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.891159569Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"client":"2","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120772024-03-06T21:02:52.152Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.891411292Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"client":"0","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120782024-03-06T21:02:52.152Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.891476267Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"client":"0","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120792024-03-06T21:02:52.152Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.891516014Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120802024-03-06T21:02:52.152Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.891539653Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"client":"1","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120812024-03-06T21:02:52.152Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.891571421Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"client":"1","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120822024-03-06T21:02:52.152Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.891595629Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120832024-03-06T21:02:52.152Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.891663304Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"client":"2","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120842024-03-06T21:02:52.152Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.891709871Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"client":"2","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120852024-03-06T21:02:52.152Z{"msg":"[0]R flush_numbers[0..12]: [191, 191, 192, 188, 191, 189, 189, 186, 175, 190, 182, 192]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.891735509Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"":"downstairs","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120862024-03-06T21:02:52.152Z{"msg":"[0]R generation[0..12]: [110, 110, 111, 110, 110, 110, 110, 109, 103, 110, 107, 111]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.891762468Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"":"downstairs","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120872024-03-06T21:02:52.152Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.891778096Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"":"downstairs","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120882024-03-06T21:02:52.152Z{"msg":"[1]R flush_numbers[0..12]: [191, 191, 192, 188, 191, 189, 189, 186, 175, 190, 182, 192]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.891799435Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"":"downstairs","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120892024-03-06T21:02:52.153Z{"msg":"[1]R generation[0..12]: [110, 110, 111, 110, 110, 110, 110, 109, 103, 110, 107, 111]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.891814144Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"":"downstairs","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120902024-03-06T21:02:52.153Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"timeWait for a query_work_queue command to finish before sending IO
120912024-03-06T21:02:52.153Z":"2024-03-06T21:02:50.891841272Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"":"downstairs","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120922024-03-06T21:02:52.153Z{"msg":"[2]R flush_numbers[0..12]: [191, 191, 192, 188, 191, 189, 189, 186, 175, 190, 182, 192]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.89187319Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"":"downstairs","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120932024-03-06T21:02:52.153Z{"msg":"[2]R generation[0..12]: [110, 110, 111, 110, 110, 110, 110, 109, 103, 110, 107, 111]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.891909307Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"":"downstairs","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120942024-03-06T21:02:52.153Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.891950804Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"":"downstairs","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120952024-03-06T21:02:52.153Z{"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.891990412Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"":"downstairs","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120962024-03-06T21:02:52.153Z{"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.89201739Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"":"downstairs","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120972024-03-06T21:02:52.153Z{"msg":"Next flush: 193","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.892056057Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"":"downstairs","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120982024-03-06T21:02:52.153Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.892093745Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"":"downstairs","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
120992024-03-06T21:02:52.153Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.892128542Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
121002024-03-06T21:02:52.153Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.892147271Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"client":"0","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
121012024-03-06T21:02:52.153Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.89216747Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"client":"1","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
121022024-03-06T21:02:52.153Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.892197658Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"client":"2","":"downstairs","session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
121032024-03-06T21:02:52.154Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.892224106Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
121042024-03-06T21:02:52.154Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.892257973Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
121052024-03-06T21:02:52.154Z{"msg":"cd34f64a-c683-48e4-bc6b-418d8db37677 is now active with session: ebf58839-75ee-4cc3-b12c-84f6b09b5c95","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.892293431Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
121062024-03-06T21:02:52.154Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:50.892327909Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587,"session_id":"ebf58839-75ee-4cc3-b12c-84f6b09b5c95"}
121072024-03-06T21:02:52.154Z{"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
121082024-03-06T21:02:52.154Z:"2024-03-06T21:02:50.892361556Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587}
121092024-03-06T21:02:52.154ZLoading write count information from file "/tmp/test_repair_verify.data"
121102024-03-06T21:02:52.154ZRun Repair workload
121112024-03-06T21:02:52.154Z01/30 Write block 167 len 40960 data: 6 6 4 3 4 6 10 9 10 9
121122024-03-06T21:02:52.161Z02/30 Write block 515 len 20480 data: 12 12 10 8 7
121132024-03-06T21:02:52.161Z03/30 Flush
121142024-03-06T21:02:52.161Z04/30 Write block 497 len 8192 data: 9 9
121152024-03-06T21:02:52.161Z05/30 Read block 99 len 12288
121162024-03-06T21:02:52.161Z06/30 Write block 575 len 4096 data: 7
121172024-03-06T21:02:52.161Z07/30 Read block 67 len 36864
121182024-03-06T21:02:52.161Z08/30 Write block 132 len 32768 data: 7 8 8 8 9 7 6 7
121192024-03-06T21:02:52.161Z09/30 Write block 321 len 8192 data: 9 9
121202024-03-06T21:02:52.161Z10/30 Read block 513 len 20480
121212024-03-06T21:02:52.161Z11/30 Write block 98 len 24576 data: 9 9 6 8 8 8
121222024-03-06T21:02:52.162Z12/30 Read block 437 len 20480
121232024-03-06T21:02:52.162Z13/30 Flush
121242024-03-06T21:02:52.162Z14/30 Write block 373 len 28672 data: 11 10 11 8 7 7 9
121252024-03-06T21:02:52.162Z15/30 Write block 428 len 20480 data: 12 10 9 7 8
121262024-03-06T21:02:52.162Z16/30 Read block 571 len 4096
121272024-03-06T21:02:52.162Z17/30 Flush
121282024-03-06T21:02:52.169Z18/30 Write block 114 len 32768 data: 5 5 4 6 6 7 6 6
121292024-03-06T21:02:52.169Z19/30 Write block 227 len 4096 data: 8
121302024-03-06T21:02:52.169Z20/30 Write block 500 len 40960 data: 8 5 6 6 7 7 7 8 7 8
121312024-03-06T21:02:52.169Z21/30 Read block 248 len 8192
121322024-03-06T21:02:52.170Z22/30 Write block 75 len 4096 data: 7
121332024-03-06T21:02:52.170Z23/30 Write block 378 len 16384 data: 8 10 10 9
121342024-03-06T21:02:52.183Z24/30 Flush
121352024-03-06T21:02:52.184Z25/30 Write block 231 len 20480 data: 9 8 7 9 10
121362024-03-06T21:02:52.184Z26/30 Read block 432 len 8192
121372024-03-06T21:02:52.184Z27/30 Read block 0 len 24576
121382024-03-06T21:02:52.184Z28/30 Write block 321 len 40960 data: 10 10 8 9 8 9 7 7 7 6
121392024-03-06T21:02:52.184Z29/30 Write block 397 len 36864 data: 8 8 5 7 5 5 7 6 9
121402024-03-06T21:02:52.184Z30/30 Read block 440 len 28672
121412024-03-06T21:02:52.185Z----------------------------------------------------------------
121422024-03-06T21:02:52.185Z Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30
121432024-03-06T21:02:52.185ZStates: Active Active Active
121442024-03-06T21:02:52.185ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
121452024-03-06T21:02:52.185Z 1 Acked 1000 Write 10 Done Done Done false
121462024-03-06T21:02:52.185Z 2 Acked 1001 Write 5 Done Done Sent false
121472024-03-06T21:02:52.185Z 3 Acked 1002 Flush 0 Done Done Sent false
121482024-03-06T21:02:52.186Z 4 Acked 1003 Write 2 Done Done Sent false
121492024-03-06T21:02:52.186Z 5 Acked 1004 Read 3 Done Done Sent false
121502024-03-06T21:02:52.186Z 6 Acked 1005 Write 1 Done Done Sent false
121512024-03-06T21:02:52.186Z 7 Acked 1006 Read 9 Done Done Sent false
121522024-03-06T21:02:52.186Z 8 Acked 1007 Write 8 Done Done Sent false
121532024-03-06T21:02:52.186Z 9 Acked 1008 Write 2 Done Done Sent false
121542024-03-06T21:02:52.186Z 10 Acked 1009 Read 5 Done Done Sent false
121552024-03-06T21:02:52.186Z 11 Acked 1010 Write 6 Done Done Sent false
121562024-03-06T21:02:52.187Z 12 Acked 1011 Read 5 Done Done Sent false
121572024-03-06T21:02:52.187Z 13 Acked 1012 Flush 0 Done Done Sent false
121582024-03-06T21:02:52.187Z 14 Acked 1013 Write 7 Done Done Sent false
121592024-03-06T21:02:52.187Z 15 Acked 1014 Write 5 Done Done Sent false
121602024-03-06T21:02:52.187Z 16 Acked 1015 Read 1 Done Done Sent false
121612024-03-06T21:02:52.187Z 17 Acked 1016 Flush 0 Done Done Sent false
121622024-03-06T21:02:52.187Z 18 Acked 1017 Write 8 Done Done Sent false
121632024-03-06T21:02:52.188Z 19 Acked 1018 Write 1 Done Done Sent false
121642024-03-06T21:02:52.188Z 20 Acked 1019 Write 10 Done Done Sent false
121652024-03-06T21:02:52.188Z 21 Acked 1020 Read 2 Done Done Sent false
121662024-03-06T21:02:52.188Z 22 Acked 1021 Write 1 Done Done Sent false
121672024-03-06T21:02:52.189Z 23 Acked 1022 Write 4 Done Done Sent false
121682024-03-06T21:02:52.189Z 24 Acked 1023 Flush 0 Done Done Sent false
121692024-03-06T21:02:52.189Z 25 Acked 1024 Write 5 Done Done Sent false
121702024-03-06T21:02:52.189Z 26 Acked 1025 Read 2 Done Done Sent false
121712024-03-06T21:02:52.189Z 27 Acked 1026 Read 6 Done Done Sent false
121722024-03-06T21:02:52.190Z 28 Acked 1027 Write 10 Done Done Sent false
121732024-03-06T21:02:52.190Z 29 Acked 1028 Write 9 Done Done Sent false
121742024-03-06T21:02:52.190Z 30 Acked 1029 Read 7 Done Done Sent false
121752024-03-06T21:02:52.190Z STATES DS:0 DS:1 DS:2 TOTAL
121762024-03-06T21:02:52.190Z New 0 0 0 0
121772024-03-06T21:02:52.191Z Sent 0 0 29 29
121782024-03-06T21:02:52.191Z Done 30 30 1 61
121792024-03-06T21:02:52.191Z Skipped 0 0 0 0
121802024-03-06T21:02:52.191Z Error 0 0 0 0
121812024-03-06T21:02:52.191ZLast Flush: 1023 1023 0
121822024-03-06T21:02:52.192ZDownstairs last five completed:
121832024-03-06T21:02:52.192ZUpstairs last five completed: 30 29 28 27 26
121842024-03-06T21:02:52.192Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:02:50.928764047Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1587}
121852024-03-06T21:02:52.192ZWrote out file "/tmp/test_repair_verify.data"
121862024-03-06T21:02:52.200Z
121872024-03-06T21:02:52.200ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
121882024-03-06T21:02:52.218ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
121892024-03-06T21:02:52.218Z 0 000-019 110 110 110 191 191 191 F F F
121902024-03-06T21:02:52.218Z 1 020-039 110 110 110 191 191 191 F F F
121912024-03-06T21:02:52.218Z 2 040-059 111 111 111 192 192 192 F F F
121922024-03-06T21:02:52.219Z 3 060-079 112 112 110 196 196 188 F F F <---
121932024-03-06T21:02:52.219Z 4 080-099 112 112 110 194 194 191 F F F <---
121942024-03-06T21:02:52.219Z 5 100-119 112 112 110 196 196 189 F F F <---
121952024-03-06T21:02:52.219Z 6 120-139 112 112 110 196 196 189 F F F <---
121962024-03-06T21:02:52.219Z 7 140-159 109 109 109 186 186 186 F F F
121972024-03-06T21:02:52.219Z 8 160-179 112 112 103 193 193 175 F F T <---
121982024-03-06T21:02:52.219Z 9 180-199 110 110 110 190 190 190 F F F
121992024-03-06T21:02:52.219Z 10 200-219 107 107 107 182 182 182 F F F
122002024-03-06T21:02:52.220Z 11 220-239 112 112 111 196 196 192 T T F <---
122012024-03-06T21:02:52.220Z 12 240-259 111 111 111 192 192 192 F F F
122022024-03-06T21:02:52.220Z 13 260-279 98 98 98 168 168 168 F F F
122032024-03-06T21:02:52.220Z 14 280-299 91 91 91 156 156 156 F F F
122042024-03-06T21:02:52.220Z 15 300-319 108 108 108 185 185 185 F F F
122052024-03-06T21:02:52.220Z 16 320-339 112 112 111 194 194 192 T T F <---
122062024-03-06T21:02:52.220Z 17 340-359 111 111 111 192 192 192 F F F
122072024-03-06T21:02:52.220Z 18 360-379 112 112 105 196 196 178 F F F <---
122082024-03-06T21:02:52.220Z 19 380-399 112 112 109 196 196 186 T T F <---
122092024-03-06T21:02:52.221Z 20 400-419 110 110 110 188 188 188 T T F <---
122102024-03-06T21:02:52.221Z 21 420-439 112 112 108 195 195 184 F F F <---
122112024-03-06T21:02:52.221Z 22 440-459 110 110 110 190 190 190 F F F
122122024-03-06T21:02:52.221Z 23 460-479 108 108 108 185 185 185 F F F
122132024-03-06T21:02:52.221Z 24 480-499 112 112 110 194 194 191 F F F <---
122142024-03-06T21:02:52.221Z 25 500-519 112 112 108 196 196 185 F F F <---
122152024-03-06T21:02:52.222Z 26 520-539 109 109 109 186 186 186 F F F
122162024-03-06T21:02:52.222Z 27 540-559 98 98 98 169 169 169 F F F
122172024-03-06T21:02:52.222Z 28 560-579 112 112 104 194 194 176 F F F <---
122182024-03-06T21:02:52.222Z 29 580-599 109 109 109 186 186 186 F F F
122192024-03-06T21:02:52.222ZMax gen: 112, Max flush: 196
122202024-03-06T21:02:52.222ZError: Difference in extent metadata found!
122212024-03-06T21:02:52.239ZA Difference in extent metadata is expected here
122222024-03-06T21:02:52.239ZOn loop 56
122232024-03-06T21:02:52.239Z
122242024-03-06T21:02:52.239ZVerifying data now
122252024-03-06T21:02:57.942ZNo change after verify
122262024-03-06T21:02:57.942ZLoop: 56 Downstairs dump after verify (and repair):
122272024-03-06T21:02:57.957ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
122282024-03-06T21:02:57.957Z 0 000-019 110 110 110 191 191 191 F F F
122292024-03-06T21:02:57.958Z 1 020-039 110 110 110 191 191 191 F F F
122302024-03-06T21:02:57.958Z 2 040-059 111 111 111 192 192 192 F F F
122312024-03-06T21:02:57.958Z 3 060-079 112 112 112 196 196 196 F F F
122322024-03-06T21:02:57.958Z 4 080-099 112 112 112 194 194 194 F F F
122332024-03-06T21:02:57.958Z 5 100-119 112 112 112 196 196 196 F F F
122342024-03-06T21:02:57.958Z 6 120-139 112 112 112 196 196 196 F F F
122352024-03-06T21:02:57.958Z 7 140-159 109 109 109 186 186 186 F F F
122362024-03-06T21:02:57.958Z 8 160-179 112 112 112 193 193 193 F F F
122372024-03-06T21:02:57.958Z 9 180-199 110 110 110 190 190 190 F F F
122382024-03-06T21:02:57.958Z 10 200-219 107 107 107 182 182 182 F F F
122392024-03-06T21:02:57.958Z 11 220-239 113 113 113 197 197 197 F F F
122402024-03-06T21:02:57.958Z 12 240-259 111 111 111 192 192 192 F F F
122412024-03-06T21:02:57.958Z 13 260-279 98 98 98 168 168 168 F F F
122422024-03-06T21:02:57.958Z 14 280-299 91 91 91 156 156 156 F F F
122432024-03-06T21:02:57.958Z 15 300-319 108 108 108 185 185 185 F F F
122442024-03-06T21:02:57.958Z 16 320-339 113 113 113 197 197 197 F F F
122452024-03-06T21:02:57.958Z 17 340-359 111 111 111 192 192 192 F F F
122462024-03-06T21:02:57.958Z 18 360-379 112 112 112 196 196 196 F F F
122472024-03-06T21:02:57.958Z 19 380-399 113 113 113 197 197 197 F F F
122482024-03-06T21:02:57.958Z 20 400-419 113 113 113 197 197 197 F F F
122492024-03-06T21:02:57.958Z 21 420-439 112 112 112 195 195 195 F F F
122502024-03-06T21:02:57.958Z 22 440-459 110 110 110 190 190 190 F F F
122512024-03-06T21:02:57.958Z 23 460-479 108 108 108 185 185 185 F F F
122522024-03-06T21:02:57.959Z 24 480-499 112 112 112 194 194 194 F F F
122532024-03-06T21:02:57.959Z 25 500-519 112 112 112 196 196 196 F F F
122542024-03-06T21:02:57.962Z 26 520-539 109 109 109 186 186 186 F F F
122552024-03-06T21:02:57.962Z 27 540-559 98 98 98 169 169 169 F F F
122562024-03-06T21:02:57.962Z 28 560-579 112 112 112 194 194 194 F F F
122572024-03-06T21:02:57.962Z 29 580-599 109 109 109 186 186 186 F F F
122582024-03-06T21:02:57.962ZMax gen: 113, Max flush: 197
122592024-03-06T21:02:57.962Z
122602024-03-06T21:02:57.962ZBegin loop 57
122612024-03-06T21:02:57.962ZDownstairs to restart: 1
122622024-03-06T21:02:57.985Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.737500967Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595}
122632024-03-06T21:02:57.985Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.737791857Z","hostname":"Crucible runtime is spawned
122642024-03-06T21:02:57.986Zw-01HRARQTS0TVE14DKZ437MF5TB","pid":1595}
122652024-03-06T21:02:57.991Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.737852902Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595}
122662024-03-06T21:02:57.991Z{"msg":"Crucible 5d0c95a3-54e3-4395-a703-972fd1b01b1a has session id: b9ca09c9-12c1-43bd-b269-04aa2a4b1c68","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.73788958Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122672024-03-06T21:02:57.991Z{"msg":"Upstairs opts: Upstairs UUID: 5d0c95a3-54e3-4395-a703-972fd1b01b1a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.737947246Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122682024-03-06T21:02:57.991Z{"msg":"Crucible stats registered with UUID: 5d0c95a3-54e3-4395-a703-972fd1b01b1a","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.737983424Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122692024-03-06T21:02:57.992Z{"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.738006152Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595}
122702024-03-06T21:02:57.992Z{"msg":"5d0c95a3-54e3-4395-a703-972fd1b01b1a active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.73817048Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122712024-03-06T21:02:57.992Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.738210568Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"client":"0","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122722024-03-06T21:02:57.992Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.738235186Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"client":"0","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122732024-03-06T21:02:57.992Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.738286592Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"client":"1","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122742024-03-06T21:02:57.992Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.73832467Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"client":"1","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122752024-03-06T21:02:57.992Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.738376426Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"client":"2","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122762024-03-06T21:02:57.992Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.738406954Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"client":"2","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122772024-03-06T21:02:57.992Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.738445921Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122782024-03-06T21:02:57.992Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.738498028Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122792024-03-06T21:02:57.992Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.738538045Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122802024-03-06T21:02:57.992Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.738578602Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122812024-03-06T21:02:57.992Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.738776949Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122822024-03-06T21:02:57.992Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.738839265Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122832024-03-06T21:02:57.992Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.73934129Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"client":"0","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122842024-03-06T21:02:57.992Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.739381627Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"client":"0","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122852024-03-06T21:02:57.992Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.739398566Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"client":"2","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122862024-03-06T21:02:57.992Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.739416545Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"client":"2","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122872024-03-06T21:02:57.992Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.739873763Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"client":"1","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122882024-03-06T21:02:57.992Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.73991768Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"client":"1","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122892024-03-06T21:02:57.992Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.740135095Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"client":"2","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122902024-03-06T21:02:57.992Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.740171073Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"client":"0","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122912024-03-06T21:02:57.992Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.740583574Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"client":"1","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122922024-03-06T21:02:57.992Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.74078552Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"client":"0","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122932024-03-06T21:02:57.993Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.740847626Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"client":"0","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122942024-03-06T21:02:57.993Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.740890873Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122952024-03-06T21:02:57.993Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.740907172Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"client":"2","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122962024-03-06T21:02:57.993Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.74093435Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"client":"2","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122972024-03-06T21:02:57.993Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.740948879Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122982024-03-06T21:02:57.993Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.741094419Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"client":"1","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
122992024-03-06T21:02:57.993Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.741134796Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"client":"1","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
123002024-03-06T21:02:57.993Z{"msg":"[0]R flush_numbers[0..12]: [191, 191, 192, 196, 194, 196, 196, 186, 193, 190, 182, 197]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.741154945Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"":"downstairs","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
123012024-03-06T21:02:57.993Z{"msg":"[0]R generation[0..12]: [110, 110, 111, 112, 112, 112, 112, 109, 112, 110, 107, 113]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.741186913Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"":"downstairs","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
123022024-03-06T21:02:57.993Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.741209631Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"":"downstairs","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
123032024-03-06T21:02:57.993Z{"msg":"[1]R flush_numbers[0..12]: [191, 191, 192, 196, 194, 196, 196, 186, 193, 190, 182, 197]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.741240629Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"":"downstairs","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
123042024-03-06T21:02:57.993Z{"msg":"[1]R generation[0..12]: [110, 110, 111, 112, 112, 112, 112, 109, 112, 110, 107, 113]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.741277017Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"":"downstairs","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
123052024-03-06T21:02:57.993Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
123062024-03-06T21:02:57.997Z,"time":"2024-03-06T21:02:56.741293185Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"":"downstairs","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
123072024-03-06T21:02:57.997Z{"msg":"[2]R flush_numbers[0..12]: [191, 191, 192, 196, 194, 196, 196, 186, 193, 190, 182, 197]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.741328333Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"":"downstairs","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
123082024-03-06T21:02:57.998Z{"msg":"[2]R generation[0..12]: [110, 110, 111, 112, 112, 112, 112, 109, 112, 110, 107, 113]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.741345702Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"":"downstairs","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
123092024-03-06T21:02:57.998Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.74137622Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"":"downstairs","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
123102024-03-06T21:02:57.998Z{"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.741391209Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"":"downstairs","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
123112024-03-06T21:02:57.998Z{"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.741406357Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"":"downstairs","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
123122024-03-06T21:02:57.998Z{"msg":"Next flush: 198","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.741435966Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"":"downstairs","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
123132024-03-06T21:02:57.998Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.741450765Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"":"downstairs","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
123142024-03-06T21:02:57.998Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.741480772Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
123152024-03-06T21:02:57.998Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.741495191Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"client":"0","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
123162024-03-06T21:02:57.998Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.741525569Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"client":"1","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
123172024-03-06T21:02:57.998Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.741545908Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"client":"2","":"downstairs","session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
123182024-03-06T21:02:57.998Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.741573446Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
123192024-03-06T21:02:57.998Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.741590005Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
123202024-03-06T21:02:57.998Z{"msg":"5d0c95a3-54e3-4395-a703-972fd1b01b1a is now active with session: b9ca09c9-12c1-43bd-b269-04aa2a4b1c68","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
123212024-03-06T21:02:57.998Z2024-03-06T21:02:56.741617963Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
123222024-03-06T21:02:57.998Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.741640781Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595,"session_id":"b9ca09c9-12c1-43bd-b269-04aa2a4b1c68"}
123232024-03-06T21:02:57.998Z{"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:02:56.741669229Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595}
123242024-03-06T21:02:57.998ZLoading write count information from file "/tmp/test_repair_verify.data"
123252024-03-06T21:02:57.998ZRun Repair workload
123262024-03-06T21:02:57.998Z01/30 Write block 573 len 16384 data: 7 6 8 9
123272024-03-06T21:02:57.998Z02/30 Flush
123282024-03-06T21:02:57.998Z03/30 Write block 321 len 24576 data: 11 11 9 10 9 10
123292024-03-06T21:02:57.998Z04/30 Write block 227 len 8192 data: 9 7
123302024-03-06T21:02:57.998Z05/30 Write block 265 len 20480 data: 8 7 7 8 8
123312024-03-06T21:02:57.999Z06/30 Read block 153 len 12288
123322024-03-06T21:02:58.002Z07/30 Write block 420 len 4096 data: 9
123332024-03-06T21:02:58.002Z08/30 Flush
123342024-03-06T21:02:58.002Z09/30 Write block 3 len 20480 data: 7 9 10 10 10
123352024-03-06T21:02:58.009Z10/30 Read block 437 len 40960
123362024-03-06T21:02:58.009Z11/30 Read block 395 len 16384
123372024-03-06T21:02:58.009Z12/30 Read block 163 len 16384
123382024-03-06T21:02:58.013Z13/30 Read block 205 len 32768
123392024-03-06T21:02:58.013Z14/30 Read block 304 len 16384
123402024-03-06T21:02:58.013Z15/30 Write block 195 len 24576 data: 8 11 11 12 10 11
123412024-03-06T21:02:58.013Z16/30 Read block 210 len 4096
123422024-03-06T21:02:58.013Z17/30 Write block 416 len 32768 data: 6 6 7 8 10 15 13 13
123432024-03-06T21:02:58.013Z18/30 Read block 1 len 24576
123442024-03-06T21:02:58.013Z19/30 Read block 270 len 12288
123452024-03-06T21:02:58.013Z20/30 Flush
123462024-03-06T21:02:58.016Z21/30 Write block 309 len 16384 data: 11 10 10 10
123472024-03-06T21:02:58.019Z22/30 Read block 398 len 32768
123482024-03-06T21:02:58.019Z23/30 Read block 535 len 32768
123492024-03-06T21:02:58.020Z24/30 Flush
123502024-03-06T21:02:58.023Z25/30 Write block 563 len 4096 data: 8
123512024-03-06T21:02:58.023Z26/30 Read block 386 len 28672
123522024-03-06T21:02:58.026Z27/30 Write block 546 len 36864 data: 13 12 10 9 8 6 6 5 6
123532024-03-06T21:02:58.026Z28/30 Write block 103 len 32768 data: 9 7 6 10 12 9 9 8
123542024-03-06T21:02:58.026Z29/30 Read block 38 len 4096
123552024-03-06T21:02:58.026Z30/30 Read block 219 len 28672
123562024-03-06T21:02:58.030Z----------------------------------------------------------------
123572024-03-06T21:02:58.030Z Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:22
123582024-03-06T21:02:58.030ZStates: Active Active Active
123592024-03-06T21:02:58.030ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
123602024-03-06T21:02:58.030Z 9 Acked 1008 Write 5 Done Done Done false
123612024-03-06T21:02:58.030Z 10 Acked 1009 Read 10 Done Done Done false
123622024-03-06T21:02:58.030Z 11 Acked 1010 Read 4 Done Done Done false
123632024-03-06T21:02:58.030Z 12 Acked 1011 Read 4 Done Done Done false
123642024-03-06T21:02:58.030Z 13 Acked 1012 Read 8 Done Done Done false
123652024-03-06T21:02:58.030Z 14 Acked 1013 Read 4 Done Done Done false
123662024-03-06T21:02:58.030Z 15 Acked 1014 Write 6 Done Sent Done false
123672024-03-06T21:02:58.030Z 16 Acked 1015 Read 1 Done Sent Done false
123682024-03-06T21:02:58.031Z 17 Acked 1016 Write 8 Done Sent Done false
123692024-03-06T21:02:58.031Z 18 Acked 1017 Read 6 Done Sent Done false
123702024-03-06T21:02:58.031Z 19 Acked 1018 Read 3 Done Sent Done false
123712024-03-06T21:02:58.031Z 20 Acked 1019 Flush 0 Done Sent Done false
123722024-03-06T21:02:58.031Z 21 Acked 1020 Write 4 Done Sent Done false
123732024-03-06T21:02:58.031Z 22 Acked 1021 Read 8 Done Sent Done false
123742024-03-06T21:02:58.031Z 23 Acked 1022 Read 8 Done Sent Done false
123752024-03-06T21:02:58.031Z 24 Acked 1023 Flush 0 Done Sent Done false
123762024-03-06T21:02:58.031Z 25 Acked 1024 Write 1 Done Sent Done false
123772024-03-06T21:02:58.031Z 26 Acked 1025 Read 7 Done Sent Done false
123782024-03-06T21:02:58.031Z 27 Acked 1026 Write 9 Done Sent Done false
123792024-03-06T21:02:58.031Z 28 Acked 1027 Write 8 Done Sent Done false
123802024-03-06T21:02:58.031Z 29 Acked 1028 Read 1 Done Sent Done false
123812024-03-06T21:02:58.031Z 30 Acked 1029 Read 7 Done Sent Done false
123822024-03-06T21:02:58.031Z STATES DS:0 DS:1 DS:2 TOTAL
123832024-03-06T21:02:58.031Z New 0 0 0 0
123842024-03-06T21:02:58.031Z Sent 0 16 0 16
123852024-03-06T21:02:58.031Z Done 22 6 22 50
123862024-03-06T21:02:58.031Z Skipped 0 0 0 0
123872024-03-06T21:02:58.031Z Error 0 0 0 0
123882024-03-06T21:02:58.031ZLast Flush: 1023 1007 1023
123892024-03-06T21:02:58.031ZDownstairs last five completed: 1007 1006 1005 1004 1003
123902024-03-06T21:02:58.031ZUpstairs last five completed: 30 29 28 27 26
123912024-03-06T21:02:58.031Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:02:56.779862677Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1595}
123922024-03-06T21:02:58.032ZWrote out file "/tmp/test_repair_verify.data"
123932024-03-06T21:02:58.035Z
123942024-03-06T21:02:58.038ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
123952024-03-06T21:02:58.055ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
123962024-03-06T21:02:58.055Z 0 000-019 114 110 114 200 191 200 F T F <---
123972024-03-06T21:02:58.055Z 1 020-039 110 110 110 191 191 191 F F F
123982024-03-06T21:02:58.055Z 2 040-059 111 111 111 192 192 192 F F F
123992024-03-06T21:02:58.055Z 3 060-079 112 112 112 196 196 196 F F F
124002024-03-06T21:02:58.055Z 4 080-099 112 112 112 194 194 194 F F F
124012024-03-06T21:02:58.055Z 5 100-119 112 112 112 196 196 196 T F T <---
124022024-03-06T21:02:58.055Z 6 120-139 112 112 112 196 196 196 F F F
124032024-03-06T21:02:58.055Z 7 140-159 109 109 109 186 186 186 F F F
124042024-03-06T21:02:58.055Z 8 160-179 112 112 112 193 193 193 F F F
124052024-03-06T21:02:58.055Z 9 180-199 114 110 114 200 190 200 F F F <---
124062024-03-06T21:02:58.055Z 10 200-219 114 107 114 200 182 200 F F F <---
124072024-03-06T21:02:58.055Z 11 220-239 114 114 114 199 199 199 F F F
124082024-03-06T21:02:58.055Z 12 240-259 111 111 111 192 192 192 F F F
124092024-03-06T21:02:58.055Z 13 260-279 114 114 114 199 199 199 F F F
124102024-03-06T21:02:58.055Z 14 280-299 91 91 91 156 156 156 F F F
124112024-03-06T21:02:58.055Z 15 300-319 114 108 114 201 185 201 F F F <---
124122024-03-06T21:02:58.055Z 16 320-339 114 114 114 199 199 199 F F F
124132024-03-06T21:02:58.055Z 17 340-359 111 111 111 192 192 192 F F F
124142024-03-06T21:02:58.055Z 18 360-379 112 112 112 196 196 196 F F F
124152024-03-06T21:02:58.059Z 19 380-399 113 113 113 197 197 197 F F F
124162024-03-06T21:02:58.059Z 20 400-419 114 113 114 200 197 200 F F F <---
124172024-03-06T21:02:58.059Z 21 420-439 114 114 114 200 199 200 F F F <---
124182024-03-06T21:02:58.059Z 22 440-459 110 110 110 190 190 190 F F F
124192024-03-06T21:02:58.059Z 23 460-479 108 108 108 185 185 185 F F F
124202024-03-06T21:02:58.059Z 24 480-499 112 112 112 194 194 194 F F F
124212024-03-06T21:02:58.059Z 25 500-519 112 112 112 196 196 196 F F F
124222024-03-06T21:02:58.060Z 26 520-539 109 109 109 186 186 186 F F F
124232024-03-06T21:02:58.060Z 27 540-559 98 98 98 169 169 169 T F T <---
124242024-03-06T21:02:58.060Z 28 560-579 114 114 114 198 198 198 T F T <---
124252024-03-06T21:02:58.060Z 29 580-599 109 109 109 186 186 186 F F F
124262024-03-06T21:02:58.060ZMax gen: 114, Max flush: 201
124272024-03-06T21:02:58.060ZError: Difference in extent metadata found!
124282024-03-06T21:02:58.063ZA Difference in extent metadata is expected here
124292024-03-06T21:02:58.063ZOn loop 57
124302024-03-06T21:02:58.063Z
124312024-03-06T21:02:58.063ZVerifying data now
124322024-03-06T21:03:02.573ZNo change after verify
124332024-03-06T21:03:02.573ZLoop: 57 Downstairs dump after verify (and repair):
124342024-03-06T21:03:02.589ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
124352024-03-06T21:03:02.589Z 0 000-019 114 114 114 200 200 200 F F F
124362024-03-06T21:03:02.589Z 1 020-039 110 110 110 191 191 191 F F F
124372024-03-06T21:03:02.589Z 2 040-059 111 111 111 192 192 192 F F F
124382024-03-06T21:03:02.589Z 3 060-079 112 112 112 196 196 196 F F F
124392024-03-06T21:03:02.589Z 4 080-099 112 112 112 194 194 194 F F F
124402024-03-06T21:03:02.589Z 5 100-119 115 115 115 202 202 202 F F F
124412024-03-06T21:03:02.589Z 6 120-139 112 112 112 196 196 196 F F F
124422024-03-06T21:03:02.589Z 7 140-159 109 109 109 186 186 186 F F F
124432024-03-06T21:03:02.589Z 8 160-179 112 112 112 193 193 193 F F F
124442024-03-06T21:03:02.595Z 9 180-199 114 114 114 200 200 200 F F F
124452024-03-06T21:03:02.595Z 10 200-219 114 114 114 200 200 200 F F F
124462024-03-06T21:03:02.595Z 11 220-239 114 114 114 199 199 199 F F F
124472024-03-06T21:03:02.595Z 12 240-259 111 111 111 192 192 192 F F F
124482024-03-06T21:03:02.595Z 13 260-279 114 114 114 199 199 199 F F F
124492024-03-06T21:03:02.595Z 14 280-299 91 91 91 156 156 156 F F F
124502024-03-06T21:03:02.595Z 15 300-319 114 114 114 201 201 201 F F F
124512024-03-06T21:03:02.595Z 16 320-339 114 114 114 199 199 199 F F F
124522024-03-06T21:03:02.595Z 17 340-359 111 111 111 192 192 192 F F F
124532024-03-06T21:03:02.595Z 18 360-379 112 112 112 196 196 196 F F F
124542024-03-06T21:03:02.595Z 19 380-399 113 113 113 197 197 197 F F F
124552024-03-06T21:03:02.595Z 20 400-419 114 114 114 200 200 200 F F F
124562024-03-06T21:03:02.595Z 21 420-439 114 114 114 200 200 200 F F F
124572024-03-06T21:03:02.595Z 22 440-459 110 110 110 190 190 190 F F F
124582024-03-06T21:03:02.595Z 23 460-479 108 108 108 185 185 185 F F F
124592024-03-06T21:03:02.595Z 24 480-499 112 112 112 194 194 194 F F F
124602024-03-06T21:03:02.595Z 25 500-519 112 112 112 196 196 196 F F F
124612024-03-06T21:03:02.595Z 26 520-539 109 109 109 186 186 186 F F F
124622024-03-06T21:03:02.595Z 27 540-559 115 115 115 202 202 202 F F F
124632024-03-06T21:03:02.596Z 28 560-579 115 115 115 202 202 202 F F F
124642024-03-06T21:03:02.596Z 29 580-599 109 109 109 186 186 186 F F F
124652024-03-06T21:03:02.596ZMax gen: 115, Max flush: 202
124662024-03-06T21:03:02.596Z
124672024-03-06T21:03:02.596ZBegin loop 58
124682024-03-06T21:03:02.596ZDownstairs to restart: 0
124692024-03-06T21:03:02.616Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.367353912Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603}
124702024-03-06T21:03:02.617Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.36767274Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603}
124712024-03-06T21:03:02.617Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.367701678Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603}
124722024-03-06T21:03:02.617Z{"msg":"Crucible f29f6a63-fdcd-4fa3-9ad8-68c3ff312d59 has session id: b61a19ae-70c4-4596-b5ef-b99b01613bba","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.367736385Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
124732024-03-06T21:03:02.625Z{"msg":"Upstairs opts: Upstairs UUID: f29f6a63-fdcd-4fa3-9ad8-68c3ff312d59, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.367773373Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
124742024-03-06T21:03:02.625Z{"msg":"Crucible stats registered with UUID: f29f6a63-fdcd-4fa3-9ad8-68c3ff312d59","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.36781717Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
124752024-03-06T21:03:02.625ZCrucible runtime is spawned
124762024-03-06T21:03:02.625Z{"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.36795533Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603}
124772024-03-06T21:03:02.625Z{"msg":"f29f6a63-fdcd-4fa3-9ad8-68c3ff312d59 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.368060353Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
124782024-03-06T21:03:02.625Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.368088991Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"client":"0","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
124792024-03-06T21:03:02.625Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.36810514Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"client":"0","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
124802024-03-06T21:03:02.625Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.368120789Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"client":"1","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
124812024-03-06T21:03:02.625Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.368144757Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"client":"1","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
124822024-03-06T21:03:02.625Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.368159266Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"client":"2","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
124832024-03-06T21:03:02.625Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.368176025Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"client":"2","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
124842024-03-06T21:03:02.625Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.368254749Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
124852024-03-06T21:03:02.625Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.368307656Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
124862024-03-06T21:03:02.625Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.368324265Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
124872024-03-06T21:03:02.625Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.368584847Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
124882024-03-06T21:03:02.625Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.368638803Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
124892024-03-06T21:03:02.625Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.368673731Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
124902024-03-06T21:03:02.626Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.369383802Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"client":"1","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
124912024-03-06T21:03:02.626Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.369425529Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"client":"1","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
124922024-03-06T21:03:02.626Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.369453287Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"client":"2","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
124932024-03-06T21:03:02.626Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.369492264Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"client":"2","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
124942024-03-06T21:03:02.626Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.370048046Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"client":"2","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
124952024-03-06T21:03:02.626Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.370087293Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"client":"1","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
124962024-03-06T21:03:02.626Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.370475747Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"client":"0","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
124972024-03-06T21:03:02.626Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.370517524Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"client":"0","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
124982024-03-06T21:03:02.626Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.370653494Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"client":"2","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
124992024-03-06T21:03:02.626Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.370685452Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"client":"2","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
125002024-03-06T21:03:02.626Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.370700811Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
125012024-03-06T21:03:02.626Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.37071473Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"client":"1","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
125022024-03-06T21:03:02.626Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.370729689Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"client":"1","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
125032024-03-06T21:03:02.626Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.370743308Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
125042024-03-06T21:03:02.626Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.371080025Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"client":"0","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
125052024-03-06T21:03:02.626Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.371498846Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"client":"0","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
125062024-03-06T21:03:02.626Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.371538243Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"client":"0","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
125072024-03-06T21:03:02.626Z{"msg":"[0]R flush_numbers[0..12]: [200, 191, 192, 196, 194, 202, 196, 186, 193, 200, 200, 199]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.37159023Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"":"downstairs","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
125082024-03-06T21:03:02.626Z{"msg":"[0]R generation[0..12]: [114, 110, 111, 112, 112, 115, 112, 109, 112, 114, 114, 114]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.371616068Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"":"downstairs","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
125092024-03-06T21:03:02.626Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.371650526Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"":"downstairs","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
125102024-03-06T21:03:02.627Z{"msg":"[1]R flush_numbers[0..12]: [200, 191, 192, 196, 194, 202, 196, 186, 193, 200, 200, 199]","v":0,"name":"crucible","level":30,"timeWait for a query_work_queue command to finish before sending IO
125112024-03-06T21:03:02.627Z":"2024-03-06T21:03:01.371682863Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"":"downstairs","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
125122024-03-06T21:03:02.627Z{"msg":"[1]R generation[0..12]: [114, 110, 111, 112, 112, 115, 112, 109, 112, 114, 114, 114]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.371718701Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"":"downstairs","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
125132024-03-06T21:03:02.627Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.371752279Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"":"downstairs","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
125142024-03-06T21:03:02.627Z{"msg":"[2]R flush_numbers[0..12]: [200, 191, 192, 196, 194, 202, 196, 186, 193, 200, 200, 199]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.371784476Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"":"downstairs","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
125152024-03-06T21:03:02.627Z{"msg":"[2]R generation[0..12]: [114, 110, 111, 112, 112, 115, 112, 109, 112, 114, 114, 114]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.371817884Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"":"downstairs","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
125162024-03-06T21:03:02.627Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.371837363Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"":"downstairs","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
125172024-03-06T21:03:02.627Z{"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.37187468Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"":"downstairs","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
125182024-03-06T21:03:02.627Z{"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.371896189Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"":"downstairs","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
125192024-03-06T21:03:02.627Z{"msg":"Next flush: 203","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.371916147Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"":"downstairs","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
125202024-03-06T21:03:02.627Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.371937516Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"":"downstairs","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
125212024-03-06T21:03:02.627Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.371964704Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
125222024-03-06T21:03:02.627Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.371977783Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"client":"0","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
125232024-03-06T21:03:02.627Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.371992292Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"client":"1","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
125242024-03-06T21:03:02.627Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.37201463Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"client":"2","":"downstairs","session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
125252024-03-06T21:03:02.627Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
125262024-03-06T21:03:02.627Zlevel":30,"time":"2024-03-06T21:03:01.372049198Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
125272024-03-06T21:03:02.627Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.372063497Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
125282024-03-06T21:03:02.627Z{"msg":"f29f6a63-fdcd-4fa3-9ad8-68c3ff312d59 is now active with session: b61a19ae-70c4-4596-b5ef-b99b01613bba","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.372084146Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
125292024-03-06T21:03:02.627Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.372096475Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603,"session_id":"b61a19ae-70c4-4596-b5ef-b99b01613bba"}
125302024-03-06T21:03:02.627Z{"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:01.372115374Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603}
125312024-03-06T21:03:02.627ZLoading write count information from file "/tmp/test_repair_verify.data"
125322024-03-06T21:03:02.627ZRun Repair workload
125332024-03-06T21:03:02.634Z01/30 Write block 451 len 36864 data: 12 11 9 8 8 7 10 10 10
125342024-03-06T21:03:02.634Z02/30 Write block 379 len 28672 data: 11 11 10 11 11 13 13
125352024-03-06T21:03:02.634Z03/30 Read block 470 len 36864
125362024-03-06T21:03:02.634Z04/30 Read block 570 len 20480
125372024-03-06T21:03:02.634Z05/30 Read block 274 len 16384
125382024-03-06T21:03:02.634Z06/30 Write block 16 len 16384 data: 8 9 11 13
125392024-03-06T21:03:02.634Z07/30 Read block 5 len 28672
125402024-03-06T21:03:02.634Z08/30 Read block 583 len 4096
125412024-03-06T21:03:02.634Z09/30 Read block 452 len 40960
125422024-03-06T21:03:02.634Z10/30 Write block 227 len 20480 data: 10 8 7 9 10
125432024-03-06T21:03:02.640Z11/30 Read block 509 len 8192
125442024-03-06T21:03:02.641Z12/30 Write block 282 len 40960 data: 10 8 10 12 13 13 15 15 13 10
125452024-03-06T21:03:02.641Z13/30 Write block 320 len 36864 data: 9 12 12 10 11 10 11 8 8
125462024-03-06T21:03:02.641Z14/30 Read block 551 len 4096
125472024-03-06T21:03:02.641Z15/30 Read block 302 len 24576
125482024-03-06T21:03:02.641Z16/30 Write block 478 len 4096 data: 7
125492024-03-06T21:03:02.641Z17/30 Write block 60 len 4096 data: 15
125502024-03-06T21:03:02.641Z18/30 Read block 510 len 40960
125512024-03-06T21:03:02.641Z19/30 Read block 147 len 28672
125522024-03-06T21:03:02.646Z20/30 Read block 16 len 24576
125532024-03-06T21:03:02.646Z21/30 Write block 579 len 20480 data: 4 4 6 6 7
125542024-03-06T21:03:02.646Z22/30 Read block 578 len 4096
125552024-03-06T21:03:02.646Z23/30 Read block 32 len 8192
125562024-03-06T21:03:02.646Z24/30 Read block 547 len 32768
125572024-03-06T21:03:02.650Z25/30 Read block 221 len 16384
125582024-03-06T21:03:02.650Z26/30 Flush
125592024-03-06T21:03:02.655Z27/30 Write block 13 len 20480 data: 9 8 10 9 10
125602024-03-06T21:03:02.655Z28/30 Write block 434 len 28672 data: 12 10 11 13 12 13 11
125612024-03-06T21:03:02.655Z29/30 Read block 277 len 12288
125622024-03-06T21:03:02.655Z30/30 Write block 511 len 8192 data: 10 10
125632024-03-06T21:03:02.655Z----------------------------------------------------------------
125642024-03-06T21:03:02.656Z Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:30
125652024-03-06T21:03:02.656ZStates: Active Active Active
125662024-03-06T21:03:02.656ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
125672024-03-06T21:03:02.656Z 1 Acked 1000 Write 9 Done Done Done false
125682024-03-06T21:03:02.656Z 2 Acked 1001 Write 7 Done Done Done false
125692024-03-06T21:03:02.656Z 3 Acked 1002 Read 9 Done Done Done false
125702024-03-06T21:03:02.656Z 4 Acked 1003 Read 5 Done Done Done false
125712024-03-06T21:03:02.656Z 5 Acked 1004 Read 4 Done Done Done false
125722024-03-06T21:03:02.656Z 6 Acked 1005 Write 4 Done Done Done false
125732024-03-06T21:03:02.656Z 7 Acked 1006 Read 7 Done Done Done false
125742024-03-06T21:03:02.656Z 8 Acked 1007 Read 1 Done Done Done false
125752024-03-06T21:03:02.656Z 9 Acked 1008 Read 10 Sent Done Done false
125762024-03-06T21:03:02.656Z 10 Acked 1009 Write 5 Sent Done Done false
125772024-03-06T21:03:02.656Z 11 Acked 1010 Read 2 Sent Done Done false
125782024-03-06T21:03:02.656Z 12 Acked 1011 Write 10 Sent Done Done false
125792024-03-06T21:03:02.656Z 13 Acked 1012 Write 9 Sent Done Done false
125802024-03-06T21:03:02.656Z 14 Acked 1013 Read 1 Sent Done Done false
125812024-03-06T21:03:02.656Z 15 Acked 1014 Read 6 Sent Done Done false
125822024-03-06T21:03:02.656Z 16 Acked 1015 Write 1 Sent Done Done false
125832024-03-06T21:03:02.656Z 17 Acked 1016 Write 1 Sent Done Done false
125842024-03-06T21:03:02.656Z 18 Acked 1017 Read 10 Sent Done Done false
125852024-03-06T21:03:02.656Z 19 Acked 1018 Read 7 Sent Done Done false
125862024-03-06T21:03:02.656Z 20 Acked 1019 Read 6 Sent Done Done false
125872024-03-06T21:03:02.656Z 21 Acked 1020 Write 5 Sent Done Done false
125882024-03-06T21:03:02.657Z 22 Acked 1021 Read 1 Sent Done Done false
125892024-03-06T21:03:02.657Z 23 Acked 1022 Read 2 Sent Done Done false
125902024-03-06T21:03:02.657Z 24 Acked 1023 Read 8 Sent Done Done false
125912024-03-06T21:03:02.657Z 25 Acked 1024 Read 4 Sent Done Done false
125922024-03-06T21:03:02.657Z 26 Acked 1025 Flush 0 Sent Done Done false
125932024-03-06T21:03:02.657Z 27 Acked 1026 Write 5 Sent Done Done false
125942024-03-06T21:03:02.657Z 28 Acked 1027 Write 7 Sent Done Done false
125952024-03-06T21:03:02.657Z 29 Acked 1028 Read 3 Sent Done Done false
125962024-03-06T21:03:02.657Z 30 Acked 1029 Write 2 Sent Sent Sent false
125972024-03-06T21:03:02.657Z STATES DS:0 DS:1 DS:2 TOTAL
125982024-03-06T21:03:02.657Z New 0 0 0 0
125992024-03-06T21:03:02.657Z Sent 22 1 1 24
126002024-03-06T21:03:02.657Z Done 8 29 29 66
126012024-03-06T21:03:02.657Z Skipped 0 0 0 0
126022024-03-06T21:03:02.657Z Error 0 0 0 0
126032024-03-06T21:03:02.657ZLast Flush: 0 1025 1025
126042024-03-06T21:03:02.657ZDownstairs last five completed:
126052024-03-06T21:03:02.657ZUpstairs last five completed: 30 29 28 27 26
126062024-03-06T21:03:02.657Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:03:01.406272009Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1603}
126072024-03-06T21:03:02.662ZWrote out file "/tmp/test_repair_verify.data"
126082024-03-06T21:03:02.663Z
126092024-03-06T21:03:02.667ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
126102024-03-06T21:03:02.684ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
126112024-03-06T21:03:02.684Z 0 000-019 114 116 116 200 203 203 T T T <---
126122024-03-06T21:03:02.684Z 1 020-039 110 110 110 191 191 191 F F F
126132024-03-06T21:03:02.684Z 2 040-059 111 111 111 192 192 192 F F F
126142024-03-06T21:03:02.684Z 3 060-079 112 116 116 196 203 203 F F F <---
126152024-03-06T21:03:02.684Z 4 080-099 112 112 112 194 194 194 F F F
126162024-03-06T21:03:02.684Z 5 100-119 115 115 115 202 202 202 F F F
126172024-03-06T21:03:02.684Z 6 120-139 112 112 112 196 196 196 F F F
126182024-03-06T21:03:02.684Z 7 140-159 109 109 109 186 186 186 F F F
126192024-03-06T21:03:02.684Z 8 160-179 112 112 112 193 193 193 F F F
126202024-03-06T21:03:02.684Z 9 180-199 114 114 114 200 200 200 F F F
126212024-03-06T21:03:02.684Z 10 200-219 114 114 114 200 200 200 F F F
126222024-03-06T21:03:02.684Z 11 220-239 114 116 116 199 203 203 F F F <---
126232024-03-06T21:03:02.685Z 12 240-259 111 111 111 192 192 192 F F F
126242024-03-06T21:03:02.685Z 13 260-279 114 114 114 199 199 199 F F F
126252024-03-06T21:03:02.685Z 14 280-299 91 116 116 156 203 203 F F F <---
126262024-03-06T21:03:02.685Z 15 300-319 114 114 114 201 201 201 F F F
126272024-03-06T21:03:02.685Z 16 320-339 114 116 116 199 203 203 F F F <---
126282024-03-06T21:03:02.685Z 17 340-359 111 111 111 192 192 192 F F F
126292024-03-06T21:03:02.685Z 18 360-379 112 116 116 196 203 203 T F F <---
126302024-03-06T21:03:02.685Z 19 380-399 113 116 116 197 203 203 T F F <---
126312024-03-06T21:03:02.692Z 20 400-419 114 114 114 200 200 200 F F F
126322024-03-06T21:03:02.692Z 21 420-439 114 114 114 200 200 200 F T T <---
126332024-03-06T21:03:02.693Z 22 440-459 110 116 116 190 203 203 T T T <---
126342024-03-06T21:03:02.693Z 23 460-479 108 116 116 185 203 203 F F F <---
126352024-03-06T21:03:02.693Z 24 480-499 112 112 112 194 194 194 F F F
126362024-03-06T21:03:02.693Z 25 500-519 112 112 112 196 196 196 F T T <---
126372024-03-06T21:03:02.693Z 26 520-539 109 109 109 186 186 186 F F F
126382024-03-06T21:03:02.693Z 27 540-559 115 115 115 202 202 202 F F F
126392024-03-06T21:03:02.693Z 28 560-579 115 116 116 202 203 203 F F F <---
126402024-03-06T21:03:02.693Z 29 580-599 109 116 116 186 203 203 F F F <---
126412024-03-06T21:03:02.693ZMax gen: 116, Max flush: 203
126422024-03-06T21:03:02.693ZError: Difference in extent metadata found!
126432024-03-06T21:03:02.693ZA Difference in extent metadata is expected here
126442024-03-06T21:03:02.693ZOn loop 58
126452024-03-06T21:03:02.693Z
126462024-03-06T21:03:02.693ZVerifying data now
126472024-03-06T21:03:08.383ZNo change after verify
126482024-03-06T21:03:08.383ZLoop: 58 Downstairs dump after verify (and repair):
126492024-03-06T21:03:08.398ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
126502024-03-06T21:03:08.398Z 0 000-019 117 117 117 204 204 204 F F F
126512024-03-06T21:03:08.398Z 1 020-039 110 110 110 191 191 191 F F F
126522024-03-06T21:03:08.398Z 2 040-059 111 111 111 192 192 192 F F F
126532024-03-06T21:03:08.399Z 3 060-079 116 116 116 203 203 203 F F F
126542024-03-06T21:03:08.399Z 4 080-099 112 112 112 194 194 194 F F F
126552024-03-06T21:03:08.399Z 5 100-119 115 115 115 202 202 202 F F F
126562024-03-06T21:03:08.399Z 6 120-139 112 112 112 196 196 196 F F F
126572024-03-06T21:03:08.399Z 7 140-159 109 109 109 186 186 186 F F F
126582024-03-06T21:03:08.399Z 8 160-179 112 112 112 193 193 193 F F F
126592024-03-06T21:03:08.399Z 9 180-199 114 114 114 200 200 200 F F F
126602024-03-06T21:03:08.399Z 10 200-219 114 114 114 200 200 200 F F F
126612024-03-06T21:03:08.399Z 11 220-239 116 116 116 203 203 203 F F F
126622024-03-06T21:03:08.400Z 12 240-259 111 111 111 192 192 192 F F F
126632024-03-06T21:03:08.400Z 13 260-279 114 114 114 199 199 199 F F F
126642024-03-06T21:03:08.400Z 14 280-299 116 116 116 203 203 203 F F F
126652024-03-06T21:03:08.400Z 15 300-319 114 114 114 201 201 201 F F F
126662024-03-06T21:03:08.400Z 16 320-339 116 116 116 203 203 203 F F F
126672024-03-06T21:03:08.400Z 17 340-359 111 111 111 192 192 192 F F F
126682024-03-06T21:03:08.400Z 18 360-379 116 116 116 203 203 203 F F F
126692024-03-06T21:03:08.400Z 19 380-399 116 116 116 203 203 203 F F F
126702024-03-06T21:03:08.405Z 20 400-419 114 114 114 200 200 200 F F F
126712024-03-06T21:03:08.406Z 21 420-439 117 117 117 204 204 204 F F F
126722024-03-06T21:03:08.406Z 22 440-459 117 117 117 204 204 204 F F F
126732024-03-06T21:03:08.406Z 23 460-479 116 116 116 203 203 203 F F F
126742024-03-06T21:03:08.406Z 24 480-499 112 112 112 194 194 194 F F F
126752024-03-06T21:03:08.406Z 25 500-519 117 117 117 204 204 204 F F F
126762024-03-06T21:03:08.406Z 26 520-539 109 109 109 186 186 186 F F F
126772024-03-06T21:03:08.406Z 27 540-559 115 115 115 202 202 202 F F F
126782024-03-06T21:03:08.406Z 28 560-579 116 116 116 203 203 203 F F F
126792024-03-06T21:03:08.406Z 29 580-599 116 116 116 203 203 203 F F F
126802024-03-06T21:03:08.407ZMax gen: 117, Max flush: 204
126812024-03-06T21:03:08.407Z
126822024-03-06T21:03:08.407ZBegin loop 59
126832024-03-06T21:03:08.407ZDownstairs to restart: 0
126842024-03-06T21:03:08.446Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.173750742Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611}
126852024-03-06T21:03:08.446Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.174094788Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611Crucible runtime is spawned
126862024-03-06T21:03:08.447Z}
126872024-03-06T21:03:08.447Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.174130666Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611}
126882024-03-06T21:03:08.458Z{"msg":"Crucible 154ef0ca-6a9d-4c51-9ccb-8408ce66ecad has session id: edad3cf0-9598-45c1-abda-4b31ee0cee62","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.174167393Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
126892024-03-06T21:03:08.458Z{"msg":"Upstairs opts: Upstairs UUID: 154ef0ca-6a9d-4c51-9ccb-8408ce66ecad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.17421816Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
126902024-03-06T21:03:08.459Z{"msg":"Crucible stats registered with UUID: 154ef0ca-6a9d-4c51-9ccb-8408ce66ecad","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.174237398Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
126912024-03-06T21:03:08.459Z{"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.174267227Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611}
126922024-03-06T21:03:08.459Z{"msg":"154ef0ca-6a9d-4c51-9ccb-8408ce66ecad active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.17436068Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
126932024-03-06T21:03:08.459Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.174410486Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"client":"0","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
126942024-03-06T21:03:08.459Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.174429605Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"client":"0","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
126952024-03-06T21:03:08.460Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.174445694Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"client":"1","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
126962024-03-06T21:03:08.460Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.174474902Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"client":"1","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
126972024-03-06T21:03:08.460Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.17450207Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"client":"2","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
126982024-03-06T21:03:08.460Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.174529708Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"client":"2","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
126992024-03-06T21:03:08.460Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.174544507Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"":"io task","client":"0","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127002024-03-06T21:03:08.461Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.174586045Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127012024-03-06T21:03:08.461Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.174617932Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"":"io task","client":"2","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127022024-03-06T21:03:08.461Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.174767892Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"":"io task","client":"0","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127032024-03-06T21:03:08.461Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.174817398Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127042024-03-06T21:03:08.461Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.174845117Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"":"io task","client":"2","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127052024-03-06T21:03:08.462Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.175624273Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"client":"1","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127062024-03-06T21:03:08.462Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.17566439Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"client":"1","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127072024-03-06T21:03:08.462Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.175700648Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"client":"2","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127082024-03-06T21:03:08.462Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.175732086Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"client":"2","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127092024-03-06T21:03:08.462Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.176352593Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"client":"2","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127102024-03-06T21:03:08.462Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.176407649Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"client":"1","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127112024-03-06T21:03:08.463Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.17653747Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"client":"0","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127122024-03-06T21:03:08.463Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.176573448Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"client":"0","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127132024-03-06T21:03:08.463Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.177089232Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"client":"2","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127142024-03-06T21:03:08.463Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.177133599Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"client":"2","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127152024-03-06T21:03:08.463Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.177153137Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127162024-03-06T21:03:08.463Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.177174906Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"client":"1","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127172024-03-06T21:03:08.464Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.177189695Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"client":"1","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127182024-03-06T21:03:08.464Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.177203344Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127192024-03-06T21:03:08.464Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.177363773Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"client":"0","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127202024-03-06T21:03:08.464Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.177787564Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"client":"0","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127212024-03-06T21:03:08.464Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.177826941Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"client":"0","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127222024-03-06T21:03:08.465Z{"msg":"[0]R flush_numbers[0..12]: [204, 191, 192, 203, 194, 202, 196, 186, 193, 200, 200, 203]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.17784414Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"":"downstairs","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127232024-03-06T21:03:08.465Z{"msg":"[0]R generation[0..12]: [117, 110, 111, 116, 112, 115, 112, 109, 112, 114, 114, 116]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.177859809Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"":"downstairs","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127242024-03-06T21:03:08.465Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.177881477Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"":"downstairs","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127252024-03-06T21:03:08.465Z{"msg":"[1]R flush_numbers[0..12]: [204, 191, 192, 203, 194, 202, 196, 186, 193, 200, 200, 203]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.177896006Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"":"downstairs","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127262024-03-06T21:03:08.465Z{"msg":"[1]R generation[0..12]: [117, 110, 111, 116, 112, 115, 112, 109, 112, 114, 114, 116]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.177916635Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"":"downstairs","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127272024-03-06T21:03:08.466Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.177931444Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"":"downstairs","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127282024-03-06T21:03:08.466Z{"msg":"[2]R flush_numbers[0..12]: [204, 191, 192, 203, 194, 202, 196, 186, 193, 200, 200, 203]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.177945653Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"":"downstairs","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127292024-03-06T21:03:08.466Z{"msg":"[2]R generation[0..12]: [117, 110, 111, 116, 112, 115, 112, 109, 112, 114, 114, 116]","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO
127302024-03-06T21:03:08.466Ztime":"2024-03-06T21:03:07.177966592Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"":"downstairs","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127312024-03-06T21:03:08.466Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.17798612Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"":"downstairs","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127322024-03-06T21:03:08.466Z{"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.178006929Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"":"downstairs","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127332024-03-06T21:03:08.467Z{"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.178020818Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"":"downstairs","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127342024-03-06T21:03:08.467Z{"msg":"Next flush: 205","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.178046576Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"":"downstairs","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127352024-03-06T21:03:08.467Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.178059985Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"":"downstairs","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127362024-03-06T21:03:08.467Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.178080244Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127372024-03-06T21:03:08.467Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.178092633Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"client":"0","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127382024-03-06T21:03:08.467Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.178112471Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"client":"1","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127392024-03-06T21:03:08.468Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.178125821Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"client":"2","":"downstairs","session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127402024-03-06T21:03:08.468Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.178145499Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127412024-03-06T21:03:08.468Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.178158138Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127422024-03-06T21:03:08.468Z{"msg":"154ef0ca-6a9d-4c51-9ccb-8408ce66ecad is now active with session: edad3cf0-9598-45c1-abda-4b31ee0cee62","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.178177187Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127432024-03-06T21:03:08.468Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.178189556Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611,"session_id":"edad3cf0-9598-45c1-abda-4b31ee0cee62"}
127442024-03-06T21:03:08.468Z{"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:07.178207985Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611}
127452024-03-06T21:03:08.468ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
127462024-03-06T21:03:08.468ZLoading write count information from file "/tmp/test_repair_verify.data"
127472024-03-06T21:03:08.469ZRun Repair workload
127482024-03-06T21:03:08.469Z01/30 Write block 101 len 16384 data: 9 9 10 8
127492024-03-06T21:03:08.469Z02/30 Read block 231 len 4096
127502024-03-06T21:03:08.469Z03/30 Write block 269 len 12288 data: 9 9 10
127512024-03-06T21:03:08.469Z04/30 Write block 214 len 20480 data: 7 10 12 14 11
127522024-03-06T21:03:08.469Z05/30 Write block 246 len 8192 data: 7 6
127532024-03-06T21:03:08.469Z06/30 Read block 334 len 40960
127542024-03-06T21:03:08.469Z07/30 Read block 100 len 28672
127552024-03-06T21:03:08.469Z08/30 Read block 411 len 16384
127562024-03-06T21:03:08.469Z09/30 Flush
127572024-03-06T21:03:08.469Z10/30 Write block 491 len 28672 data: 9 9 11 9 10 10 10
127582024-03-06T21:03:08.469Z11/30 Read block 237 len 36864
127592024-03-06T21:03:08.469Z12/30 Write block 311 len 36864 data: 11 11 10 10 10 8 6 10 10
127602024-03-06T21:03:08.469Z13/30 Write block 143 len 40960 data: 5 4 4 4 5 8 9 11 10 9
127612024-03-06T21:03:08.469Z14/30 Write block 112 len 4096 data: 5
127622024-03-06T21:03:08.469Z15/30 Read block 47 len 16384
127632024-03-06T21:03:08.469Z16/30 Write block 369 len 4096 data: 10
127642024-03-06T21:03:08.469Z17/30 Write block 413 len 24576 data: 7 5 5 7 7 8
127652024-03-06T21:03:08.469Z18/30 Write block 238 len 28672 data: 15 15 11 12 11 9 9
127662024-03-06T21:03:08.469Z19/30 Write block 326 len 36864 data: 12 9 9 8 7 5 4 6 5
127672024-03-06T21:03:08.469Z20/30 Read block 208 len 20480
127682024-03-06T21:03:08.469Z21/30 Read block 56 len 28672
127692024-03-06T21:03:08.469Z22/30 Read block 370 len 12288
127702024-03-06T21:03:08.469Z23/30 Write block 39 len 4096 data: 8
127712024-03-06T21:03:08.470Z24/30 Write block 454 len 16384 data: 9 9 8 11
127722024-03-06T21:03:08.477Z25/30 Read block 182 len 16384
127732024-03-06T21:03:08.477Z26/30 Write block 375 len 36864 data: 12 9 8 9 12 12 11 12 12
127742024-03-06T21:03:08.477Z27/30 Write block 541 len 8192 data: 9 12
127752024-03-06T21:03:08.477Z28/30 Write block 41 len 16384 data: 10 12 11 10
127762024-03-06T21:03:08.477Z29/30 Write block 154 len 20480 data: 9 11 14 14 11
127772024-03-06T21:03:08.478Z30/30 Read block 116 len 12288
127782024-03-06T21:03:08.478Z----------------------------------------------------------------
127792024-03-06T21:03:08.478Z Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:30
127802024-03-06T21:03:08.478ZStates: Active Active Active
127812024-03-06T21:03:08.478ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
127822024-03-06T21:03:08.478Z 1 Acked 1000 Write 4 Done Done Done false
127832024-03-06T21:03:08.478Z 2 Acked 1001 Read 1 Done Done Done false
127842024-03-06T21:03:08.479Z 3 Acked 1002 Write 3 Done Done Done false
127852024-03-06T21:03:08.479Z 4 Acked 1003 Write 5 Sent Done Done false
127862024-03-06T21:03:08.479Z 5 Acked 1004 Write 2 Sent Done Done false
127872024-03-06T21:03:08.479Z 6 Acked 1005 Read 10 Sent Done Done false
127882024-03-06T21:03:08.479Z 7 Acked 1006 Read 7 Sent Done Done false
127892024-03-06T21:03:08.479Z 8 Acked 1007 Read 4 Sent Done Done false
127902024-03-06T21:03:08.480Z 9 Acked 1008 Flush 0 Sent Done Done false
127912024-03-06T21:03:08.480Z 10 Acked 1009 Write 7 Sent Done Done false
127922024-03-06T21:03:08.480Z 11 Acked 1010 Read 9 Sent Done Done false
127932024-03-06T21:03:08.480Z 12 Acked 1011 Write 9 Sent Done Done false
127942024-03-06T21:03:08.480Z 13 Acked 1012 Write 10 Sent Done Done false
127952024-03-06T21:03:08.480Z 14 Acked 1013 Write 1 Sent Done Done false
127962024-03-06T21:03:08.481Z 15 Acked 1014 Read 4 Sent Done Done false
127972024-03-06T21:03:08.481Z 16 Acked 1015 Write 1 Sent Done Done false
127982024-03-06T21:03:08.481Z 17 Acked 1016 Write 6 Sent Done Done false
127992024-03-06T21:03:08.481Z 18 Acked 1017 Write 7 Sent Done Done false
128002024-03-06T21:03:08.481Z 19 Acked 1018 Write 9 Sent Done Done false
128012024-03-06T21:03:08.481Z 20 Acked 1019 Read 5 Sent Done Done false
128022024-03-06T21:03:08.482Z 21 Acked 1020 Read 7 Sent Done Done false
128032024-03-06T21:03:08.482Z 22 Acked 1021 Read 3 Sent Done Done false
128042024-03-06T21:03:08.482Z 23 Acked 1022 Write 1 Sent Done Done false
128052024-03-06T21:03:08.482Z 24 Acked 1023 Write 4 Sent Done Done false
128062024-03-06T21:03:08.482Z 25 Acked 1024 Read 4 Sent Done Done false
128072024-03-06T21:03:08.482Z 26 Acked 1025 Write 9 Sent Done Done false
128082024-03-06T21:03:08.482Z 27 Acked 1026 Write 2 Sent Done Done false
128092024-03-06T21:03:08.482Z 28 Acked 1027 Write 4 Sent Done Done false
128102024-03-06T21:03:08.482Z 29 Acked 1028 Write 5 Sent Done Sent false
128112024-03-06T21:03:08.482Z 30 Acked 1029 Read 3 Sent Done Sent false
128122024-03-06T21:03:08.482Z STATES DS:0 DS:1 DS:2 TOTAL
128132024-03-06T21:03:08.482Z New 0 0 0 0
128142024-03-06T21:03:08.482Z Sent 27 0 2 29
128152024-03-06T21:03:08.483Z Done 3 30 28 61
128162024-03-06T21:03:08.483Z Skipped 0 0 0 0
128172024-03-06T21:03:08.483Z Error 0 0 0 0
128182024-03-06T21:03:08.483ZLast Flush: 0 1008 1008
128192024-03-06T21:03:08.483ZDownstairs last five completed:
128202024-03-06T21:03:08.483ZUpstairs last five completed: 30 29 28 27 26
128212024-03-06T21:03:08.483Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:03:07.210585104Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1611}
128222024-03-06T21:03:08.483ZWrote out file "/tmp/test_repair_verify.data"
128232024-03-06T21:03:08.483Z
128242024-03-06T21:03:08.483ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
128252024-03-06T21:03:08.517ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
128262024-03-06T21:03:08.517Z 0 000-019 117 117 117 204 204 204 F F F
128272024-03-06T21:03:08.517Z 1 020-039 110 110 110 191 191 191 F T T <---
128282024-03-06T21:03:08.517Z 2 040-059 111 111 111 192 192 192 F T T <---
128292024-03-06T21:03:08.517Z 3 060-079 116 116 116 203 203 203 F F F
128302024-03-06T21:03:08.517Z 4 080-099 112 112 112 194 194 194 F F F
128312024-03-06T21:03:08.517Z 5 100-119 115 118 118 202 205 205 T T T <---
128322024-03-06T21:03:08.517Z 6 120-139 112 112 112 196 196 196 F F F
128332024-03-06T21:03:08.517Z 7 140-159 109 109 109 186 186 186 F T T <---
128342024-03-06T21:03:08.517Z 8 160-179 112 112 112 193 193 193 F F F
128352024-03-06T21:03:08.517Z 9 180-199 114 114 114 200 200 200 F F F
128362024-03-06T21:03:08.517Z 10 200-219 114 118 118 200 205 205 F F F <---
128372024-03-06T21:03:08.517Z 11 220-239 116 116 116 203 203 203 F T T <---
128382024-03-06T21:03:08.518Z 12 240-259 111 118 118 192 205 205 F T T <---
128392024-03-06T21:03:08.518Z 13 260-279 114 118 118 199 205 205 T F F <---
128402024-03-06T21:03:08.518Z 14 280-299 116 116 116 203 203 203 F F F
128412024-03-06T21:03:08.518Z 15 300-319 114 114 114 201 201 201 F T T <---
128422024-03-06T21:03:08.518Z 16 320-339 116 116 116 203 203 203 F T T <---
128432024-03-06T21:03:08.518Z 17 340-359 111 111 111 192 192 192 F F F
128442024-03-06T21:03:08.518Z 18 360-379 116 116 116 203 203 203 F T T <---
128452024-03-06T21:03:08.518Z 19 380-399 116 116 116 203 203 203 F T T <---
128462024-03-06T21:03:08.518Z 20 400-419 114 114 114 200 200 200 F T T <---
128472024-03-06T21:03:08.518Z 21 420-439 117 117 117 204 204 204 F F F
128482024-03-06T21:03:08.518Z 22 440-459 117 117 117 204 204 204 F T T <---
128492024-03-06T21:03:08.518Z 23 460-479 116 116 116 203 203 203 F F F
128502024-03-06T21:03:08.518Z 24 480-499 112 112 112 194 194 194 F T T <---
128512024-03-06T21:03:08.518Z 25 500-519 117 117 117 204 204 204 F F F
128522024-03-06T21:03:08.518Z 26 520-539 109 109 109 186 186 186 F F F
128532024-03-06T21:03:08.518Z 27 540-559 115 115 115 202 202 202 F T T <---
128542024-03-06T21:03:08.518Z 28 560-579 116 116 116 203 203 203 F F F
128552024-03-06T21:03:08.518Z 29 580-599 116 116 116 203 203 203 F F F
128562024-03-06T21:03:08.518ZMax gen: 118, Max flush: 205
128572024-03-06T21:03:08.518ZError: Difference in extent metadata found!
128582024-03-06T21:03:08.518ZA Difference in extent metadata is expected here
128592024-03-06T21:03:08.518ZOn loop 59
128602024-03-06T21:03:08.518Z
128612024-03-06T21:03:08.519ZVerifying data now
128622024-03-06T21:03:14.340ZNo change after verify
128632024-03-06T21:03:14.340ZLoop: 59 Downstairs dump after verify (and repair):
128642024-03-06T21:03:14.353ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
128652024-03-06T21:03:14.353Z 0 000-019 117 117 117 204 204 204 F F F
128662024-03-06T21:03:14.353Z 1 020-039 119 119 119 206 206 206 F F F
128672024-03-06T21:03:14.353Z 2 040-059 119 119 119 206 206 206 F F F
128682024-03-06T21:03:14.353Z 3 060-079 116 116 116 203 203 203 F F F
128692024-03-06T21:03:14.353Z 4 080-099 112 112 112 194 194 194 F F F
128702024-03-06T21:03:14.353Z 5 100-119 119 119 119 206 206 206 F F F
128712024-03-06T21:03:14.353Z 6 120-139 112 112 112 196 196 196 F F F
128722024-03-06T21:03:14.353Z 7 140-159 119 119 119 206 206 206 F F F
128732024-03-06T21:03:14.353Z 8 160-179 112 112 112 193 193 193 F F F
128742024-03-06T21:03:14.353Z 9 180-199 114 114 114 200 200 200 F F F
128752024-03-06T21:03:14.353Z 10 200-219 118 118 118 205 205 205 F F F
128762024-03-06T21:03:14.353Z 11 220-239 119 119 119 206 206 206 F F F
128772024-03-06T21:03:14.353Z 12 240-259 119 119 119 206 206 206 F F F
128782024-03-06T21:03:14.353Z 13 260-279 118 118 118 205 205 205 F F F
128792024-03-06T21:03:14.353Z 14 280-299 116 116 116 203 203 203 F F F
128802024-03-06T21:03:14.353Z 15 300-319 119 119 119 206 206 206 F F F
128812024-03-06T21:03:14.354Z 16 320-339 119 119 119 206 206 206 F F F
128822024-03-06T21:03:14.354Z 17 340-359 111 111 111 192 192 192 F F F
128832024-03-06T21:03:14.354Z 18 360-379 119 119 119 206 206 206 F F F
128842024-03-06T21:03:14.354Z 19 380-399 119 119 119 206 206 206 F F F
128852024-03-06T21:03:14.354Z 20 400-419 119 119 119 206 206 206 F F F
128862024-03-06T21:03:14.354Z 21 420-439 117 117 117 204 204 204 F F F
128872024-03-06T21:03:14.354Z 22 440-459 119 119 119 206 206 206 F F F
128882024-03-06T21:03:14.354Z 23 460-479 116 116 116 203 203 203 F F F
128892024-03-06T21:03:14.354Z 24 480-499 119 119 119 206 206 206 F F F
128902024-03-06T21:03:14.357Z 25 500-519 117 117 117 204 204 204 F F F
128912024-03-06T21:03:14.358Z 26 520-539 109 109 109 186 186 186 F F F
128922024-03-06T21:03:14.358Z 27 540-559 119 119 119 206 206 206 F F F
128932024-03-06T21:03:14.358Z 28 560-579 116 116 116 203 203 203 F F F
128942024-03-06T21:03:14.358Z 29 580-599 116 116 116 203 203 203 F F F
128952024-03-06T21:03:14.358ZMax gen: 119, Max flush: 206
128962024-03-06T21:03:14.358Z
128972024-03-06T21:03:14.358ZBegin loop 60
128982024-03-06T21:03:14.358ZDownstairs to restart: 0
128992024-03-06T21:03:14.381Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.131843008Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619}
129002024-03-06T21:03:14.382Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.132167276Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619}
129012024-03-06T21:03:14.382Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
129022024-03-06T21:03:14.387Z,"time":"2024-03-06T21:03:13.132210723Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619}
129032024-03-06T21:03:14.387Z{"msg":"Crucible 77c3b740-e3bc-4a35-8ebf-93579c87325a has session id: 3700638c-0369-4573-9eb9-67df8e5bd1d4","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.132277528Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129042024-03-06T21:03:14.387Z{"msg":"Upstairs opts: Upstairs UUID: 77c3b740-e3bc-4a35-8ebf-93579c87325a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.132328405Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129052024-03-06T21:03:14.387Z{"msg":"Crucible stats registered with UUID: 77c3b740-e3bc-4a35-8ebf-93579c87325a","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.132354793Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129062024-03-06T21:03:14.387Z{"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.13239277Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619}
129072024-03-06T21:03:14.387Z{"msg":"77c3b740-e3bc-4a35-8ebf-93579c87325a active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.132532831Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129082024-03-06T21:03:14.387Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.132572628Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"client":"0","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129092024-03-06T21:03:14.387Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.132600486Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"client":"0","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129102024-03-06T21:03:14.387Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.132625074Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"client":"1","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129112024-03-06T21:03:14.387Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.132650882Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"client":"1","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129122024-03-06T21:03:14.387Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.132674341Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"client":"2","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129132024-03-06T21:03:14.387Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.132715728Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"client":"2","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129142024-03-06T21:03:14.387Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.132740166Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"":"io task","client":"2","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129152024-03-06T21:03:14.387Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.132782664Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"":"io task","client":"0","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129162024-03-06T21:03:14.387Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.132806942Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129172024-03-06T21:03:14.387Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.132987899Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"":"io task","client":"2","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129182024-03-06T21:03:14.387Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.133230602Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"":"io task","client":"0","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129192024-03-06T21:03:14.387Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.13326957Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129202024-03-06T21:03:14.388Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.134015668Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"client":"1","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129212024-03-06T21:03:14.388Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.134057525Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"client":"1","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129222024-03-06T21:03:14.388Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.134084304Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"client":"2","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129232024-03-06T21:03:14.388Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.134106922Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"client":"2","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129242024-03-06T21:03:14.388Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.134655484Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"client":"2","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129252024-03-06T21:03:14.388Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.134741938Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"client":"1","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129262024-03-06T21:03:14.388Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.135397823Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"client":"2","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129272024-03-06T21:03:14.388Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.135433501Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"client":"2","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129282024-03-06T21:03:14.388Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.135458049Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129292024-03-06T21:03:14.388Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.135511515Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"client":"0","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129302024-03-06T21:03:14.388Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.135550803Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"client":"0","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129312024-03-06T21:03:14.388Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.13558739Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"client":"1","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129322024-03-06T21:03:14.388Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.135629257Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"client":"1","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129332024-03-06T21:03:14.388Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.135660125Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129342024-03-06T21:03:14.388Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.136101135Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"client":"0","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129352024-03-06T21:03:14.393Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.136590831Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"client":"0","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129362024-03-06T21:03:14.393Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.136632558Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"client":"0","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129372024-03-06T21:03:14.393Z{"msg":"[0]R flush_numbers[0..12]: [204, 206, 206, 203, 194, 206, 196, 206, 193, 200, 205, 206]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.136673225Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"":"downstairs","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129382024-03-06T21:03:14.394Z{"msg":"[0]R generation[0..12]: [117, 119, 119, 116, 112, 119, 112, 119, 112, 114, 118, 119]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.136700023Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"":"downstairs","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129392024-03-06T21:03:14.394Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.136715512Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"":"downstairs","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129402024-03-06T21:03:14.394Z{"msg":"[1]R flush_numbers[0..12]: [204, 206, 206, 203, 194, 206, 196, 206, 193, 200, 205, 206]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.136736621Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"":"downstairs","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129412024-03-06T21:03:14.394Z{"msg":"[1]R generation[0..12]: [117, 119, 119, 116, 112, 119, 112, 119, 112, 114, 118, 119]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.13675077Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"":"downstairs","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129422024-03-06T21:03:14.394Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.136765099Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"":"downstairs","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129432024-03-06T21:03:14.394Z{"msg":"[2]R flush_numbers[0..12]: [204, 206, 206, 203, 194, 206, 196, 206, 193, 200, 205, 206]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.136778758Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"":"downstairs","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129442024-03-06T21:03:14.394Z{"msg":"[2]R generation[0..12]: [117, 119, 119, 116, 112, 119, 112, 119, 112, 114, 118, 119]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.136799277Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"":"downstairs","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129452024-03-06T21:03:14.394Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.136813596Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO
129462024-03-06T21:03:14.394Zsession_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129472024-03-06T21:03:14.394Z{"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.136839814Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"":"downstairs","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129482024-03-06T21:03:14.394Z{"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.136853413Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"":"downstairs","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129492024-03-06T21:03:14.394Z{"msg":"Next flush: 207","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.136866542Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"":"downstairs","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129502024-03-06T21:03:14.394Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.136879521Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"":"downstairs","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129512024-03-06T21:03:14.394Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.13689332Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129522024-03-06T21:03:14.394Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.136915259Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"client":"0","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129532024-03-06T21:03:14.394Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.136948186Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"client":"1","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129542024-03-06T21:03:14.394Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.136963685Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"client":"2","":"downstairs","session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129552024-03-06T21:03:14.394Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.136985114Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129562024-03-06T21:03:14.394Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.136998053Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129572024-03-06T21:03:14.394Z{"msg":"77c3b740-e3bc-4a35-8ebf-93579c87325a is now active with session: 3700638c-0369-4573-9eb9-67df8e5bd1d4","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.137011342Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129582024-03-06T21:03:14.394Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.137023661Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619,"session_id":"3700638c-0369-4573-9eb9-67df8e5bd1d4"}
129592024-03-06T21:03:14.394Z{"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:13.13704388Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619}
129602024-03-06T21:03:14.394ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
129612024-03-06T21:03:14.395ZLoading write count information from file "/tmp/test_repair_verify.data"
129622024-03-06T21:03:14.395ZRun Repair workload
129632024-03-06T21:03:14.395Z01/30 Flush
129642024-03-06T21:03:14.395Z02/30 Write block 410 len 20480 data: 10 8 6 8 6
129652024-03-06T21:03:14.395Z03/30 Read block 185 len 40960
129662024-03-06T21:03:14.395Z04/30 Read block 153 len 20480
129672024-03-06T21:03:14.398Z05/30 Read block 526 len 20480
129682024-03-06T21:03:14.398Z06/30 Flush
129692024-03-06T21:03:14.398Z07/30 Write block 578 len 40960 data: 4 5 5 7 7 8 10 11 13 14
129702024-03-06T21:03:14.398Z08/30 Read block 350 len 8192
129712024-03-06T21:03:14.398Z09/30 Write block 306 len 28672 data: 12 12 12 12 11 12 12
129722024-03-06T21:03:14.402Z10/30 Write block 235 len 20480 data: 11 12 14 16 16
129732024-03-06T21:03:14.402Z11/30 Read block 382 len 20480
129742024-03-06T21:03:14.402Z12/30 Write block 503 len 28672 data: 7 8 8 8 9 8 9
129752024-03-06T21:03:14.402Z13/30 Read block 491 len 8192
129762024-03-06T21:03:14.405Z14/30 Write block 565 len 4096 data: 8
129772024-03-06T21:03:14.405Z15/30 Write block 368 len 36864 data: 7 11 11 12 11 12 11 13 10
129782024-03-06T21:03:14.405Z16/30 Write block 273 len 12288 data: 9 13 11
129792024-03-06T21:03:14.409Z17/30 Read block 427 len 40960
129802024-03-06T21:03:14.409Z18/30 Read block 569 len 24576
129812024-03-06T21:03:14.409Z19/30 Write block 77 len 36864 data: 4 7 8 8 8 9 9 7 5
129822024-03-06T21:03:14.409Z20/30 Read block 208 len 40960
129832024-03-06T21:03:14.412Z21/30 Read block 443 len 32768
129842024-03-06T21:03:14.412Z22/30 Read block 581 len 40960
129852024-03-06T21:03:14.415Z23/30 Write block 106 len 4096 data: 11
129862024-03-06T21:03:14.415Z24/30 Write block 212 len 40960 data: 8 7 8 11 13 15 12 8 5 6
129872024-03-06T21:03:14.415Z25/30 Write block 252 len 16384 data: 5 5 5 5
129882024-03-06T21:03:14.416Z26/30 Read block 53 len 12288
129892024-03-06T21:03:14.419Z27/30 Write block 502 len 40960 data: 7 8 9 9 9 10 9 10 11 11
129902024-03-06T21:03:14.419Z28/30 Read block 431 len 8192
129912024-03-06T21:03:14.422Z29/30 Write block 192 len 32768 data: 11 8 8 9 12 12 13 11
129922024-03-06T21:03:14.422Z30/30 Write block 407 len 24576 data: 12 13 11 11 9 7
129932024-03-06T21:03:14.422Z----------------------------------------------------------------
129942024-03-06T21:03:14.422Z Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:29
129952024-03-06T21:03:14.422ZStates: Active Active Active
129962024-03-06T21:03:14.422ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
129972024-03-06T21:03:14.423Z 2 Acked 1001 Write 5 Done Done Done false
129982024-03-06T21:03:14.423Z 3 Acked 1002 Read 10 Done Done Done false
129992024-03-06T21:03:14.423Z 4 Acked 1003 Read 5 Sent Done Done false
130002024-03-06T21:03:14.423Z 5 Acked 1004 Read 5 Sent Done Done false
130012024-03-06T21:03:14.423Z 6 Acked 1005 Flush 0 Sent Done Done false
130022024-03-06T21:03:14.423Z 7 Acked 1006 Write 10 Sent Done Done false
130032024-03-06T21:03:14.423Z 8 Acked 1007 Read 2 Sent Done Done false
130042024-03-06T21:03:14.423Z 9 Acked 1008 Write 7 Sent Done Done false
130052024-03-06T21:03:14.423Z 10 Acked 1009 Write 5 Sent Done Done false
130062024-03-06T21:03:14.423Z 11 Acked 1010 Read 5 Sent Done Done false
130072024-03-06T21:03:14.423Z 12 Acked 1011 Write 7 Sent Done Done false
130082024-03-06T21:03:14.423Z 13 Acked 1012 Read 2 Sent Done Done false
130092024-03-06T21:03:14.423Z 14 Acked 1013 Write 1 Sent Done Done false
130102024-03-06T21:03:14.423Z 15 Acked 1014 Write 9 Sent Done Done false
130112024-03-06T21:03:14.423Z 16 Acked 1015 Write 3 Sent Done Done false
130122024-03-06T21:03:14.423Z 17 Acked 1016 Read 10 Sent Done Done false
130132024-03-06T21:03:14.423Z 18 Acked 1017 Read 6 Sent Done Done false
130142024-03-06T21:03:14.423Z 19 Acked 1018 Write 9 Sent Done Done false
130152024-03-06T21:03:14.423Z 20 Acked 1019 Read 10 Sent Done Done false
130162024-03-06T21:03:14.423Z 21 Acked 1020 Read 8 Sent Done Done false
130172024-03-06T21:03:14.423Z 22 Acked 1021 Read 10 Sent Done Done false
130182024-03-06T21:03:14.423Z 23 Acked 1022 Write 1 Sent Done Done false
130192024-03-06T21:03:14.423Z 24 Acked 1023 Write 10 Sent Done Done false
130202024-03-06T21:03:14.423Z 25 Acked 1024 Write 4 Sent Done Done false
130212024-03-06T21:03:14.424Z 26 Acked 1025 Read 3 Sent Done Done false
130222024-03-06T21:03:14.424Z 27 Acked 1026 Write 10 Sent Done Done false
130232024-03-06T21:03:14.424Z 28 Acked 1027 Read 2 Sent Done Done false
130242024-03-06T21:03:14.424Z 29 Acked 1028 Write 8 Sent Done Sent false
130252024-03-06T21:03:14.424Z 30 Acked 1029 Write 6 Sent Sent Sent false
130262024-03-06T21:03:14.424Z STATES DS:0 DS:1 DS:2 TOTAL
130272024-03-06T21:03:14.424Z New 0 0 0 0
130282024-03-06T21:03:14.424Z Sent 27 1 2 30
130292024-03-06T21:03:14.424Z Done 2 28 27 57
130302024-03-06T21:03:14.424Z Skipped 0 0 0 0
130312024-03-06T21:03:14.424Z Error 0 0 0 0
130322024-03-06T21:03:14.424ZLast Flush: 1000 1005 1005
130332024-03-06T21:03:14.424ZDownstairs last five completed: 1000
130342024-03-06T21:03:14.424ZUpstairs last five completed: 30 29 28 27 26
130352024-03-06T21:03:14.424Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:03:13.171911957Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1619}
130362024-03-06T21:03:14.424ZWrote out file "/tmp/test_repair_verify.data"
130372024-03-06T21:03:14.432Z
130382024-03-06T21:03:14.435ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
130392024-03-06T21:03:14.453ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
130402024-03-06T21:03:14.453Z 0 000-019 117 117 117 204 204 204 F F F
130412024-03-06T21:03:14.453Z 1 020-039 119 119 119 206 206 206 F F F
130422024-03-06T21:03:14.453Z 2 040-059 119 119 119 206 206 206 F F F
130432024-03-06T21:03:14.453Z 3 060-079 116 116 116 203 203 203 F T T <---
130442024-03-06T21:03:14.453Z 4 080-099 112 112 112 194 194 194 F T T <---
130452024-03-06T21:03:14.453Z 5 100-119 119 119 119 206 206 206 F T T <---
130462024-03-06T21:03:14.453Z 6 120-139 112 112 112 196 196 196 F F F
130472024-03-06T21:03:14.453Z 7 140-159 119 119 119 206 206 206 F F F
130482024-03-06T21:03:14.454Z 8 160-179 112 112 112 193 193 193 F F F
130492024-03-06T21:03:14.454Z 9 180-199 114 114 114 200 200 200 F T T <---
130502024-03-06T21:03:14.454Z 10 200-219 118 118 118 205 205 205 F T T <---
130512024-03-06T21:03:14.454Z 11 220-239 119 119 119 206 206 206 F T T <---
130522024-03-06T21:03:14.454Z 12 240-259 119 119 119 206 206 206 F T T <---
130532024-03-06T21:03:14.454Z 13 260-279 118 118 118 205 205 205 F T T <---
130542024-03-06T21:03:14.454Z 14 280-299 116 116 116 203 203 203 F F F
130552024-03-06T21:03:14.454Z 15 300-319 119 119 119 206 206 206 F T T <---
130562024-03-06T21:03:14.454Z 16 320-339 119 119 119 206 206 206 F F F
130572024-03-06T21:03:14.457Z 17 340-359 111 111 111 192 192 192 F F F
130582024-03-06T21:03:14.458Z 18 360-379 119 119 119 206 206 206 F T T <---
130592024-03-06T21:03:14.458Z 19 380-399 119 119 119 206 206 206 F F F
130602024-03-06T21:03:14.458Z 20 400-419 119 120 120 206 208 208 T T T <---
130612024-03-06T21:03:14.458Z 21 420-439 117 117 117 204 204 204 F F F
130622024-03-06T21:03:14.458Z 22 440-459 119 119 119 206 206 206 F F F
130632024-03-06T21:03:14.458Z 23 460-479 116 116 116 203 203 203 F F F
130642024-03-06T21:03:14.458Z 24 480-499 119 119 119 206 206 206 F F F
130652024-03-06T21:03:14.458Z 25 500-519 117 117 117 204 204 204 F T T <---
130662024-03-06T21:03:14.458Z 26 520-539 109 109 109 186 186 186 F F F
130672024-03-06T21:03:14.458Z 27 540-559 119 119 119 206 206 206 F F F
130682024-03-06T21:03:14.458Z 28 560-579 116 116 116 203 203 203 F T T <---
130692024-03-06T21:03:14.458Z 29 580-599 116 116 116 203 203 203 F T T <---
130702024-03-06T21:03:14.458ZMax gen: 120, Max flush: 208
130712024-03-06T21:03:14.458ZError: Difference in extent metadata found!
130722024-03-06T21:03:14.461ZA Difference in extent metadata is expected here
130732024-03-06T21:03:14.461ZOn loop 60
130742024-03-06T21:03:14.461Z
130752024-03-06T21:03:14.461ZVerifying data now
130762024-03-06T21:03:19.225ZNo change after verify
130772024-03-06T21:03:19.225ZLoop: 60 Downstairs dump after verify (and repair):
130782024-03-06T21:03:19.240ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
130792024-03-06T21:03:19.240Z 0 000-019 117 117 117 204 204 204 F F F
130802024-03-06T21:03:19.241Z 1 020-039 119 119 119 206 206 206 F F F
130812024-03-06T21:03:19.241Z 2 040-059 119 119 119 206 206 206 F F F
130822024-03-06T21:03:19.241Z 3 060-079 121 121 121 209 209 209 F F F
130832024-03-06T21:03:19.241Z 4 080-099 121 121 121 209 209 209 F F F
130842024-03-06T21:03:19.242Z 5 100-119 121 121 121 209 209 209 F F F
130852024-03-06T21:03:19.242Z 6 120-139 112 112 112 196 196 196 F F F
130862024-03-06T21:03:19.242Z 7 140-159 119 119 119 206 206 206 F F F
130872024-03-06T21:03:19.242Z 8 160-179 112 112 112 193 193 193 F F F
130882024-03-06T21:03:19.242Z 9 180-199 121 121 121 209 209 209 F F F
130892024-03-06T21:03:19.243Z 10 200-219 121 121 121 209 209 209 F F F
130902024-03-06T21:03:19.243Z 11 220-239 121 121 121 209 209 209 F F F
130912024-03-06T21:03:19.243Z 12 240-259 121 121 121 209 209 209 F F F
130922024-03-06T21:03:19.244Z 13 260-279 121 121 121 209 209 209 F F F
130932024-03-06T21:03:19.244Z 14 280-299 116 116 116 203 203 203 F F F
130942024-03-06T21:03:19.244Z 15 300-319 121 121 121 209 209 209 F F F
130952024-03-06T21:03:19.244Z 16 320-339 119 119 119 206 206 206 F F F
130962024-03-06T21:03:19.244Z 17 340-359 111 111 111 192 192 192 F F F
130972024-03-06T21:03:19.254Z 18 360-379 121 121 121 209 209 209 F F F
130982024-03-06T21:03:19.254Z 19 380-399 119 119 119 206 206 206 F F F
130992024-03-06T21:03:19.254Z 20 400-419 121 121 121 209 209 209 F F F
131002024-03-06T21:03:19.254Z 21 420-439 117 117 117 204 204 204 F F F
131012024-03-06T21:03:19.254Z 22 440-459 119 119 119 206 206 206 F F F
131022024-03-06T21:03:19.254Z 23 460-479 116 116 116 203 203 203 F F F
131032024-03-06T21:03:19.254Z 24 480-499 119 119 119 206 206 206 F F F
131042024-03-06T21:03:19.254Z 25 500-519 121 121 121 209 209 209 F F F
131052024-03-06T21:03:19.254Z 26 520-539 109 109 109 186 186 186 F F F
131062024-03-06T21:03:19.254Z 27 540-559 119 119 119 206 206 206 F F F
131072024-03-06T21:03:19.254Z 28 560-579 121 121 121 209 209 209 F F F
131082024-03-06T21:03:19.254Z 29 580-599 121 121 121 209 209 209 F F F
131092024-03-06T21:03:19.255ZMax gen: 121, Max flush: 209
131102024-03-06T21:03:19.255Z
131112024-03-06T21:03:19.255ZBegin loop 61
131122024-03-06T21:03:19.255ZDownstairs to restart: 0
131132024-03-06T21:03:19.274Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.016203682Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627}
131142024-03-06T21:03:19.274Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.016491982Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627}
131152024-03-06T21:03:19.275Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.016513481Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627}
131162024-03-06T21:03:19.275Z{"msg":"Crucible 3bd7c0de-f748-4da5-a622-535542830d27 has session id: e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.01653541Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131172024-03-06T21:03:19.717Z{"msg":"Upstairs opts: Upstairs UUID: 3bd7c0de-f748-4da5-a622-535542830d27, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.016576247Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131182024-03-06T21:03:19.717Z{Crucible runtime is spawned
131192024-03-06T21:03:19.717Z"msg":"Crucible stats registered with UUID: 3bd7c0de-f748-4da5-a622-535542830d27","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.016620904Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131202024-03-06T21:03:19.717Z{"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.016684859Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627}
131212024-03-06T21:03:19.717Z{"msg":"3bd7c0de-f748-4da5-a622-535542830d27 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.016945441Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131222024-03-06T21:03:19.718Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.016974239Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"client":"0","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131232024-03-06T21:03:19.718Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.017008217Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"client":"0","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131242024-03-06T21:03:19.718Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.017051034Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"client":"1","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131252024-03-06T21:03:19.718Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.017087792Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"client":"1","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131262024-03-06T21:03:19.718Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.01711315Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"client":"2","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131272024-03-06T21:03:19.718Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.017139808Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"client":"2","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131282024-03-06T21:03:19.718Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.017172516Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"":"io task","client":"2","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131292024-03-06T21:03:19.718Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.017203613Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"":"io task","client":"0","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131302024-03-06T21:03:19.718Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.017229962Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"":"io task","client":"1","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131312024-03-06T21:03:19.719Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.01740022Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"":"io task","client":"0","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131322024-03-06T21:03:19.719Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.017646423Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"":"io task","client":"2","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131332024-03-06T21:03:19.719Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.017699599Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"":"io task","client":"1","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131342024-03-06T21:03:19.719Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.018296918Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"client":"1","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131352024-03-06T21:03:19.719Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.018329416Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"client":"1","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131362024-03-06T21:03:19.719Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.018345465Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"client":"2","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131372024-03-06T21:03:19.719Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.018359674Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"client":"2","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131382024-03-06T21:03:19.719Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.019165818Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"client":"2","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131392024-03-06T21:03:19.719Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.019199506Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"client":"0","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131402024-03-06T21:03:19.719Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.019214535Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"client":"0","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131412024-03-06T21:03:19.720Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.019245993Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"client":"1","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131422024-03-06T21:03:19.720Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.019849281Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"client":"2","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131432024-03-06T21:03:19.720Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.019905177Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"client":"2","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131442024-03-06T21:03:19.720Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.019972293Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131452024-03-06T21:03:19.720Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.020002221Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"client":"1","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131462024-03-06T21:03:19.720Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.020018999Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"client":"1","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131472024-03-06T21:03:19.720Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.020068466Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131482024-03-06T21:03:19.720Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.020083475Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"client":"0","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131492024-03-06T21:03:19.721Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.02044599Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"client":"0","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131502024-03-06T21:03:19.721Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.020480668Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"client":"0","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131512024-03-06T21:03:19.721Z{"msg":"[0]R flush_numbers[0..12]: [204, 206, 206, 209, 209, 209, 196, 206, 193, 209, 209, 209]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.020496637Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"":"downstairs","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131522024-03-06T21:03:19.721Z{"msg":"[0]R generation[0..12]: [117, 119, 119, 121, 121, 121, 112, 119, 112, 121, 121, 121]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.020517955Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"":"downstairs","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131532024-03-06T21:03:19.721Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.020539054Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"":"downstairs","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131542024-03-06T21:03:19.721Z{"msg":"[1]R flush_numbers[0..12]: [204, 206, 206, 209, 209, 209, 196, 206, 193, 209, 209, 209]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.020553713Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"":"downstairs","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131552024-03-06T21:03:19.721Z{"msg":"[1]R generation[0..12]: [117, 119, 119, 121, 121, 121, 112, 119, 112, 121, 121, 121]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.020568662Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"":"downstairs","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131562024-03-06T21:03:19.722Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.02058986Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"":"downstairs","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131572024-03-06T21:03:19.722Z{"msg":"[2]R flush_numbers[0..12]: [204, 206, 206, 209, 209, 209, 196, 206, 193, 209, 209, 209]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.020604859Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"":"downstairs","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131582024-03-06T21:03:19.722Z{"msg":"[2]R generation[0..12]: [117, 119, 119, 121, 121, 121, 112, 119, 112, 121, 121, 121]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.020625228Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"":"downstairs","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131592024-03-06T21:03:19.722Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.020639427Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"":"downstairs","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131602024-03-06T21:03:19.722Z{"msg":"Max found gen is 122","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
131612024-03-06T21:03:19.722Z,"time":"2024-03-06T21:03:18.020659745Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"":"downstairs","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131622024-03-06T21:03:19.722Z{"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.020675504Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"":"downstairs","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131632024-03-06T21:03:19.722Z{"msg":"Next flush: 210","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.020689773Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"":"downstairs","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131642024-03-06T21:03:19.722Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.020709342Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"":"downstairs","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131652024-03-06T21:03:19.723Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.020722891Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131662024-03-06T21:03:19.723Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.02074143Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"client":"0","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131672024-03-06T21:03:19.723Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.020756249Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"client":"1","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131682024-03-06T21:03:19.723Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.020770248Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"client":"2","":"downstairs","session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131692024-03-06T21:03:19.724Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.020783717Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131702024-03-06T21:03:19.724Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.020796076Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131712024-03-06T21:03:19.724Z{"msg":"3bd7c0de-f748-4da5-a622-535542830d27 is now active with session: e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.020815785Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131722024-03-06T21:03:19.724Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.020827964Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627,"session_id":"e9f680b2-47f9-4cc7-b53d-52ed9bc3f8e8"}
131732024-03-06T21:03:19.724Z{"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:18.020840203Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627}
131742024-03-06T21:03:19.724ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
131752024-03-06T21:03:19.725ZLoading write count information from file "/tmp/test_repair_verify.data"
131762024-03-06T21:03:19.725ZRun Repair workload
131772024-03-06T21:03:19.725Z01/30 Write block 296 len 8192 data: 8 7
131782024-03-06T21:03:19.725Z02/30 Write block 410 len 40960 data: 12 10 8 9 7 6 8 8 9 9
131792024-03-06T21:03:19.725Z03/30 Read block 483 len 20480
131802024-03-06T21:03:19.726Z04/30 Flush
131812024-03-06T21:03:19.726Z05/30 Write block 480 len 40960 data: 5 6 6 8 9 11 9 9 9 10
131822024-03-06T21:03:19.726Z06/30 Read block 123 len 12288
131832024-03-06T21:03:20.275Z07/30 Read block 563 len 16384
131842024-03-06T21:03:20.276Z08/30 Write block 133 len 20480 data: 9 9 9 10 8
131852024-03-06T21:03:20.276Z09/30 Write block 2 len 12288 data: 7 8 10
131862024-03-06T21:03:20.276Z10/30 Write block 140 len 4096 data: 7
131872024-03-06T21:03:20.276Z11/30 Read block 568 len 12288
131882024-03-06T21:03:20.276Z12/30 Write block 454 len 32768 data: 10 10 9 12 11 11 10 13
131892024-03-06T21:03:20.276Z13/30 Read block 324 len 24576
131902024-03-06T21:03:20.276Z14/30 Write block 417 len 4096 data: 9
131912024-03-06T21:03:20.276Z15/30 Read block 468 len 16384
131922024-03-06T21:03:20.276Z16/30 Write block 51 len 20480 data: 6 8 8 6 9
131932024-03-06T21:03:20.277Z17/30 Read block 76 len 40960
131942024-03-06T21:03:20.277Z18/30 Write block 245 len 16384 data: 9 8 7 6
131952024-03-06T21:03:20.277Z19/30 Read block 460 len 36864
131962024-03-06T21:03:20.277Z20/30 Read block 23 len 40960
131972024-03-06T21:03:20.277Z21/30 Write block 71 len 24576 data: 8 10 9 9 8 6
131982024-03-06T21:03:20.277Z22/30 Read block 91 len 12288
131992024-03-06T21:03:20.277Z23/30 Write block 505 len 8192 data: 10 10
132002024-03-06T21:03:20.277Z24/30 Write block 91 len 16384 data: 5 5 6 7
132012024-03-06T21:03:20.277Z25/30 Flush
132022024-03-06T21:03:20.277Z26/30 Write block 300 len 20480 data: 7 6 8 8 9
132032024-03-06T21:03:20.277Z27/30 Read block 415 len 16384
132042024-03-06T21:03:20.277Z28/30 Write block 85 len 20480 data: 6 6 4 5 5
132052024-03-06T21:03:20.277Z29/30 Read block 109 len 8192
132062024-03-06T21:03:20.278Z30/30 Read block 334 len 12288
132072024-03-06T21:03:20.278Z----------------------------------------------------------------
132082024-03-06T21:03:20.278Z Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:26
132092024-03-06T21:03:20.278ZStates: Active Active Active
132102024-03-06T21:03:20.278ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
132112024-03-06T21:03:20.278Z 5 Acked 1004 Write 10 Done Done Done false
132122024-03-06T21:03:20.278Z 6 Acked 1005 Read 3 Done Done Done false
132132024-03-06T21:03:20.278Z 7 Acked 1006 Read 4 Done Done Done false
132142024-03-06T21:03:20.278Z 8 Acked 1007 Write 5 Done Done Done false
132152024-03-06T21:03:20.278Z 9 Acked 1008 Write 3 Done Done Done false
132162024-03-06T21:03:20.278Z 10 Acked 1009 Write 1 Done Done Done false
132172024-03-06T21:03:20.279Z 11 Acked 1010 Read 3 Done Done Done false
132182024-03-06T21:03:20.279Z 12 Acked 1011 Write 8 Done Done Done false
132192024-03-06T21:03:20.279Z 13 Acked 1012 Read 6 Done Done Done false
132202024-03-06T21:03:20.279Z 14 Acked 1013 Write 1 Done Done Done false
132212024-03-06T21:03:20.279Z 15 Acked 1014 Read 4 Done Done Done false
132222024-03-06T21:03:20.279Z 16 Acked 1015 Write 5 Done Done Done false
132232024-03-06T21:03:20.279Z 17 Acked 1016 Read 10 Done Done Done false
132242024-03-06T21:03:20.279Z 18 Acked 1017 Write 4 Sent Done Done false
132252024-03-06T21:03:20.279Z 19 Acked 1018 Read 9 Sent Done Done false
132262024-03-06T21:03:20.279Z 20 Acked 1019 Read 10 Sent Done Done false
132272024-03-06T21:03:20.279Z 21 Acked 1020 Write 6 Sent Done Done false
132282024-03-06T21:03:20.279Z 22 Acked 1021 Read 3 Sent Done Done false
132292024-03-06T21:03:20.279Z 23 Acked 1022 Write 2 Sent Done Done false
132302024-03-06T21:03:20.280Z 24 Acked 1023 Write 4 Sent Done Done false
132312024-03-06T21:03:20.280Z 25 Acked 1024 Flush 0 Sent Done Done false
132322024-03-06T21:03:20.280Z 26 Acked 1025 Write 5 Sent Done Done false
132332024-03-06T21:03:20.280Z 27 Acked 1026 Read 4 Sent Done Done false
132342024-03-06T21:03:20.280Z 28 Acked 1027 Write 5 Sent Done Done false
132352024-03-06T21:03:20.280Z 29 Acked 1028 Read 2 Sent Done Done false
132362024-03-06T21:03:20.280Z 30 Acked 1029 Read 3 Sent Done Done false
132372024-03-06T21:03:20.280Z STATES DS:0 DS:1 DS:2 TOTAL
132382024-03-06T21:03:20.280Z New 0 0 0 0
132392024-03-06T21:03:20.281Z Sent 13 0 0 13
132402024-03-06T21:03:20.281Z Done 13 26 26 65
132412024-03-06T21:03:20.281Z Skipped 0 0 0 0
132422024-03-06T21:03:20.281Z Error 0 0 0 0
132432024-03-06T21:03:20.281ZLast Flush: 1003 1024 1024
132442024-03-06T21:03:20.281ZDownstairs last five completed: 1003 1002 1001 1000
132452024-03-06T21:03:20.281ZUpstairs last five completed: 30 29 28 27 26
132462024-03-06T21:03:20.281Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:03:18.059244726Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1627}
132472024-03-06T21:03:20.281ZWrote out file "/tmp/test_repair_verify.data"
132482024-03-06T21:03:20.281Z
132492024-03-06T21:03:20.281ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
132502024-03-06T21:03:20.281ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
132512024-03-06T21:03:20.281Z 0 000-019 117 122 122 204 211 211 T F F <---
132522024-03-06T21:03:20.282Z 1 020-039 119 119 119 206 206 206 F F F
132532024-03-06T21:03:20.282Z 2 040-059 119 122 122 206 211 211 T F F <---
132542024-03-06T21:03:20.282Z 3 060-079 121 122 122 209 211 211 F F F <---
132552024-03-06T21:03:20.282Z 4 080-099 121 122 122 209 211 211 F T T <---
132562024-03-06T21:03:20.282Z 5 100-119 121 121 121 209 209 209 F F F
132572024-03-06T21:03:20.282Z 6 120-139 112 122 122 196 211 211 T F F <---
132582024-03-06T21:03:20.282Z 7 140-159 119 122 122 206 211 211 T F F <---
132592024-03-06T21:03:20.282Z 8 160-179 112 112 112 193 193 193 F F F
132602024-03-06T21:03:20.282Z 9 180-199 121 121 121 209 209 209 F F F
132612024-03-06T21:03:20.282Z 10 200-219 121 121 121 209 209 209 F F F
132622024-03-06T21:03:20.282Z 11 220-239 121 121 121 209 209 209 F F F
132632024-03-06T21:03:20.282Z 12 240-259 121 122 122 209 211 211 F F F <---
132642024-03-06T21:03:20.282Z 13 260-279 121 121 121 209 209 209 F F F
132652024-03-06T21:03:20.282Z 14 280-299 122 122 122 210 210 210 F F F
132662024-03-06T21:03:20.283Z 15 300-319 121 121 121 209 209 209 F T T <---
132672024-03-06T21:03:20.283Z 16 320-339 119 119 119 206 206 206 F F F
132682024-03-06T21:03:20.283Z 17 340-359 111 111 111 192 192 192 F F F
132692024-03-06T21:03:20.283Z 18 360-379 121 121 121 209 209 209 F F F
132702024-03-06T21:03:20.283Z 19 380-399 119 119 119 206 206 206 F F F
132712024-03-06T21:03:20.283Z 20 400-419 122 122 122 210 211 211 T F F <---
132722024-03-06T21:03:20.283Z 21 420-439 117 117 117 204 204 204 F F F
132732024-03-06T21:03:20.283Z 22 440-459 119 122 122 206 211 211 T F F <---
132742024-03-06T21:03:20.283Z 23 460-479 116 122 122 203 211 211 T F F <---
132752024-03-06T21:03:20.283Z 24 480-499 119 122 122 206 211 211 T F F <---
132762024-03-06T21:03:20.283Z 25 500-519 121 122 122 209 211 211 F F F <---
132772024-03-06T21:03:20.283Z 26 520-539 109 109 109 186 186 186 F F F
132782024-03-06T21:03:20.283Z 27 540-559 119 119 119 206 206 206 F F F
132792024-03-06T21:03:20.283Z 28 560-579 121 121 121 209 209 209 F F F
132802024-03-06T21:03:20.283Z 29 580-599 121 121 121 209 209 209 F F F
132812024-03-06T21:03:20.284ZMax gen: 122, Max flush: 211
132822024-03-06T21:03:20.284ZError: Difference in extent metadata found!
132832024-03-06T21:03:20.315ZA Difference in extent metadata is expected here
132842024-03-06T21:03:20.316ZOn loop 61
132852024-03-06T21:03:20.316Z
132862024-03-06T21:03:20.316ZVerifying data now
132872024-03-06T21:03:25.038ZNo change after verify
132882024-03-06T21:03:25.038ZLoop: 61 Downstairs dump after verify (and repair):
132892024-03-06T21:03:25.048ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
132902024-03-06T21:03:25.048Z 0 000-019 122 122 122 211 211 211 F F F
132912024-03-06T21:03:25.048Z 1 020-039 119 119 119 206 206 206 F F F
132922024-03-06T21:03:25.048Z 2 040-059 122 122 122 211 211 211 F F F
132932024-03-06T21:03:25.048Z 3 060-079 122 122 122 211 211 211 F F F
132942024-03-06T21:03:25.049Z 4 080-099 123 123 123 212 212 212 F F F
132952024-03-06T21:03:25.049Z 5 100-119 121 121 121 209 209 209 F F F
132962024-03-06T21:03:25.049Z 6 120-139 122 122 122 211 211 211 F F F
132972024-03-06T21:03:25.049Z 7 140-159 122 122 122 211 211 211 F F F
132982024-03-06T21:03:25.049Z 8 160-179 112 112 112 193 193 193 F F F
132992024-03-06T21:03:25.049Z 9 180-199 121 121 121 209 209 209 F F F
133002024-03-06T21:03:25.049Z 10 200-219 121 121 121 209 209 209 F F F
133012024-03-06T21:03:25.050Z 11 220-239 121 121 121 209 209 209 F F F
133022024-03-06T21:03:25.050Z 12 240-259 122 122 122 211 211 211 F F F
133032024-03-06T21:03:25.050Z 13 260-279 121 121 121 209 209 209 F F F
133042024-03-06T21:03:25.050Z 14 280-299 122 122 122 210 210 210 F F F
133052024-03-06T21:03:25.050Z 15 300-319 123 123 123 212 212 212 F F F
133062024-03-06T21:03:25.051Z 16 320-339 119 119 119 206 206 206 F F F
133072024-03-06T21:03:25.051Z 17 340-359 111 111 111 192 192 192 F F F
133082024-03-06T21:03:25.051Z 18 360-379 121 121 121 209 209 209 F F F
133092024-03-06T21:03:25.051Z 19 380-399 119 119 119 206 206 206 F F F
133102024-03-06T21:03:25.051Z 20 400-419 122 122 122 211 211 211 F F F
133112024-03-06T21:03:25.051Z 21 420-439 117 117 117 204 204 204 F F F
133122024-03-06T21:03:25.051Z 22 440-459 122 122 122 211 211 211 F F F
133132024-03-06T21:03:25.056Z 23 460-479 122 122 122 211 211 211 F F F
133142024-03-06T21:03:25.056Z 24 480-499 122 122 122 211 211 211 F F F
133152024-03-06T21:03:25.056Z 25 500-519 122 122 122 211 211 211 F F F
133162024-03-06T21:03:25.056Z 26 520-539 109 109 109 186 186 186 F F F
133172024-03-06T21:03:25.056Z 27 540-559 119 119 119 206 206 206 F F F
133182024-03-06T21:03:25.056Z 28 560-579 121 121 121 209 209 209 F F F
133192024-03-06T21:03:25.056Z 29 580-599 121 121 121 209 209 209 F F F
133202024-03-06T21:03:25.056ZMax gen: 123, Max flush: 212
133212024-03-06T21:03:25.056Z
133222024-03-06T21:03:25.057ZBegin loop 62
133232024-03-06T21:03:25.057ZDownstairs to restart: 0
133242024-03-06T21:03:25.076Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.824136327Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635}
133252024-03-06T21:03:25.076Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.824426117Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635}
133262024-03-06T21:03:25.076Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.824449356Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":Crucible runtime is spawned
133272024-03-06T21:03:25.076Z1635}
133282024-03-06T21:03:25.085Z{"msg":"Crucible 7e67cc93-74cb-4880-b694-081a286b8403 has session id: e543626d-68ee-4236-8505-ac3ed2b9b126","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.824499642Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133292024-03-06T21:03:25.086Z{"msg":"Upstairs opts: Upstairs UUID: 7e67cc93-74cb-4880-b694-081a286b8403, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.824560498Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133302024-03-06T21:03:25.086Z{"msg":"Crucible stats registered with UUID: 7e67cc93-74cb-4880-b694-081a286b8403","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.824602725Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133312024-03-06T21:03:25.086Z{"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.824633083Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635}
133322024-03-06T21:03:25.086Z{"msg":"7e67cc93-74cb-4880-b694-081a286b8403 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.824746755Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133332024-03-06T21:03:25.086Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.824791642Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"client":"0","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133342024-03-06T21:03:25.086Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.82482311Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"client":"0","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133352024-03-06T21:03:25.087Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.824874066Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"client":"1","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133362024-03-06T21:03:25.087Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.824906994Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"client":"1","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133372024-03-06T21:03:25.087Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.824947871Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"client":"2","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133382024-03-06T21:03:25.087Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.825093571Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"client":"2","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133392024-03-06T21:03:25.087Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.825156007Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"":"io task","client":"2","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133402024-03-06T21:03:25.088Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.825203434Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"":"io task","client":"0","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133412024-03-06T21:03:25.088Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.825246281Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"":"io task","client":"1","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133422024-03-06T21:03:25.088Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.825275439Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"":"io task","client":"1","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133432024-03-06T21:03:25.088Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.825303047Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"":"io task","client":"0","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133442024-03-06T21:03:25.088Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.825352933Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"":"io task","client":"2","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133452024-03-06T21:03:25.088Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.826037546Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"client":"2","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133462024-03-06T21:03:25.088Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.826079033Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"client":"2","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133472024-03-06T21:03:25.088Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.826102222Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"client":"1","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133482024-03-06T21:03:25.088Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.82612068Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"client":"1","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133492024-03-06T21:03:25.111Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.826743557Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"client":"1","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133502024-03-06T21:03:25.112Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.826781565Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"client":"2","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133512024-03-06T21:03:25.112Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.827080334Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"client":"0","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133522024-03-06T21:03:25.112Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.827121831Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"client":"0","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133532024-03-06T21:03:25.112Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.827354135Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"client":"1","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133542024-03-06T21:03:25.112Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.827402652Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"client":"1","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133552024-03-06T21:03:25.112Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.827443179Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133562024-03-06T21:03:25.113Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.827471827Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"client":"2","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133572024-03-06T21:03:25.113Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.827493596Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"client":"2","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133582024-03-06T21:03:25.113Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.827513915Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133592024-03-06T21:03:25.113Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.827695352Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"client":"0","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133602024-03-06T21:03:25.113Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.828109793Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"client":"0","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133612024-03-06T21:03:25.113Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.828150611Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"client":"0","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133622024-03-06T21:03:25.113Z{"msg":"[0]R flush_numbers[0..12]: [211, 206, 211, 211, 212, 209, 211, 211, 193, 209, 209, 209]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.828193138Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"":"downstairs","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133632024-03-06T21:03:25.113Z{"msg":"[0]R generation[0..12]: [122, 119, 122, 122, 123, 121, 122, 122, 112, 121, 121, 121]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.828218756Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"":"downstairs","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133642024-03-06T21:03:25.114Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.828240164Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"":"downstairs","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133652024-03-06T21:03:25.114Z{"msg":"[1]R flush_numbers[0..12]: [211, 206, 211, 211, 212, 209, 211, 211, 193, 209, 209, 209]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.828260793Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"":"downstairs","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133662024-03-06T21:03:25.114Z{"msg":"[1]R generation[0..12]: [122, 119, 122, 122, 123, 121, 122, 122, 112, 121, 121, 121]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.828281442Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"":"downstairs","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133672024-03-06T21:03:25.114ZWait for a query_work_queue command to finish before sending IO
133682024-03-06T21:03:25.114Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.82830676Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"":"downstairs","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133692024-03-06T21:03:25.114Z{"msg":"[2]R flush_numbers[0..12]: [211, 206, 211, 211, 212, 209, 211, 211, 193, 209, 209, 209]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.828327888Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"":"downstairs","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133702024-03-06T21:03:25.114Z{"msg":"[2]R generation[0..12]: [122, 119, 122, 122, 123, 121, 122, 122, 112, 121, 121, 121]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.828348387Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"":"downstairs","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133712024-03-06T21:03:25.115Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.828369136Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"":"downstairs","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133722024-03-06T21:03:25.115Z{"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.828396074Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"":"downstairs","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133732024-03-06T21:03:25.115Z{"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.828430821Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"":"downstairs","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133742024-03-06T21:03:25.115Z{"msg":"Next flush: 213","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.828463739Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"":"downstairs","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133752024-03-06T21:03:25.115Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.828497587Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"":"downstairs","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133762024-03-06T21:03:25.115Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.828531144Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133772024-03-06T21:03:25.115Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
133782024-03-06T21:03:25.115Z,"time":"2024-03-06T21:03:23.828561172Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"client":"0","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133792024-03-06T21:03:25.115Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.82859713Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"client":"1","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133802024-03-06T21:03:25.116Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.828623708Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"client":"2","":"downstairs","session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133812024-03-06T21:03:25.116Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.828644197Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133822024-03-06T21:03:25.116Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.828662845Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133832024-03-06T21:03:25.116Z{"msg":"7e67cc93-74cb-4880-b694-081a286b8403 is now active with session: e543626d-68ee-4236-8505-ac3ed2b9b126","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.828682054Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133842024-03-06T21:03:25.116Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.828700463Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635,"session_id":"e543626d-68ee-4236-8505-ac3ed2b9b126"}
133852024-03-06T21:03:25.116Z{"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:23.828712942Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635}
133862024-03-06T21:03:25.116ZLoading write count information from file "/tmp/test_repair_verify.data"
133872024-03-06T21:03:25.116ZRun Repair workload
133882024-03-06T21:03:25.117Z01/30 Flush
133892024-03-06T21:03:25.117Z02/30 Write block 595 len 8192 data: 9 6
133902024-03-06T21:03:25.117Z03/30 Write block 165 len 12288 data: 7 6 7
133912024-03-06T21:03:25.117Z04/30 Write block 334 len 20480 data: 6 5 7 8 11
133922024-03-06T21:03:25.117Z05/30 Flush
133932024-03-06T21:03:25.117Z06/30 Write block 370 len 24576 data: 12 13 12 13 12 14
133942024-03-06T21:03:25.117Z07/30 Read block 533 len 8192
133952024-03-06T21:03:25.117Z08/30 Write block 521 len 16384 data: 4 4 5 7
133962024-03-06T21:03:25.118Z09/30 Read block 411 len 16384
133972024-03-06T21:03:25.118Z10/30 Write block 562 len 4096 data: 9
133982024-03-06T21:03:25.118Z11/30 Write block 27 len 24576 data: 10 8 9 9 8 9
133992024-03-06T21:03:25.118Z12/30 Write block 47 len 32768 data: 9 8 7 5 7 9 9 7
134002024-03-06T21:03:25.118Z13/30 Read block 226 len 16384
134012024-03-06T21:03:25.136Z14/30 Read block 83 len 32768
134022024-03-06T21:03:25.136Z15/30 Write block 357 len 24576 data: 12 12 10 12 16 12
134032024-03-06T21:03:25.136Z16/30 Read block 321 len 8192
134042024-03-06T21:03:25.136Z17/30 Read block 460 len 12288
134052024-03-06T21:03:25.137Z18/30 Flush
134062024-03-06T21:03:25.137Z19/30 Write block 242 len 32768 data: 12 10 10 10 9 8 7 6
134072024-03-06T21:03:25.137Z20/30 Read block 172 len 28672
134082024-03-06T21:03:25.137Z21/30 Read block 364 len 8192
134092024-03-06T21:03:25.137Z22/30 Read block 171 len 24576
134102024-03-06T21:03:25.137Z23/30 Write block 576 len 24576 data: 10 7 5 6 6 8
134112024-03-06T21:03:25.137Z24/30 Write block 474 len 40960 data: 5 5 6 6 8 4 6 7 7 9
134122024-03-06T21:03:25.138Z25/30 Write block 9 len 28672 data: 7 7 10 9 10 9 11
134132024-03-06T21:03:25.138Z26/30 Flush
134142024-03-06T21:03:25.138Z27/30 Write block 17 len 20480 data: 11 12 14 15 12
134152024-03-06T21:03:25.138Z28/30 Write block 233 len 32768 data: 8 10 12 13 15 17 17 12
134162024-03-06T21:03:25.139Z29/30 Write block 304 len 20480 data: 10 11 13 13 13
134172024-03-06T21:03:25.139Z30/30 Read block 543 len 24576
134182024-03-06T21:03:25.139Z----------------------------------------------------------------
134192024-03-06T21:03:25.139Z Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:29
134202024-03-06T21:03:25.139ZStates: Active Active Active
134212024-03-06T21:03:25.139ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
134222024-03-06T21:03:25.139Z 2 Acked 1001 Write 2 Sent Done Done false
134232024-03-06T21:03:25.140Z 3 Acked 1002 Write 3 Sent Done Done false
134242024-03-06T21:03:25.140Z 4 Acked 1003 Write 5 Sent Done Done false
134252024-03-06T21:03:25.140Z 5 Acked 1004 Flush 0 Sent Done Done false
134262024-03-06T21:03:25.140Z 6 Acked 1005 Write 6 Sent Done Done false
134272024-03-06T21:03:25.140Z 7 Acked 1006 Read 2 Sent Done Done false
134282024-03-06T21:03:25.141Z 8 Acked 1007 Write 4 Sent Done Done false
134292024-03-06T21:03:25.141Z 9 Acked 1008 Read 4 Sent Done Done false
134302024-03-06T21:03:25.141Z 10 Acked 1009 Write 1 Sent Done Done false
134312024-03-06T21:03:25.141Z 11 Acked 1010 Write 6 Sent Done Done false
134322024-03-06T21:03:25.142Z 12 Acked 1011 Write 8 Sent Done Done false
134332024-03-06T21:03:25.142Z 13 Acked 1012 Read 4 Sent Done Done false
134342024-03-06T21:03:25.142Z 14 Acked 1013 Read 8 Sent Done Done false
134352024-03-06T21:03:25.142Z 15 Acked 1014 Write 6 Sent Done Done false
134362024-03-06T21:03:25.142Z 16 Acked 1015 Read 2 Sent Done Done false
134372024-03-06T21:03:25.142Z 17 Acked 1016 Read 3 Sent Done Done false
134382024-03-06T21:03:25.143Z 18 Acked 1017 Flush 0 Sent Done Done false
134392024-03-06T21:03:25.143Z 19 Acked 1018 Write 8 Sent Done Done false
134402024-03-06T21:03:25.143Z 20 Acked 1019 Read 7 Sent Done Done false
134412024-03-06T21:03:25.143Z 21 Acked 1020 Read 2 Sent Done Done false
134422024-03-06T21:03:25.143Z 22 Acked 1021 Read 6 Sent Done Done false
134432024-03-06T21:03:25.143Z 23 Acked 1022 Write 6 Sent Done Done false
134442024-03-06T21:03:25.144Z 24 Acked 1023 Write 10 Sent Done Done false
134452024-03-06T21:03:25.144Z 25 Acked 1024 Write 7 Sent Done Done false
134462024-03-06T21:03:25.144Z 26 Acked 1025 Flush 0 Sent Done Done false
134472024-03-06T21:03:25.144Z 27 Acked 1026 Write 5 Sent Done Done false
134482024-03-06T21:03:25.144Z 28 Acked 1027 Write 8 Sent Done Done false
134492024-03-06T21:03:25.144Z 29 Acked 1028 Write 5 Sent Done Done false
134502024-03-06T21:03:25.144Z 30 Acked 1029 Read 6 Sent Done Sent false
134512024-03-06T21:03:25.145Z STATES DS:0 DS:1 DS:2 TOTAL
134522024-03-06T21:03:25.145Z New 0 0 0 0
134532024-03-06T21:03:25.145Z Sent 29 0 1 30
134542024-03-06T21:03:25.145Z Done 0 29 28 57
134552024-03-06T21:03:25.146Z Skipped 0 0 0 0
134562024-03-06T21:03:25.146Z Error 0 0 0 0
134572024-03-06T21:03:25.146ZLast Flush: 1000 1025 1025
134582024-03-06T21:03:25.146ZDownstairs last five completed: 1000
134592024-03-06T21:03:25.146ZUpstairs last five completed: 30 29 28 27 26
134602024-03-06T21:03:25.146Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:03:23.864689862Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1635}
134612024-03-06T21:03:25.147ZWrote out file "/tmp/test_repair_verify.data"
134622024-03-06T21:03:25.147Z
134632024-03-06T21:03:25.155ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
134642024-03-06T21:03:25.155ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
134652024-03-06T21:03:25.155Z 0 000-019 122 124 124 211 216 216 F T T <---
134662024-03-06T21:03:25.155Z 1 020-039 119 124 124 206 215 215 F T T <---
134672024-03-06T21:03:25.155Z 2 040-059 122 124 124 211 215 215 F F F <---
134682024-03-06T21:03:25.155Z 3 060-079 122 122 122 211 211 211 F F F
134692024-03-06T21:03:25.155Z 4 080-099 123 123 123 212 212 212 F F F
134702024-03-06T21:03:25.155Z 5 100-119 121 121 121 209 209 209 F F F
134712024-03-06T21:03:25.156Z 6 120-139 122 122 122 211 211 211 F F F
134722024-03-06T21:03:25.156Z 7 140-159 122 122 122 211 211 211 F F F
134732024-03-06T21:03:25.156Z 8 160-179 112 124 124 193 214 214 F F F <---
134742024-03-06T21:03:25.156Z 9 180-199 121 121 121 209 209 209 F F F
134752024-03-06T21:03:25.156Z 10 200-219 121 121 121 209 209 209 F F F
134762024-03-06T21:03:25.156Z 11 220-239 121 121 121 209 209 209 F T T <---
134772024-03-06T21:03:25.156Z 12 240-259 122 124 124 211 216 216 F T T <---
134782024-03-06T21:03:25.157Z 13 260-279 121 121 121 209 209 209 F F F
134792024-03-06T21:03:25.157Z 14 280-299 122 122 122 210 210 210 F F F
134802024-03-06T21:03:25.157Z 15 300-319 123 123 123 212 212 212 F T T <---
134812024-03-06T21:03:25.157Z 16 320-339 119 124 124 206 214 214 F F F <---
134822024-03-06T21:03:25.157Z 17 340-359 111 124 124 192 215 215 F F F <---
134832024-03-06T21:03:25.157Z 18 360-379 121 124 124 209 215 215 F F F <---
134842024-03-06T21:03:25.158Z 19 380-399 119 119 119 206 206 206 F F F
134852024-03-06T21:03:25.158Z 20 400-419 122 122 122 211 211 211 F F F
134862024-03-06T21:03:25.158Z 21 420-439 117 117 117 204 204 204 F F F
134872024-03-06T21:03:25.158Z 22 440-459 122 122 122 211 211 211 F F F
134882024-03-06T21:03:25.158Z 23 460-479 122 124 124 211 216 216 F F F <---
134892024-03-06T21:03:25.158Z 24 480-499 122 124 124 211 216 216 F F F <---
134902024-03-06T21:03:25.158Z 25 500-519 122 122 122 211 211 211 F F F
134912024-03-06T21:03:25.158Z 26 520-539 109 124 124 186 215 215 F F F <---
134922024-03-06T21:03:25.158Z 27 540-559 119 119 119 206 206 206 F F F
134932024-03-06T21:03:25.159Z 28 560-579 121 124 124 209 216 216 F F F <---
134942024-03-06T21:03:25.159Z 29 580-599 121 124 124 209 216 216 F F F <---
134952024-03-06T21:03:25.159ZMax gen: 124, Max flush: 216
134962024-03-06T21:03:25.159ZError: Difference in extent metadata found!
134972024-03-06T21:03:25.159ZA Difference in extent metadata is expected here
134982024-03-06T21:03:25.159ZOn loop 62
134992024-03-06T21:03:25.160Z
135002024-03-06T21:03:25.160ZVerifying data now
135012024-03-06T21:03:29.939ZNo change after verify
135022024-03-06T21:03:29.939ZLoop: 62 Downstairs dump after verify (and repair):
135032024-03-06T21:03:29.949ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
135042024-03-06T21:03:29.949Z 0 000-019 125 125 125 217 217 217 F F F
135052024-03-06T21:03:29.949Z 1 020-039 125 125 125 217 217 217 F F F
135062024-03-06T21:03:29.949Z 2 040-059 124 124 124 215 215 215 F F F
135072024-03-06T21:03:29.949Z 3 060-079 122 122 122 211 211 211 F F F
135082024-03-06T21:03:29.949Z 4 080-099 123 123 123 212 212 212 F F F
135092024-03-06T21:03:29.949Z 5 100-119 121 121 121 209 209 209 F F F
135102024-03-06T21:03:29.949Z 6 120-139 122 122 122 211 211 211 F F F
135112024-03-06T21:03:29.949Z 7 140-159 122 122 122 211 211 211 F F F
135122024-03-06T21:03:29.949Z 8 160-179 124 124 124 214 214 214 F F F
135132024-03-06T21:03:29.949Z 9 180-199 121 121 121 209 209 209 F F F
135142024-03-06T21:03:29.950Z 10 200-219 121 121 121 209 209 209 F F F
135152024-03-06T21:03:29.950Z 11 220-239 125 125 125 217 217 217 F F F
135162024-03-06T21:03:29.950Z 12 240-259 125 125 125 217 217 217 F F F
135172024-03-06T21:03:29.950Z 13 260-279 121 121 121 209 209 209 F F F
135182024-03-06T21:03:29.950Z 14 280-299 122 122 122 210 210 210 F F F
135192024-03-06T21:03:29.950Z 15 300-319 125 125 125 217 217 217 F F F
135202024-03-06T21:03:29.950Z 16 320-339 124 124 124 214 214 214 F F F
135212024-03-06T21:03:29.950Z 17 340-359 124 124 124 215 215 215 F F F
135222024-03-06T21:03:29.950Z 18 360-379 124 124 124 215 215 215 F F F
135232024-03-06T21:03:29.950Z 19 380-399 119 119 119 206 206 206 F F F
135242024-03-06T21:03:29.950Z 20 400-419 122 122 122 211 211 211 F F F
135252024-03-06T21:03:29.950Z 21 420-439 117 117 117 204 204 204 F F F
135262024-03-06T21:03:29.950Z 22 440-459 122 122 122 211 211 211 F F F
135272024-03-06T21:03:29.951Z 23 460-479 124 124 124 216 216 216 F F F
135282024-03-06T21:03:29.951Z 24 480-499 124 124 124 216 216 216 F F F
135292024-03-06T21:03:29.951Z 25 500-519 122 122 122 211 211 211 F F F
135302024-03-06T21:03:29.951Z 26 520-539 124 124 124 215 215 215 F F F
135312024-03-06T21:03:29.951Z 27 540-559 119 119 119 206 206 206 F F F
135322024-03-06T21:03:29.951Z 28 560-579 124 124 124 216 216 216 F F F
135332024-03-06T21:03:29.951Z 29 580-599 124 124 124 216 216 216 F F F
135342024-03-06T21:03:29.951ZMax gen: 125, Max flush: 217
135352024-03-06T21:03:29.955Z
135362024-03-06T21:03:29.955ZBegin loop 63
135372024-03-06T21:03:29.955ZDownstairs to restart: 0
135382024-03-06T21:03:29.979Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.720052995Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643}
135392024-03-06T21:03:29.979Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.720341185Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643}
135402024-03-06T21:03:29.979Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.720374453Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643}
135412024-03-06T21:03:30.007Z{"msg":"Crucible f2399c6c-7511-4b82-8ed6-e095988ff630 has session id: 2d56a69b-9ed3-4083-be06-8eb1a173030b","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.720405181Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643Crucible runtime is spawned
135422024-03-06T21:03:30.007Z,"session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135432024-03-06T21:03:30.007Z{"msg":"Upstairs opts: Upstairs UUID: f2399c6c-7511-4b82-8ed6-e095988ff630, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.720509864Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135442024-03-06T21:03:30.007Z{"msg":"Crucible stats registered with UUID: f2399c6c-7511-4b82-8ed6-e095988ff630","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.720553121Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135452024-03-06T21:03:30.007Z{"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.720592698Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643}
135462024-03-06T21:03:30.007Z{"msg":"f2399c6c-7511-4b82-8ed6-e095988ff630 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.720636675Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135472024-03-06T21:03:30.007Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.720669073Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"client":"0","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135482024-03-06T21:03:30.007Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.720700231Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"client":"0","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135492024-03-06T21:03:30.007Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.720738858Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"client":"1","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135502024-03-06T21:03:30.007Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.720771175Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"client":"1","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135512024-03-06T21:03:30.007Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.720810733Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"client":"2","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135522024-03-06T21:03:30.007Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.720841751Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"client":"2","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135532024-03-06T21:03:30.007Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.720868419Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"":"io task","client":"2","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135542024-03-06T21:03:30.007Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.72098974Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"":"io task","client":"0","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135552024-03-06T21:03:30.008Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.721026848Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"":"io task","client":"1","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135562024-03-06T21:03:30.008Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.721054316Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"":"io task","client":"2","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135572024-03-06T21:03:30.008Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.721081754Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"":"io task","client":"1","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135582024-03-06T21:03:30.008Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.721107092Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"":"io task","client":"0","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135592024-03-06T21:03:30.008Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.72186569Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"client":"2","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135602024-03-06T21:03:30.008Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.721906497Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"client":"2","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135612024-03-06T21:03:30.008Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.721924546Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"client":"1","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135622024-03-06T21:03:30.008Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.721938555Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"client":"1","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135632024-03-06T21:03:30.008Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.722604439Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"client":"1","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135642024-03-06T21:03:30.008Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.722658825Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"client":"2","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135652024-03-06T21:03:30.008Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.723182649Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"client":"2","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135662024-03-06T21:03:30.008Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.723224617Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"client":"2","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135672024-03-06T21:03:30.008Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.723241565Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135682024-03-06T21:03:30.008Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.723263084Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"client":"1","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135692024-03-06T21:03:30.008Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.723277733Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"client":"1","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135702024-03-06T21:03:30.008Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.723292592Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135712024-03-06T21:03:30.008Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.723472969Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"client":"0","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135722024-03-06T21:03:30.009Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.723499798Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"client":"0","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135732024-03-06T21:03:30.009Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.723975765Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"client":"0","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135742024-03-06T21:03:30.009Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.724385086Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"client":"0","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135752024-03-06T21:03:30.009Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.724418904Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"client":"0","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135762024-03-06T21:03:30.009Z{"msg":"[0]R flush_numbers[0..12]: [217, 217, 215, 211, 212, 209, 211, 211, 214, 209, 209, 217]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.724434313Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"":"downstairs","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135772024-03-06T21:03:30.009Z{"msg":"[0]R generation[0..12]: [125, 125, 124, 122, 123, 121, 122, 122, 124, 121, 121, 125]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.724448792Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"":"downstairs","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135782024-03-06T21:03:30.009Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.724470111Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"":"downstairs","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135792024-03-06T21:03:30.009Z{"msg":"[1]R flush_numbers[0..12]: [217, 217, 215, 211, 212, 209, 211, 211, 214, 209, 209, 217]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.72448471Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"":"downstairs","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135802024-03-06T21:03:30.009Z{"msg":"[1]R generation[0..12]: [125, 125, 124, 122, 123, 121, 122, 122, 124, 121, 121, 125]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.724505178Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"":"downstairs","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135812024-03-06T21:03:30.009Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.724519707Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"":"downstairs","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135822024-03-06T21:03:30.009Z{"msg":"[2]R flush_numbers[0..12]: [217, 217, 215, 211, 212, 209, 211, 211, 214, 209, 209, 217]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.724546555Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"":"downstairs","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135832024-03-06T21:03:30.009Z{"msg":"[2]R generation[0..12]: [125, 125, 124, 122, 123, 121, 122, 122, 124, 121, 121, 125]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.724560614Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"":"downstairs","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135842024-03-06T21:03:30.009ZWait for a query_work_queue command to finish before sending IO
135852024-03-06T21:03:30.009Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.724587692Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"":"downstairs","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135862024-03-06T21:03:30.009Z{"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.724601082Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"":"downstairs","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135872024-03-06T21:03:30.009Z{"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.72462119Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"":"downstairs","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135882024-03-06T21:03:30.009Z{"msg":"Next flush: 218","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.724634769Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"":"downstairs","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135892024-03-06T21:03:30.009Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.724647748Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"":"downstairs","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135902024-03-06T21:03:30.010Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.724667447Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135912024-03-06T21:03:30.010Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.724679566Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"client":"0","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135922024-03-06T21:03:30.010Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.724699145Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"client":"1","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135932024-03-06T21:03:30.010Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.724712544Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"client":"2","":"downstairs","session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135942024-03-06T21:03:30.010Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.724726773Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135952024-03-06T21:03:30.010Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.724739152Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135962024-03-06T21:03:30.010Z{"msg":"f2399c6c-7511-4b82-8ed6-e095988ff630 is now active with session: 2d56a69b-9ed3-4083-be06-8eb1a173030b","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.724758631Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135972024-03-06T21:03:30.010Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.72477104Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643,"session_id":"2d56a69b-9ed3-4083-be06-8eb1a173030b"}
135982024-03-06T21:03:30.010Z{"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:28.724789739Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643}
135992024-03-06T21:03:30.010ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
136002024-03-06T21:03:30.010ZLoading write count information from file "/tmp/test_repair_verify.data"
136012024-03-06T21:03:30.010ZRun Repair workload
136022024-03-06T21:03:30.010Z01/30 Write block 86 len 28672 data: 7 5 6 6 3 6 6
136032024-03-06T21:03:30.010Z02/30 Flush
136042024-03-06T21:03:30.010Z03/30 Flush
136052024-03-06T21:03:30.010Z04/30 Write block 480 len 24576 data: 7 8 8 10 10 12
136062024-03-06T21:03:30.010Z05/30 Read block 557 len 4096
136072024-03-06T21:03:30.010Z06/30 Read block 457 len 40960
136082024-03-06T21:03:30.010Z07/30 Read block 172 len 20480
136092024-03-06T21:03:30.016Z08/30 Read block 578 len 32768
136102024-03-06T21:03:30.016Z09/30 Flush
136112024-03-06T21:03:30.016Z10/30 Write block 203 len 12288 data: 7 7 7
136122024-03-06T21:03:30.016Z11/30 Read block 17 len 20480
136132024-03-06T21:03:30.016Z12/30 Flush
136142024-03-06T21:03:30.016Z13/30 Write block 518 len 36864 data: 9 8 5 5 5 6 8 7 7
136152024-03-06T21:03:30.016Z14/30 Write block 525 len 12288 data: 8 8 8
136162024-03-06T21:03:30.016Z15/30 Write block 86 len 8192 data: 8 6
136172024-03-06T21:03:30.016Z16/30 Read block 422 len 20480
136182024-03-06T21:03:30.016Z17/30 Flush
136192024-03-06T21:03:30.016Z18/30 Write block 506 len 40960 data: 11 11 10 11 12 12 11 10 10 13
136202024-03-06T21:03:30.016Z19/30 Read block 510 len 36864
136212024-03-06T21:03:30.017Z20/30 Read block 545 len 40960
136222024-03-06T21:03:30.017Z21/30 Write block 459 len 24576 data: 12 11 14 12 14 14
136232024-03-06T21:03:30.017Z22/30 Write block 79 len 12288 data: 9 9 9
136242024-03-06T21:03:30.017Z23/30 Write block 339 len 8192 data: 9 10
136252024-03-06T21:03:30.017Z24/30 Write block 180 len 32768 data: 12 14 13 12 12 11 9 8
136262024-03-06T21:03:30.017Z25/30 Write block 127 len 20480 data: 12 12 12 9 7
136272024-03-06T21:03:30.017Z26/30 Read block 435 len 12288
136282024-03-06T21:03:30.017Z27/30 Read block 228 len 40960
136292024-03-06T21:03:30.017Z28/30 Write block 413 len 24576 data: 10 8 7 9 10 10
136302024-03-06T21:03:30.017Z29/30 Write block 61 len 24576 data: 17 16 19 16 15 14
136312024-03-06T21:03:30.017Z30/30 Write block 340 len 20480 data: 11 11 10 11 12
136322024-03-06T21:03:30.017Z----------------------------------------------------------------
136332024-03-06T21:03:30.017Z Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:28
136342024-03-06T21:03:30.017ZStates: Active Active Active
136352024-03-06T21:03:30.017ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
136362024-03-06T21:03:30.017Z 3 Acked 1002 Flush 0 Sent Done Done false
136372024-03-06T21:03:30.017Z 4 Acked 1003 Write 6 Sent Done Done false
136382024-03-06T21:03:30.017Z 5 Acked 1004 Read 1 Sent Done Done false
136392024-03-06T21:03:30.017Z 6 Acked 1005 Read 10 Sent Done Done false
136402024-03-06T21:03:30.017Z 7 Acked 1006 Read 5 Sent Done Done false
136412024-03-06T21:03:30.017Z 8 Acked 1007 Read 8 Sent Done Done false
136422024-03-06T21:03:30.017Z 9 Acked 1008 Flush 0 Sent Done Done false
136432024-03-06T21:03:30.018Z 10 Acked 1009 Write 3 Sent Done Done false
136442024-03-06T21:03:30.018Z 11 Acked 1010 Read 5 Sent Done Done false
136452024-03-06T21:03:30.018Z 12 Acked 1011 Flush 0 Sent Done Done false
136462024-03-06T21:03:30.018Z 13 Acked 1012 Write 9 Sent Done Done false
136472024-03-06T21:03:30.018Z 14 Acked 1013 Write 3 Sent Done Done false
136482024-03-06T21:03:30.018Z 15 Acked 1014 Write 2 Sent Done Done false
136492024-03-06T21:03:30.018Z 16 Acked 1015 Read 5 Sent Done Done false
136502024-03-06T21:03:30.018Z 17 Acked 1016 Flush 0 Sent Done Done false
136512024-03-06T21:03:30.018Z 18 Acked 1017 Write 10 Sent Done Done false
136522024-03-06T21:03:30.018Z 19 Acked 1018 Read 9 Sent Done Done false
136532024-03-06T21:03:30.018Z 20 Acked 1019 Read 10 Sent Done Done false
136542024-03-06T21:03:30.018Z 21 Acked 1020 Write 6 Sent Done Done false
136552024-03-06T21:03:30.018Z 22 Acked 1021 Write 3 Sent Done Done false
136562024-03-06T21:03:30.018Z 23 Acked 1022 Write 2 Sent Done Done false
136572024-03-06T21:03:30.018Z 24 Acked 1023 Write 8 Sent Done Done false
136582024-03-06T21:03:30.018Z 25 Acked 1024 Write 5 Sent Done Done false
136592024-03-06T21:03:30.018Z 26 Acked 1025 Read 3 Sent Done Done false
136602024-03-06T21:03:30.018Z 27 Acked 1026 Read 10 Sent Done Done false
136612024-03-06T21:03:30.018Z 28 Acked 1027 Write 6 Sent Done Done false
136622024-03-06T21:03:30.018Z 29 Acked 1028 Write 6 Sent Sent Sent false
136632024-03-06T21:03:30.018Z 30 Acked 1029 Write 5 Sent Sent Sent false
136642024-03-06T21:03:30.018Z STATES DS:0 DS:1 DS:2 TOTAL
136652024-03-06T21:03:30.018Z New 0 0 0 0
136662024-03-06T21:03:30.018Z Sent 28 2 2 32
136672024-03-06T21:03:30.019Z Done 0 26 26 52
136682024-03-06T21:03:30.019Z Skipped 0 0 0 0
136692024-03-06T21:03:30.019Z Error 0 0 0 0
136702024-03-06T21:03:30.019ZLast Flush: 1001 1016 1016
136712024-03-06T21:03:30.019ZDownstairs last five completed: 1001 1000
136722024-03-06T21:03:30.019ZUpstairs last five completed: 30 29 28 27 26
136732024-03-06T21:03:30.019Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:03:28.762850966Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1643}
136742024-03-06T21:03:30.019ZWrote out file "/tmp/test_repair_verify.data"
136752024-03-06T21:03:30.022Z
136762024-03-06T21:03:30.025ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
136772024-03-06T21:03:30.042ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
136782024-03-06T21:03:30.042Z 0 000-019 125 125 125 217 217 217 F F F
136792024-03-06T21:03:30.042Z 1 020-039 125 125 125 217 217 217 F F F
136802024-03-06T21:03:30.043Z 2 040-059 124 124 124 215 215 215 F F F
136812024-03-06T21:03:30.043Z 3 060-079 122 122 122 211 211 211 F T T <---
136822024-03-06T21:03:30.043Z 4 080-099 126 126 126 218 222 222 F T T <---
136832024-03-06T21:03:30.043Z 5 100-119 121 121 121 209 209 209 F F F
136842024-03-06T21:03:30.043Z 6 120-139 122 122 122 211 211 211 F T T <---
136852024-03-06T21:03:30.043Z 7 140-159 122 122 122 211 211 211 F F F
136862024-03-06T21:03:30.043Z 8 160-179 124 124 124 214 214 214 F F F
136872024-03-06T21:03:30.043Z 9 180-199 121 121 121 209 209 209 F T T <---
136882024-03-06T21:03:30.043Z 10 200-219 121 126 126 209 221 221 F F F <---
136892024-03-06T21:03:30.043Z 11 220-239 125 125 125 217 217 217 F F F
136902024-03-06T21:03:30.043Z 12 240-259 125 125 125 217 217 217 F F F
136912024-03-06T21:03:30.043Z 13 260-279 121 121 121 209 209 209 F F F
136922024-03-06T21:03:30.043Z 14 280-299 122 122 122 210 210 210 F F F
136932024-03-06T21:03:30.043Z 15 300-319 125 125 125 217 217 217 F F F
136942024-03-06T21:03:30.043Z 16 320-339 124 124 124 214 214 214 F T T <---
136952024-03-06T21:03:30.043Z 17 340-359 124 124 124 215 215 215 F T T <---
136962024-03-06T21:03:30.047Z 18 360-379 124 124 124 215 215 215 F F F
136972024-03-06T21:03:30.047Z 19 380-399 119 119 119 206 206 206 F F F
136982024-03-06T21:03:30.047Z 20 400-419 122 122 122 211 211 211 F T T <---
136992024-03-06T21:03:30.047Z 21 420-439 117 117 117 204 204 204 F F F
137002024-03-06T21:03:30.047Z 22 440-459 122 122 122 211 211 211 F T T <---
137012024-03-06T21:03:30.047Z 23 460-479 124 124 124 216 216 216 F T T <---
137022024-03-06T21:03:30.047Z 24 480-499 124 126 126 216 220 220 F F F <---
137032024-03-06T21:03:30.047Z 25 500-519 122 126 126 211 222 222 F T T <---
137042024-03-06T21:03:30.047Z 26 520-539 124 126 126 215 222 222 F F F <---
137052024-03-06T21:03:30.047Z 27 540-559 119 119 119 206 206 206 F F F
137062024-03-06T21:03:30.047Z 28 560-579 124 124 124 216 216 216 F F F
137072024-03-06T21:03:30.047Z 29 580-599 124 124 124 216 216 216 F F F
137082024-03-06T21:03:30.047ZMax gen: 126, Max flush: 222
137092024-03-06T21:03:30.047ZError: Difference in extent metadata found!
137102024-03-06T21:03:30.047ZA Difference in extent metadata is expected here
137112024-03-06T21:03:30.047ZOn loop 63
137122024-03-06T21:03:30.047Z
137132024-03-06T21:03:30.051ZVerifying data now
137142024-03-06T21:03:34.762ZNo change after verify
137152024-03-06T21:03:34.762ZLoop: 63 Downstairs dump after verify (and repair):
137162024-03-06T21:03:34.777ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
137172024-03-06T21:03:34.777Z 0 000-019 125 125 125 217 217 217 F F F
137182024-03-06T21:03:34.777Z 1 020-039 125 125 125 217 217 217 F F F
137192024-03-06T21:03:34.777Z 2 040-059 124 124 124 215 215 215 F F F
137202024-03-06T21:03:34.777Z 3 060-079 127 127 127 223 223 223 F F F
137212024-03-06T21:03:34.777Z 4 080-099 127 127 127 223 223 223 F F F
137222024-03-06T21:03:34.777Z 5 100-119 121 121 121 209 209 209 F F F
137232024-03-06T21:03:34.777Z 6 120-139 127 127 127 223 223 223 F F F
137242024-03-06T21:03:34.777Z 7 140-159 122 122 122 211 211 211 F F F
137252024-03-06T21:03:34.777Z 8 160-179 124 124 124 214 214 214 F F F
137262024-03-06T21:03:34.778Z 9 180-199 127 127 127 223 223 223 F F F
137272024-03-06T21:03:34.778Z 10 200-219 126 126 126 221 221 221 F F F
137282024-03-06T21:03:34.778Z 11 220-239 125 125 125 217 217 217 F F F
137292024-03-06T21:03:34.778Z 12 240-259 125 125 125 217 217 217 F F F
137302024-03-06T21:03:34.778Z 13 260-279 121 121 121 209 209 209 F F F
137312024-03-06T21:03:34.778Z 14 280-299 122 122 122 210 210 210 F F F
137322024-03-06T21:03:34.778Z 15 300-319 125 125 125 217 217 217 F F F
137332024-03-06T21:03:34.778Z 16 320-339 127 127 127 223 223 223 F F F
137342024-03-06T21:03:34.778Z 17 340-359 127 127 127 223 223 223 F F F
137352024-03-06T21:03:34.778Z 18 360-379 124 124 124 215 215 215 F F F
137362024-03-06T21:03:34.781Z 19 380-399 119 119 119 206 206 206 F F F
137372024-03-06T21:03:34.782Z 20 400-419 127 127 127 223 223 223 F F F
137382024-03-06T21:03:34.782Z 21 420-439 117 117 117 204 204 204 F F F
137392024-03-06T21:03:34.782Z 22 440-459 127 127 127 223 223 223 F F F
137402024-03-06T21:03:34.782Z 23 460-479 127 127 127 223 223 223 F F F
137412024-03-06T21:03:34.782Z 24 480-499 126 126 126 220 220 220 F F F
137422024-03-06T21:03:34.782Z 25 500-519 127 127 127 223 223 223 F F F
137432024-03-06T21:03:34.782Z 26 520-539 126 126 126 222 222 222 F F F
137442024-03-06T21:03:34.782Z 27 540-559 119 119 119 206 206 206 F F F
137452024-03-06T21:03:34.782Z 28 560-579 124 124 124 216 216 216 F F F
137462024-03-06T21:03:34.782Z 29 580-599 124 124 124 216 216 216 F F F
137472024-03-06T21:03:34.782ZMax gen: 127, Max flush: 223
137482024-03-06T21:03:34.782Z
137492024-03-06T21:03:34.782ZBegin loop 64
137502024-03-06T21:03:34.782ZDownstairs to restart: 1
137512024-03-06T21:03:34.803Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.551385725Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651}
137522024-03-06T21:03:34.803Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.551645157Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651}
137532024-03-06T21:03:34.803Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.551663076Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651}
137542024-03-06T21:03:34.803Z{"msg":"Crucible 94d36d08-9946-4e32-9a97-53630f596323 has session id: 72d1efce-ca55-488d-a0fe-d2214733990d","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.551678545Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137552024-03-06T21:03:34.803ZCrucible runtime is spawned
137562024-03-06T21:03:34.809Z{"msg":"Upstairs opts: Upstairs UUID: 94d36d08-9946-4e32-9a97-53630f596323, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.55173916Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137572024-03-06T21:03:34.809Z{"msg":"Crucible stats registered with UUID: 94d36d08-9946-4e32-9a97-53630f596323","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.551772398Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137582024-03-06T21:03:34.809Z{"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.551864312Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651}
137592024-03-06T21:03:34.809Z{"msg":"94d36d08-9946-4e32-9a97-53630f596323 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.551959965Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137602024-03-06T21:03:34.809Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.551985654Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"client":"0","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137612024-03-06T21:03:34.809Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.55203301Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"client":"0","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137622024-03-06T21:03:34.809Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.552067488Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"client":"1","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137632024-03-06T21:03:34.809Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.552108045Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"client":"1","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137642024-03-06T21:03:34.809Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.552145752Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"client":"2","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137652024-03-06T21:03:34.809Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.552170491Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"client":"2","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137662024-03-06T21:03:34.809Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.552192609Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"":"io task","client":"0","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137672024-03-06T21:03:34.809Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.552224227Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"":"io task","client":"1","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137682024-03-06T21:03:34.810Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.552253265Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"":"io task","client":"2","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137692024-03-06T21:03:34.810Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.552459541Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"":"io task","client":"0","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137702024-03-06T21:03:34.810Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.552515947Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"":"io task","client":"2","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137712024-03-06T21:03:34.810Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.552636119Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"":"io task","client":"1","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137722024-03-06T21:03:34.810Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.553175781Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"client":"2","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137732024-03-06T21:03:34.810Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.553212809Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"client":"2","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137742024-03-06T21:03:34.810Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.553228878Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"client":"0","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137752024-03-06T21:03:34.810Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.553243057Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"client":"0","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137762024-03-06T21:03:34.810Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.553731423Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"client":"0","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137772024-03-06T21:03:34.810Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.553858934Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"client":"2","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137782024-03-06T21:03:34.810Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.554338411Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"client":"0","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137792024-03-06T21:03:34.810Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.554396227Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"client":"0","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137802024-03-06T21:03:34.810Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.554427125Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137812024-03-06T21:03:34.810Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.554449634Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"client":"2","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137822024-03-06T21:03:34.810Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.554481091Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"client":"2","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137832024-03-06T21:03:34.810Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.55450562Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137842024-03-06T21:03:34.810Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.554722955Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"client":"1","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137852024-03-06T21:03:34.810Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.554775271Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"client":"1","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137862024-03-06T21:03:34.810Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.5552223Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"client":"1","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137872024-03-06T21:03:34.810Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.555641871Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"client":"1","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137882024-03-06T21:03:34.810Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.555683549Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"client":"1","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137892024-03-06T21:03:34.810Z{"msg":"[0]R flush_numbers[0..12]: [217, 217, 215, 223, 223, 209, 223, 211, 214, 223, 221, 217]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.555703057Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"":"downstairs","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137902024-03-06T21:03:34.810Z{"msg":"[0]R generation[0..12]: [125, 125, 124, 127, 127, 121, 127, 122, 124, 127, 126, 125]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.555729056Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"":"downstairs","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137912024-03-06T21:03:34.811Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.555747664Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"":"downstairs","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137922024-03-06T21:03:34.811Z{"msg":"[1]R flush_numbers[0..12]: [217, 217, 215, 223, 223, 209, 223, 211, 214, 223, 221, 217]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.555773233Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"":"downstairs","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137932024-03-06T21:03:34.811Z{"msg":"[1]R generation[0..12]: [125, 125, 124, 127, 127, 121, 127, 122, 124, 127, 126, 125]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.555791631Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"":"downstairs","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137942024-03-06T21:03:34.811Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":Wait for a query_work_queue command to finish before sending IO
137952024-03-06T21:03:34.811Z"2024-03-06T21:03:33.55581711Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"":"downstairs","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137962024-03-06T21:03:34.811Z{"msg":"[2]R flush_numbers[0..12]: [217, 217, 215, 223, 223, 209, 223, 211, 214, 223, 221, 217]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.555838458Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"":"downstairs","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137972024-03-06T21:03:34.811Z{"msg":"[2]R generation[0..12]: [125, 125, 124, 127, 127, 121, 127, 122, 124, 127, 126, 125]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.555864316Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"":"downstairs","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137982024-03-06T21:03:34.811Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.555882745Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"":"downstairs","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
137992024-03-06T21:03:34.811Z{"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.555907603Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"":"downstairs","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
138002024-03-06T21:03:34.811Z{"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.555924992Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"":"downstairs","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
138012024-03-06T21:03:34.811Z{"msg":"Next flush: 224","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.555941401Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"":"downstairs","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
138022024-03-06T21:03:34.811Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.5559578Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"":"downstairs","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
138032024-03-06T21:03:34.816Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.555989447Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
138042024-03-06T21:03:34.817Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.556046284Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"client":"0","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
138052024-03-06T21:03:34.817Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.556066402Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"client":"1","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
138062024-03-06T21:03:34.817Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.556092291Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"client":"2","":"downstairs","session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
138072024-03-06T21:03:34.817Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.556109159Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
138082024-03-06T21:03:34.817Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.556124478Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
138092024-03-06T21:03:34.817Z{"msg":"94d36d08-9946-4e32-9a97-53630f596323 is now active with session: 72d1efce-ca55-488d-a0fe-d2214733990d","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.556148577Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
138102024-03-06T21:03:34.817Z"session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
138112024-03-06T21:03:34.817Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.556169965Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651,"session_id":"72d1efce-ca55-488d-a0fe-d2214733990d"}
138122024-03-06T21:03:34.817Z{"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:33.556193733Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651}
138132024-03-06T21:03:34.817ZLoading write count information from file "/tmp/test_repair_verify.data"
138142024-03-06T21:03:34.817ZRun Repair workload
138152024-03-06T21:03:34.817Z01/30 Write block 65 len 16384 data: 16 15 14 12
138162024-03-06T21:03:34.817Z02/30 Read block 357 len 32768
138172024-03-06T21:03:34.817Z03/30 Write block 228 len 20480 data: 9 8 10 11 9
138182024-03-06T21:03:34.817Z04/30 Write block 126 len 8192 data: 11 13
138192024-03-06T21:03:34.817Z05/30 Write block 11 len 28672 data: 11 10 11 10 12 10 12
138202024-03-06T21:03:34.817Z06/30 Flush
138212024-03-06T21:03:34.821Z07/30 Write block 256 len 40960 data: 4 4 5 5 4 6 5 7 7 9
138222024-03-06T21:03:34.821Z08/30 Read block 511 len 12288
138232024-03-06T21:03:34.821Z09/30 Read block 18 len 4096
138242024-03-06T21:03:34.821Z10/30 Read block 208 len 8192
138252024-03-06T21:03:34.825Z11/30 Write block 223 len 36864 data: 7 6 6 8 11 10 9 11 12
138262024-03-06T21:03:34.825Z12/30 Write block 379 len 28672 data: 13 13 12 13 13 14 14
138272024-03-06T21:03:34.825Z13/30 Write block 265 len 20480 data: 10 8 8 9 10
138282024-03-06T21:03:34.825Z14/30 Write block 195 len 20480 data: 10 13 13 14 12
138292024-03-06T21:03:34.825Z15/30 Read block 106 len 40960
138302024-03-06T21:03:34.829Z16/30 Read block 7 len 36864
138312024-03-06T21:03:34.829Z17/30 Read block 535 len 20480
138322024-03-06T21:03:34.829Z18/30 Flush
138332024-03-06T21:03:34.832Z19/30 Write block 118 len 20480 data: 7 8 7 7 7
138342024-03-06T21:03:34.832Z20/30 Read block 293 len 20480
138352024-03-06T21:03:34.837Z21/30 Read block 553 len 16384
138362024-03-06T21:03:34.837Z22/30 Read block 97 len 4096
138372024-03-06T21:03:34.838Z23/30 Write block 527 len 40960 data: 9 11 11 12 11 10 9 8 10 8
138382024-03-06T21:03:34.838Z24/30 Write block 147 len 12288 data: 6 9 10
138392024-03-06T21:03:34.841Z25/30 Write block 263 len 24576 data: 8 8 11 9 9 10
138402024-03-06T21:03:34.841Z26/30 Write block 153 len 28672 data: 9 10 12 15 15 12 9
138412024-03-06T21:03:34.841Z27/30 Write block 9 len 4096 data: 8
138422024-03-06T21:03:34.841Z28/30 Write block 514 len 4096 data: 11
138432024-03-06T21:03:34.841Z29/30 Write block 291 len 8192 data: 11 8
138442024-03-06T21:03:34.841Z30/30 Write block 495 len 28672 data: 11 11 11 10 9 9 6
138452024-03-06T21:03:34.842Z----------------------------------------------------------------
138462024-03-06T21:03:34.842Z Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:30
138472024-03-06T21:03:34.842ZStates: Active Active Active
138482024-03-06T21:03:34.842ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
138492024-03-06T21:03:34.842Z 1 Acked 1000 Write 4 Done Done Done false
138502024-03-06T21:03:34.842Z 2 Acked 1001 Read 8 Done Done Done false
138512024-03-06T21:03:34.842Z 3 Acked 1002 Write 5 Done Sent Done false
138522024-03-06T21:03:34.842Z 4 Acked 1003 Write 2 Done Sent Done false
138532024-03-06T21:03:34.842Z 5 Acked 1004 Write 7 Done Sent Done false
138542024-03-06T21:03:34.842Z 6 Acked 1005 Flush 0 Done Sent Done false
138552024-03-06T21:03:34.842Z 7 Acked 1006 Write 10 Done Sent Done false
138562024-03-06T21:03:34.842Z 8 Acked 1007 Read 3 Done Sent Done false
138572024-03-06T21:03:34.842Z 9 Acked 1008 Read 1 Done Sent Done false
138582024-03-06T21:03:34.842Z 10 Acked 1009 Read 2 Done Sent Done false
138592024-03-06T21:03:34.842Z 11 Acked 1010 Write 9 Done Sent Done false
138602024-03-06T21:03:34.842Z 12 Acked 1011 Write 7 Done Sent Done false
138612024-03-06T21:03:34.842Z 13 Acked 1012 Write 5 Done Sent Done false
138622024-03-06T21:03:34.842Z 14 Acked 1013 Write 5 Done Sent Done false
138632024-03-06T21:03:34.842Z 15 Acked 1014 Read 10 Done Sent Done false
138642024-03-06T21:03:34.842Z 16 Acked 1015 Read 9 Done Sent Done false
138652024-03-06T21:03:34.842Z 17 Acked 1016 Read 5 Done Sent Done false
138662024-03-06T21:03:34.842Z 18 Acked 1017 Flush 0 Done Sent Done false
138672024-03-06T21:03:34.842Z 19 Acked 1018 Write 5 Done Sent Done false
138682024-03-06T21:03:34.842Z 20 Acked 1019 Read 5 Done Sent Done false
138692024-03-06T21:03:34.843Z 21 Acked 1020 Read 4 Done Sent Done false
138702024-03-06T21:03:34.843Z 22 Acked 1021 Read 1 Done Sent Done false
138712024-03-06T21:03:34.843Z 23 Acked 1022 Write 10 Done Sent Done false
138722024-03-06T21:03:34.843Z 24 Acked 1023 Write 3 Done Sent Done false
138732024-03-06T21:03:34.843Z 25 Acked 1024 Write 6 Done Sent Done false
138742024-03-06T21:03:34.843Z 26 Acked 1025 Write 7 Done Sent Done false
138752024-03-06T21:03:34.843Z 27 Acked 1026 Write 1 Done Sent Done false
138762024-03-06T21:03:34.843Z 28 Acked 1027 Write 1 Done Sent Done false
138772024-03-06T21:03:34.843Z 29 Acked 1028 Write 2 Done Sent Done false
138782024-03-06T21:03:34.843Z 30 Acked 1029 Write 7 Sent Sent Sent false
138792024-03-06T21:03:34.843Z STATES DS:0 DS:1 DS:2 TOTAL
138802024-03-06T21:03:34.843Z New 0 0 0 0
138812024-03-06T21:03:34.843Z Sent 1 28 1 30
138822024-03-06T21:03:34.843Z Done 29 2 29 60
138832024-03-06T21:03:34.843Z Skipped 0 0 0 0
138842024-03-06T21:03:34.843Z Error 0 0 0 0
138852024-03-06T21:03:34.843ZLast Flush: 1017 0 1017
138862024-03-06T21:03:34.843ZDownstairs last five completed:
138872024-03-06T21:03:34.843ZUpstairs last five completed: 30 29 28 27 26
138882024-03-06T21:03:34.847Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:03:33.590349889Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1651}
138892024-03-06T21:03:34.847ZWrote out file "/tmp/test_repair_verify.data"
138902024-03-06T21:03:34.847Z
138912024-03-06T21:03:34.851ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
138922024-03-06T21:03:34.870ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
138932024-03-06T21:03:34.870Z 0 000-019 128 125 128 224 217 224 T F T <---
138942024-03-06T21:03:34.870Z 1 020-039 125 125 125 217 217 217 F F F
138952024-03-06T21:03:34.871Z 2 040-059 124 124 124 215 215 215 F F F
138962024-03-06T21:03:34.871Z 3 060-079 128 127 128 224 223 224 F T F <---
138972024-03-06T21:03:34.871Z 4 080-099 127 127 127 223 223 223 F F F
138982024-03-06T21:03:34.871Z 5 100-119 121 121 121 209 209 209 T F T <---
138992024-03-06T21:03:34.871Z 6 120-139 128 127 128 224 223 224 T F T <---
139002024-03-06T21:03:34.871Z 7 140-159 122 122 122 211 211 211 T F T <---
139012024-03-06T21:03:34.871Z 8 160-179 124 124 124 214 214 214 F F F
139022024-03-06T21:03:34.871Z 9 180-199 128 127 128 225 223 225 F F F <---
139032024-03-06T21:03:34.871Z 10 200-219 126 126 126 221 221 221 F F F
139042024-03-06T21:03:34.871Z 11 220-239 128 125 128 225 217 225 F F F <---
139052024-03-06T21:03:34.871Z 12 240-259 128 125 128 225 217 225 F F F <---
139062024-03-06T21:03:34.871Z 13 260-279 128 121 128 225 209 225 T F T <---
139072024-03-06T21:03:34.871Z 14 280-299 122 122 122 210 210 210 T F T <---
139082024-03-06T21:03:34.871Z 15 300-319 125 125 125 217 217 217 F F F
139092024-03-06T21:03:34.871Z 16 320-339 127 127 127 223 223 223 F F F
139102024-03-06T21:03:34.871Z 17 340-359 127 127 127 223 223 223 F F F
139112024-03-06T21:03:34.871Z 18 360-379 128 124 128 225 215 225 F F F <---
139122024-03-06T21:03:34.876Z 19 380-399 128 119 128 225 206 225 F F F <---
139132024-03-06T21:03:34.876Z 20 400-419 127 127 127 223 223 223 F F F
139142024-03-06T21:03:34.876Z 21 420-439 117 117 117 204 204 204 F F F
139152024-03-06T21:03:34.876Z 22 440-459 127 127 127 223 223 223 F F F
139162024-03-06T21:03:34.876Z 23 460-479 127 127 127 223 223 223 F F F
139172024-03-06T21:03:34.876Z 24 480-499 126 126 126 220 220 220 T F T <---
139182024-03-06T21:03:34.876Z 25 500-519 127 127 127 223 223 223 T F T <---
139192024-03-06T21:03:34.876Z 26 520-539 126 126 126 222 222 222 T F T <---
139202024-03-06T21:03:34.876Z 27 540-559 119 119 119 206 206 206 F F F
139212024-03-06T21:03:34.876Z 28 560-579 124 124 124 216 216 216 F F F
139222024-03-06T21:03:34.876Z 29 580-599 124 124 124 216 216 216 F F F
139232024-03-06T21:03:34.876ZMax gen: 128, Max flush: 225
139242024-03-06T21:03:34.877ZError: Difference in extent metadata found!
139252024-03-06T21:03:34.880ZA Difference in extent metadata is expected here
139262024-03-06T21:03:34.880ZOn loop 64
139272024-03-06T21:03:34.880Z
139282024-03-06T21:03:34.880ZVerifying data now
139292024-03-06T21:03:39.669ZNo change after verify
139302024-03-06T21:03:39.669ZLoop: 64 Downstairs dump after verify (and repair):
139312024-03-06T21:03:39.691ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
139322024-03-06T21:03:39.691Z 0 000-019 129 129 129 226 226 226 F F F
139332024-03-06T21:03:39.691Z 1 020-039 125 125 125 217 217 217 F F F
139342024-03-06T21:03:39.691Z 2 040-059 124 124 124 215 215 215 F F F
139352024-03-06T21:03:39.691Z 3 060-079 128 128 128 224 224 224 F F F
139362024-03-06T21:03:39.692Z 4 080-099 127 127 127 223 223 223 F F F
139372024-03-06T21:03:39.692Z 5 100-119 129 129 129 226 226 226 F F F
139382024-03-06T21:03:39.692Z 6 120-139 129 129 129 226 226 226 F F F
139392024-03-06T21:03:39.692Z 7 140-159 129 129 129 226 226 226 F F F
139402024-03-06T21:03:39.692Z 8 160-179 124 124 124 214 214 214 F F F
139412024-03-06T21:03:39.692Z 9 180-199 128 128 128 225 225 225 F F F
139422024-03-06T21:03:39.692Z 10 200-219 126 126 126 221 221 221 F F F
139432024-03-06T21:03:39.692Z 11 220-239 128 128 128 225 225 225 F F F
139442024-03-06T21:03:39.692Z 12 240-259 128 128 128 225 225 225 F F F
139452024-03-06T21:03:39.692Z 13 260-279 129 129 129 226 226 226 F F F
139462024-03-06T21:03:39.692Z 14 280-299 129 129 129 226 226 226 F F F
139472024-03-06T21:03:39.692Z 15 300-319 125 125 125 217 217 217 F F F
139482024-03-06T21:03:39.692Z 16 320-339 127 127 127 223 223 223 F F F
139492024-03-06T21:03:39.693Z 17 340-359 127 127 127 223 223 223 F F F
139502024-03-06T21:03:39.693Z 18 360-379 128 128 128 225 225 225 F F F
139512024-03-06T21:03:39.693Z 19 380-399 128 128 128 225 225 225 F F F
139522024-03-06T21:03:39.693Z 20 400-419 127 127 127 223 223 223 F F F
139532024-03-06T21:03:39.693Z 21 420-439 117 117 117 204 204 204 F F F
139542024-03-06T21:03:39.693Z 22 440-459 127 127 127 223 223 223 F F F
139552024-03-06T21:03:39.693Z 23 460-479 127 127 127 223 223 223 F F F
139562024-03-06T21:03:39.693Z 24 480-499 129 129 129 226 226 226 F F F
139572024-03-06T21:03:39.699Z 25 500-519 129 129 129 226 226 226 F F F
139582024-03-06T21:03:39.699Z 26 520-539 129 129 129 226 226 226 F F F
139592024-03-06T21:03:39.699Z 27 540-559 119 119 119 206 206 206 F F F
139602024-03-06T21:03:39.699Z 28 560-579 124 124 124 216 216 216 F F F
139612024-03-06T21:03:39.699Z 29 580-599 124 124 124 216 216 216 F F F
139622024-03-06T21:03:39.699ZMax gen: 129, Max flush: 226
139632024-03-06T21:03:39.699Z
139642024-03-06T21:03:39.700ZBegin loop 65
139652024-03-06T21:03:39.700ZDownstairs to restart: 0
139662024-03-06T21:03:39.713Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.461531592Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659}
139672024-03-06T21:03:39.713Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.461802314Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659}
139682024-03-06T21:03:39.713Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.461820983Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659}
139692024-03-06T21:03:39.714Z{"msg":"Crucible b1ea6d2f-3cf4-4bc3-a18a-a1dfb34a82d2 has session id: 207a4353-728b-4469-9e39-2a55c204ee2b","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.461844061Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
139702024-03-06T21:03:39.722Z{"msg":"Upstairs opts: Upstairs UUID: b1ea6d2f-3cf4-4bc3-a18a-a1dfb34a82d2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.461884428Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"session_id":"Crucible runtime is spawned
139712024-03-06T21:03:39.722Z207a4353-728b-4469-9e39-2a55c204ee2b"}
139722024-03-06T21:03:39.722Z{"msg":"Crucible stats registered with UUID: b1ea6d2f-3cf4-4bc3-a18a-a1dfb34a82d2","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.461917966Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
139732024-03-06T21:03:39.723Z{"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.461983131Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659}
139742024-03-06T21:03:39.723Z{"msg":"b1ea6d2f-3cf4-4bc3-a18a-a1dfb34a82d2 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.462140201Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
139752024-03-06T21:03:39.723Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.462172568Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"client":"0","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
139762024-03-06T21:03:39.723Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.462190457Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"client":"0","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
139772024-03-06T21:03:39.723Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.462229334Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"client":"1","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
139782024-03-06T21:03:39.723Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.462256442Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"client":"1","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
139792024-03-06T21:03:39.723Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.46228634Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"client":"2","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
139802024-03-06T21:03:39.723Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.462310429Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"client":"2","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
139812024-03-06T21:03:39.723Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.462384484Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"":"io task","client":"2","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
139822024-03-06T21:03:39.724Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.46243557Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"":"io task","client":"0","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
139832024-03-06T21:03:39.724Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.462463408Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"":"io task","client":"1","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
139842024-03-06T21:03:39.724Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.462502805Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"":"io task","client":"1","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
139852024-03-06T21:03:39.724Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.462601359Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"":"io task","client":"0","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
139862024-03-06T21:03:39.725Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.462640966Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"":"io task","client":"2","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
139872024-03-06T21:03:39.725Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.463276572Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"client":"1","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
139882024-03-06T21:03:39.725Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.463319129Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"client":"1","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
139892024-03-06T21:03:39.725Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.463346307Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"client":"2","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
139902024-03-06T21:03:39.726Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.463394254Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"client":"2","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
139912024-03-06T21:03:39.726Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.463970024Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"client":"2","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
139922024-03-06T21:03:39.726Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.464013581Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"client":"1","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
139932024-03-06T21:03:39.726Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.464089896Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"client":"0","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
139942024-03-06T21:03:39.726Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.464127633Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"client":"0","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
139952024-03-06T21:03:39.727Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.464550474Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"client":"2","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
139962024-03-06T21:03:39.727Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.46461833Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"client":"2","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
139972024-03-06T21:03:39.727Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.464636298Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
139982024-03-06T21:03:39.728Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.464707593Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"client":"1","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
139992024-03-06T21:03:39.728Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.464735782Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"client":"1","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
140002024-03-06T21:03:39.728Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.46474979Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
140012024-03-06T21:03:39.728Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.464995764Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"client":"0","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
140022024-03-06T21:03:39.729Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.465455242Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"client":"0","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
140032024-03-06T21:03:39.729Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.46548929Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"client":"0","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
140042024-03-06T21:03:39.729Z{"msg":"[0]R flush_numbers[0..12]: [226, 217, 215, 224, 223, 226, 226, 226, 214, 225, 221, 225]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.465504118Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"":"downstairs","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
140052024-03-06T21:03:39.729Z{"msg":"[0]R generation[0..12]: [129, 125, 124, 128, 127, 129, 129, 129, 124, 128, 126, 128]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.465586903Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"":"downstairs","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
140062024-03-06T21:03:39.730Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.46562614Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"":"downstairs","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
140072024-03-06T21:03:39.730Z{"msg":"[1]R flush_numbers[0..12]: [226, 217, 215, 224, 223, 226, 226, 226, 214, 225, 221, 225]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.465680897Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"":"downstairs","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
140082024-03-06T21:03:39.730Z{"msg":"[1]R generation[0..12]: [129, 125, 124, 128, 127, 129, 129, 129, 124, 128, 126, 128]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
140092024-03-06T21:03:39.730Z,"time":"2024-03-06T21:03:38.465718214Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"":"downstairs","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
140102024-03-06T21:03:39.730Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.465749332Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"":"downstairs","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
140112024-03-06T21:03:39.731Z{"msg":"[2]R flush_numbers[0..12]: [226, 217, 215, 224, 223, 226, 226, 226, 214, 225, 221, 225]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.465786469Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"":"downstairs","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
140122024-03-06T21:03:39.731Z{"msg":"[2]R generation[0..12]: [129, 125, 124, 128, 127, 129, 129, 129, 124, 128, 126, 128]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.465827016Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"":"downstairs","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
140132024-03-06T21:03:39.731Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.465866134Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"":"downstairs","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
140142024-03-06T21:03:39.732Z{"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.465902401Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"":"downstairs","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
140152024-03-06T21:03:39.732Z{"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.465937139Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"":"downstairs","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
140162024-03-06T21:03:39.732Z{"msg":"Next flush: 227","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.465972036Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"":"downstairs","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
140172024-03-06T21:03:39.732Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
140182024-03-06T21:03:39.733Z2024-03-06T21:03:38.466025363Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"":"downstairs","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
140192024-03-06T21:03:39.733Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.46606901Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
140202024-03-06T21:03:39.733Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.466093058Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"client":"0","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
140212024-03-06T21:03:39.733Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.466107297Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"client":"1","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
140222024-03-06T21:03:39.734Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.466127526Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"client":"2","":"downstairs","session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
140232024-03-06T21:03:39.734Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.466141195Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
140242024-03-06T21:03:39.734Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.466159983Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
140252024-03-06T21:03:39.734Z{"msg":"b1ea6d2f-3cf4-4bc3-a18a-a1dfb34a82d2 is now active with session: 207a4353-728b-4469-9e39-2a55c204ee2b","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.466172852Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
140262024-03-06T21:03:39.735Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.466191511Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659,"session_id":"207a4353-728b-4469-9e39-2a55c204ee2b"}
140272024-03-06T21:03:39.735Z{"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:38.46620414Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659}
140282024-03-06T21:03:39.745ZLoading write count information from file "/tmp/test_repair_verify.data"
140292024-03-06T21:03:39.745ZRun Repair workload
140302024-03-06T21:03:39.746Z01/30 Write block 557 len 40960 data: 6 6 5 9 9 10 9 10 9 9
140312024-03-06T21:03:39.746Z02/30 Read block 339 len 24576
140322024-03-06T21:03:39.746Z03/30 Write block 557 len 40960 data: 7 7 6 10 10 11 10 11 10 10
140332024-03-06T21:03:39.746Z04/30 Write block 367 len 20480 data: 10 8 12 13 14
140342024-03-06T21:03:39.746Z05/30 Write block 228 len 32768 data: 11 10 12 13 10 9 11 13
140352024-03-06T21:03:39.747Z06/30 Read block 420 len 40960
140362024-03-06T21:03:39.747Z07/30 Write block 441 len 36864 data: 9 9 8 9 8 9 7 11 10
140372024-03-06T21:03:39.747Z08/30 Read block 515 len 20480
140382024-03-06T21:03:39.747Z09/30 Read block 148 len 28672
140392024-03-06T21:03:39.747Z10/30 Read block 236 len 8192
140402024-03-06T21:03:39.748Z11/30 Write block 189 len 28672 data: 12 13 10 12 9 9 11
140412024-03-06T21:03:39.748Z12/30 Read block 204 len 36864
140422024-03-06T21:03:39.748Z13/30 Write block 61 len 32768 data: 18 17 20 17 17 16 15 13
140432024-03-06T21:03:39.748Z14/30 Write block 29 len 28672 data: 10 10 9 10 9 9 8
140442024-03-06T21:03:39.749Z15/30 Read block 358 len 28672
140452024-03-06T21:03:39.749Z16/30 Write block 202 len 36864 data: 8 8 8 8 9 5 8 7 9
140462024-03-06T21:03:39.749Z17/30 Write block 204 len 28672 data: 9 9 10 6 9 8 10
140472024-03-06T21:03:39.749Z18/30 Write block 581 len 24576 data: 9 8 9 11 12 14
140482024-03-06T21:03:39.749Z19/30 Write block 253 len 4096 data: 6
140492024-03-06T21:03:39.749Z20/30 Read block 564 len 12288
140502024-03-06T21:03:39.750Z21/30 Read block 484 len 12288
140512024-03-06T21:03:39.750Z22/30 Read block 423 len 8192
140522024-03-06T21:03:39.765Z23/30 Read block 171 len 40960
140532024-03-06T21:03:39.765Z24/30 Read block 89 len 40960
140542024-03-06T21:03:39.769Z25/30 Flush
140552024-03-06T21:03:39.769Z26/30 Write block 155 len 40960 data: 13 16 16 13 10 9 10 10 8 8
140562024-03-06T21:03:39.769Z27/30 Read block 207 len 28672
140572024-03-06T21:03:39.769Z28/30 Write block 16 len 40960 data: 11 13 13 15 16 13 14 14 11 10
140582024-03-06T21:03:39.770Z29/30 Read block 583 len 28672
140592024-03-06T21:03:39.770Z30/30 Write block 438 len 16384 data: 13 14 12 10
140602024-03-06T21:03:39.771Z----------------------------------------------------------------
140612024-03-06T21:03:39.771Z Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:30
140622024-03-06T21:03:39.771ZStates: Active Active Active
140632024-03-06T21:03:39.772ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
140642024-03-06T21:03:39.772Z 1 Acked 1000 Write 10 Done Done Done false
140652024-03-06T21:03:39.772Z 2 Acked 1001 Read 6 Done Done Done false
140662024-03-06T21:03:39.773Z 3 Acked 1002 Write 10 Done Done Done false
140672024-03-06T21:03:39.773Z 4 Acked 1003 Write 5 Done Done Done false
140682024-03-06T21:03:39.774Z 5 Acked 1004 Write 8 Done Done Done false
140692024-03-06T21:03:39.774Z 6 Acked 1005 Read 10 Done Done Done false
140702024-03-06T21:03:39.774Z 7 Acked 1006 Write 9 Sent Done Done false
140712024-03-06T21:03:39.775Z 8 Acked 1007 Read 5 Sent Done Done false
140722024-03-06T21:03:39.775Z 9 Acked 1008 Read 7 Sent Done Done false
140732024-03-06T21:03:39.776Z 10 Acked 1009 Read 2 Sent Done Done false
140742024-03-06T21:03:39.776Z 11 Acked 1010 Write 7 Sent Done Done false
140752024-03-06T21:03:39.777Z 12 Acked 1011 Read 9 Sent Done Done false
140762024-03-06T21:03:39.777Z 13 Acked 1012 Write 8 Sent Done Done false
140772024-03-06T21:03:39.777Z 14 Acked 1013 Write 7 Sent Done Done false
140782024-03-06T21:03:39.778Z 15 Acked 1014 Read 7 Sent Done Done false
140792024-03-06T21:03:39.778Z 16 Acked 1015 Write 9 Sent Done Done false
140802024-03-06T21:03:39.779Z 17 Acked 1016 Write 7 Sent Done Done false
140812024-03-06T21:03:39.779Z 18 Acked 1017 Write 6 Sent Done Done false
140822024-03-06T21:03:39.779Z 19 Acked 1018 Write 1 Sent Done Done false
140832024-03-06T21:03:39.780Z 20 Acked 1019 Read 3 Sent Done Done false
140842024-03-06T21:03:39.780Z 21 Acked 1020 Read 3 Sent Done Done false
140852024-03-06T21:03:39.780Z 22 Acked 1021 Read 2 Sent Done Done false
140862024-03-06T21:03:39.781Z 23 Acked 1022 Read 10 Sent Done Done false
140872024-03-06T21:03:39.781Z 24 Acked 1023 Read 10 Sent Done Done false
140882024-03-06T21:03:39.781Z 25 Acked 1024 Flush 0 Sent Done Done false
140892024-03-06T21:03:39.781Z 26 Acked 1025 Write 10 Sent Done Done false
140902024-03-06T21:03:39.782Z 27 Acked 1026 Read 7 Sent Done Done false
140912024-03-06T21:03:39.782Z 28 Acked 1027 Write 10 Sent Done Done false
140922024-03-06T21:03:39.782Z 29 Acked 1028 Read 7 Sent Done Done false
140932024-03-06T21:03:39.782Z 30 Acked 1029 Write 4 Sent Sent Sent false
140942024-03-06T21:03:39.782Z STATES DS:0 DS:1 DS:2 TOTAL
140952024-03-06T21:03:39.783Z New 0 0 0 0
140962024-03-06T21:03:39.783Z Sent 24 1 1 26
140972024-03-06T21:03:39.783Z Done 6 29 29 64
140982024-03-06T21:03:39.783Z Skipped 0 0 0 0
140992024-03-06T21:03:39.783Z Error 0 0 0 0
141002024-03-06T21:03:39.783ZLast Flush: 0 1024 1024
141012024-03-06T21:03:39.784ZDownstairs last five completed:
141022024-03-06T21:03:39.784ZUpstairs last five completed: 30 29 28 27 26
141032024-03-06T21:03:39.784Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:03:38.504761983Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1659}
141042024-03-06T21:03:39.785ZWrote out file "/tmp/test_repair_verify.data"
141052024-03-06T21:03:39.792Z
141062024-03-06T21:03:39.792ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
141072024-03-06T21:03:39.792ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
141082024-03-06T21:03:39.793Z 0 000-019 129 129 129 226 226 226 F T T <---
141092024-03-06T21:03:39.793Z 1 020-039 125 130 130 217 227 227 F T T <---
141102024-03-06T21:03:39.793Z 2 040-059 124 124 124 215 215 215 F F F
141112024-03-06T21:03:39.793Z 3 060-079 128 130 130 224 227 227 F F F <---
141122024-03-06T21:03:39.793Z 4 080-099 127 127 127 223 223 223 F F F
141132024-03-06T21:03:39.794Z 5 100-119 129 129 129 226 226 226 F F F
141142024-03-06T21:03:39.794Z 6 120-139 129 129 129 226 226 226 F F F
141152024-03-06T21:03:39.794Z 7 140-159 129 129 129 226 226 226 F T T <---
141162024-03-06T21:03:39.794Z 8 160-179 124 124 124 214 214 214 F T T <---
141172024-03-06T21:03:39.794Z 9 180-199 128 130 130 225 227 227 F F F <---
141182024-03-06T21:03:39.794Z 10 200-219 126 130 130 221 227 227 F F F <---
141192024-03-06T21:03:39.795Z 11 220-239 128 130 130 225 227 227 T F F <---
141202024-03-06T21:03:39.795Z 12 240-259 128 130 130 225 227 227 F F F <---
141212024-03-06T21:03:39.795Z 13 260-279 129 129 129 226 226 226 F F F
141222024-03-06T21:03:39.795Z 14 280-299 129 129 129 226 226 226 F F F
141232024-03-06T21:03:39.795Z 15 300-319 125 125 125 217 217 217 F F F
141242024-03-06T21:03:39.796Z 16 320-339 127 127 127 223 223 223 F F F
141252024-03-06T21:03:39.796Z 17 340-359 127 127 127 223 223 223 F F F
141262024-03-06T21:03:39.796Z 18 360-379 128 130 130 225 227 227 T F F <---
141272024-03-06T21:03:39.796Z 19 380-399 128 128 128 225 225 225 F F F
141282024-03-06T21:03:39.796Z 20 400-419 127 127 127 223 223 223 F F F
141292024-03-06T21:03:39.796Z 21 420-439 117 117 117 204 204 204 F T T <---
141302024-03-06T21:03:39.796Z 22 440-459 127 130 130 223 227 227 F T T <---
141312024-03-06T21:03:39.796Z 23 460-479 127 127 127 223 223 223 F F F
141322024-03-06T21:03:39.797Z 24 480-499 129 129 129 226 226 226 F F F
141332024-03-06T21:03:39.797Z 25 500-519 129 129 129 226 226 226 F F F
141342024-03-06T21:03:39.797Z 26 520-539 129 129 129 226 226 226 F F F
141352024-03-06T21:03:39.797Z 27 540-559 119 130 130 206 227 227 T F F <---
141362024-03-06T21:03:39.797Z 28 560-579 124 130 130 216 227 227 T F F <---
141372024-03-06T21:03:39.797Z 29 580-599 124 130 130 216 227 227 F F F <---
141382024-03-06T21:03:39.798ZMax gen: 130, Max flush: 227
141392024-03-06T21:03:39.798ZError: Difference in extent metadata found!
141402024-03-06T21:03:39.798ZA Difference in extent metadata is expected here
141412024-03-06T21:03:39.798ZOn loop 65
141422024-03-06T21:03:39.799Z
141432024-03-06T21:03:39.807ZVerifying data now
141442024-03-06T21:03:45.592ZNo change after verify
141452024-03-06T21:03:45.592ZLoop: 65 Downstairs dump after verify (and repair):
141462024-03-06T21:03:45.620ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
141472024-03-06T21:03:45.621Z 0 000-019 131 131 131 228 228 228 F F F
141482024-03-06T21:03:45.621Z 1 020-039 131 131 131 228 228 228 F F F
141492024-03-06T21:03:45.621Z 2 040-059 124 124 124 215 215 215 F F F
141502024-03-06T21:03:45.622Z 3 060-079 130 130 130 227 227 227 F F F
141512024-03-06T21:03:45.622Z 4 080-099 127 127 127 223 223 223 F F F
141522024-03-06T21:03:45.622Z 5 100-119 129 129 129 226 226 226 F F F
141532024-03-06T21:03:45.622Z 6 120-139 129 129 129 226 226 226 F F F
141542024-03-06T21:03:45.623Z 7 140-159 131 131 131 228 228 228 F F F
141552024-03-06T21:03:45.623Z 8 160-179 131 131 131 228 228 228 F F F
141562024-03-06T21:03:45.623Z 9 180-199 130 130 130 227 227 227 F F F
141572024-03-06T21:03:45.623Z 10 200-219 130 130 130 227 227 227 F F F
141582024-03-06T21:03:45.624Z 11 220-239 130 130 130 227 227 227 F F F
141592024-03-06T21:03:45.624Z 12 240-259 130 130 130 227 227 227 F F F
141602024-03-06T21:03:45.624Z 13 260-279 129 129 129 226 226 226 F F F
141612024-03-06T21:03:45.624Z 14 280-299 129 129 129 226 226 226 F F F
141622024-03-06T21:03:45.624Z 15 300-319 125 125 125 217 217 217 F F F
141632024-03-06T21:03:45.625Z 16 320-339 127 127 127 223 223 223 F F F
141642024-03-06T21:03:45.625Z 17 340-359 127 127 127 223 223 223 F F F
141652024-03-06T21:03:45.625Z 18 360-379 130 130 130 227 227 227 F F F
141662024-03-06T21:03:45.625Z 19 380-399 128 128 128 225 225 225 F F F
141672024-03-06T21:03:45.626Z 20 400-419 127 127 127 223 223 223 F F F
141682024-03-06T21:03:45.626Z 21 420-439 131 131 131 228 228 228 F F F
141692024-03-06T21:03:45.626Z 22 440-459 131 131 131 228 228 228 F F F
141702024-03-06T21:03:45.627Z 23 460-479 127 127 127 223 223 223 F F F
141712024-03-06T21:03:45.627Z 24 480-499 129 129 129 226 226 226 F F F
141722024-03-06T21:03:45.627Z 25 500-519 129 129 129 226 226 226 F F F
141732024-03-06T21:03:45.628Z 26 520-539 129 129 129 226 226 226 F F F
141742024-03-06T21:03:45.628Z 27 540-559 130 130 130 227 227 227 F F F
141752024-03-06T21:03:45.628Z 28 560-579 130 130 130 227 227 227 F F F
141762024-03-06T21:03:45.629Z 29 580-599 130 130 130 227 227 227 F F F
141772024-03-06T21:03:45.629ZMax gen: 131, Max flush: 228
141782024-03-06T21:03:45.655Z
141792024-03-06T21:03:45.655ZBegin loop 66
141802024-03-06T21:03:45.655ZDownstairs to restart: 2
141812024-03-06T21:03:45.655Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.374094316Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667}
141822024-03-06T21:03:45.655Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.374385656Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667}
141832024-03-06T21:03:45.655Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.374416084Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667}
141842024-03-06T21:03:45.655Z{"msg":"Crucible runtime is spawned
141852024-03-06T21:03:45.656ZCrucible 5f9f33ef-55cf-468c-bd22-b245b0e0c515 has session id: 983034d7-178a-4607-899e-1c93f0e6d2c1","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.374462171Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
141862024-03-06T21:03:45.656Z{"msg":"Upstairs opts: Upstairs UUID: 5f9f33ef-55cf-468c-bd22-b245b0e0c515, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.374503858Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
141872024-03-06T21:03:45.656Z{"msg":"Crucible stats registered with UUID: 5f9f33ef-55cf-468c-bd22-b245b0e0c515","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.374535985Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
141882024-03-06T21:03:45.656Z{"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.374559224Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667}
141892024-03-06T21:03:45.656Z{"msg":"5f9f33ef-55cf-468c-bd22-b245b0e0c515 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.374708224Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
141902024-03-06T21:03:45.656Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.374750891Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"client":"0","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
141912024-03-06T21:03:45.656Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.374797047Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"client":"0","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
141922024-03-06T21:03:45.656Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.374825286Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"client":"1","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
141932024-03-06T21:03:45.656Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.374867093Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"client":"1","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
141942024-03-06T21:03:45.656Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.37490873Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"client":"2","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
141952024-03-06T21:03:45.657Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.374942357Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"client":"2","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
141962024-03-06T21:03:45.657Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.374974895Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"":"io task","client":"0","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
141972024-03-06T21:03:45.657Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.375074608Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"":"io task","client":"1","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
141982024-03-06T21:03:45.657Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.375136634Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"":"io task","client":"2","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
141992024-03-06T21:03:45.657Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.375185651Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"":"io task","client":"0","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142002024-03-06T21:03:45.657Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.375245347Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"":"io task","client":"1","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142012024-03-06T21:03:45.657Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.375295733Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"":"io task","client":"2","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142022024-03-06T21:03:45.657Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.376031413Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"client":"1","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142032024-03-06T21:03:45.657Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.376082209Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"client":"1","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142042024-03-06T21:03:45.657Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.376109437Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"client":"0","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142052024-03-06T21:03:45.657Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.376131536Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"client":"0","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142062024-03-06T21:03:45.657Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.376547027Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"client":"2","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142072024-03-06T21:03:45.658Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.376612353Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"client":"2","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142082024-03-06T21:03:45.658Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.376775021Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"client":"1","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142092024-03-06T21:03:45.658Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.376810039Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"client":"0","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142102024-03-06T21:03:45.658Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.377355421Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"client":"0","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142112024-03-06T21:03:45.658Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.377407998Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"client":"0","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142122024-03-06T21:03:45.658Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.377457614Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142132024-03-06T21:03:45.658Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.377496041Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"client":"2","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142142024-03-06T21:03:45.658Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.377532529Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"client":"1","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142152024-03-06T21:03:45.658Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.377567657Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"client":"1","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142162024-03-06T21:03:45.658Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.377606264Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142172024-03-06T21:03:45.658Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.378026665Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"client":"2","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142182024-03-06T21:03:45.659Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.378060543Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"client":"2","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142192024-03-06T21:03:45.659Z{"msg":"[0]R flush_numbers[0..12]: [228, 228, 215, 227, 223, 226, 226, 228, 228, 227, 227, 227]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.378076732Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"":"downstairs","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142202024-03-06T21:03:45.659Z{"msg":"[0]R generation[0..12]: [131, 131, 124, 130, 127, 129, 129, 131, 131, 130, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.37809806Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"":"downstairs","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142212024-03-06T21:03:45.659Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.378119869Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"":"downstairs","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142222024-03-06T21:03:45.659Z{"msg":"[1]R flush_numbers[0..12]: [228, 228, 215, 227, 223, 226, 226, 228, 228, 227, 227, 227]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.378134337Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"":"downstairs","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142232024-03-06T21:03:45.659Z{"msg":"[1]R generation[0..12]: [131, 131, 124, 130, 127, 129, 129, 131, 131, 130, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.378154926Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"":"downstairs","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142242024-03-06T21:03:45.659Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.378169685Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"":"downstairs","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142252024-03-06T21:03:45.659Z{"msg":"[2]R flush_numbers[0..12]: [228, 228, 215, 227, 223, 226, 226, 228, 228, 227, 227, 227]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.378190334Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"":"downstairs","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142262024-03-06T21:03:45.660Z{"msg":"[2]R generation[0..12]: [131, 131, 124, 130, 127, 129, 129, 131, 131, 130, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.378204833Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"":"downstairs",""Wait for a query_work_queue command to finish before sending IO
142272024-03-06T21:03:45.660Z:"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142282024-03-06T21:03:45.660Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.378222371Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"":"downstairs","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142292024-03-06T21:03:45.660Z{"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.378236261Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"":"downstairs","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142302024-03-06T21:03:45.660Z{"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.37824961Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"":"downstairs","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142312024-03-06T21:03:45.660Z{"msg":"Next flush: 229","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.378269378Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"":"downstairs","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142322024-03-06T21:03:45.660Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.378283407Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"":"downstairs","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142332024-03-06T21:03:45.660Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.378303456Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142342024-03-06T21:03:45.660Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.378315845Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"client":"0","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142352024-03-06T21:03:45.660Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.378329134Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"client":"1","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142362024-03-06T21:03:45.661Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.378348603Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"client":"2","":"downstairs","session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142372024-03-06T21:03:45.661Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.378362032Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142382024-03-06T21:03:45.661Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.378374191Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142392024-03-06T21:03:45.661Z{"msg":"5f9f33ef-55cf-468c-bd22-b245b0e0c515 is now active with session: 983034d7-178a-4607-899e-1c93f0e6d2c1","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.37839391Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142402024-03-06T21:03:45.661Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.378406139Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667,"session_id":"983034d7-178a-4607-899e-1c93f0e6d2c1"}
142412024-03-06T21:03:45.661Z{"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:44.378418248Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667}
142422024-03-06T21:03:45.661ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
142432024-03-06T21:03:45.661ZLoading write count information from file "/tmp/test_repair_verify.data"
142442024-03-06T21:03:45.661ZRun Repair workload
142452024-03-06T21:03:45.662Z01/30 Write block 188 len 24576 data: 8 13 14 11 13 10
142462024-03-06T21:03:45.662Z02/30 Write block 40 len 36864 data: 8 11 13 12 11 7 8 10 9
142472024-03-06T21:03:45.662Z03/30 Write block 238 len 24576 data: 18 18 13 13 13 11
142482024-03-06T21:03:45.662Z04/30 Write block 534 len 16384 data: 9 11 9 10
142492024-03-06T21:03:45.662Z05/30 Read block 395 len 28672
142502024-03-06T21:03:45.662Z06/30 Write block 352 len 32768 data: 9 10 11 11 15 13 13 11
142512024-03-06T21:03:45.662Z07/30 Read block 462 len 36864
142522024-03-06T21:03:45.662Z08/30 Write block 165 len 40960 data: 8 7 8 7 5 4 5 7 11 10
142532024-03-06T21:03:45.672Z09/30 Write block 324 len 24576 data: 12 11 13 10 10 9
142542024-03-06T21:03:45.672Z10/30 Write block 179 len 40960 data: 11 13 15 14 13 13 12 10 9 9
142552024-03-06T21:03:45.672Z11/30 Write block 571 len 12288 data: 7 7 8
142562024-03-06T21:03:45.672Z12/30 Flush
142572024-03-06T21:03:45.672Z13/30 Write block 168 len 36864 data: 8 6 5 6 8 12 11 11 10
142582024-03-06T21:03:45.672Z14/30 Write block 430 len 8192 data: 10 8
142592024-03-06T21:03:45.673Z15/30 Flush
142602024-03-06T21:03:45.673Z16/30 Write block 153 len 8192 data: 10 11
142612024-03-06T21:03:45.673Z17/30 Write block 463 len 12288 data: 15 15 14
142622024-03-06T21:03:45.673Z18/30 Read block 283 len 40960
142632024-03-06T21:03:45.673Z19/30 Read block 285 len 28672
142642024-03-06T21:03:45.673Z20/30 Read block 257 len 32768
142652024-03-06T21:03:45.673Z21/30 Write block 429 len 16384 data: 11 11 9 9
142662024-03-06T21:03:45.673Z22/30 Read block 418 len 12288
142672024-03-06T21:03:45.673Z23/30 Flush
142682024-03-06T21:03:45.673Z24/30 Write block 159 len 28672 data: 11 10 11 11 9 9 9
142692024-03-06T21:03:45.673Z25/30 Read block 175 len 12288
142702024-03-06T21:03:45.674Z26/30 Read block 534 len 40960
142712024-03-06T21:03:45.674Z27/30 Read block 241 len 28672
142722024-03-06T21:03:45.674Z28/30 Read block 544 len 16384
142732024-03-06T21:03:45.674Z29/30 Read block 534 len 8192
142742024-03-06T21:03:45.674Z30/30 Write block 302 len 24576 data: 9 9 11 12 14 14
142752024-03-06T21:03:45.679Z----------------------------------------------------------------
142762024-03-06T21:03:45.679Z Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:30
142772024-03-06T21:03:45.679ZStates: Active Active Active
142782024-03-06T21:03:45.679ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
142792024-03-06T21:03:45.680Z 1 Acked 1000 Write 6 Done Done Sent false
142802024-03-06T21:03:45.680Z 2 Acked 1001 Write 9 Done Done Sent false
142812024-03-06T21:03:45.680Z 3 Acked 1002 Write 6 Done Done Sent false
142822024-03-06T21:03:45.680Z 4 Acked 1003 Write 4 Done Done Sent false
142832024-03-06T21:03:45.680Z 5 Acked 1004 Read 7 Done Done Sent false
142842024-03-06T21:03:45.680Z 6 Acked 1005 Write 8 Done Done Sent false
142852024-03-06T21:03:45.680Z 7 Acked 1006 Read 9 Done Done Sent false
142862024-03-06T21:03:45.680Z 8 Acked 1007 Write 10 Done Done Sent false
142872024-03-06T21:03:45.680Z 9 Acked 1008 Write 6 Done Done Sent false
142882024-03-06T21:03:45.681Z 10 Acked 1009 Write 10 Done Done Sent false
142892024-03-06T21:03:45.681Z 11 Acked 1010 Write 3 Done Done Sent false
142902024-03-06T21:03:45.681Z 12 Acked 1011 Flush 0 Done Done Sent false
142912024-03-06T21:03:45.681Z 13 Acked 1012 Write 9 Done Done Sent false
142922024-03-06T21:03:45.681Z 14 Acked 1013 Write 2 Done Done Sent false
142932024-03-06T21:03:45.681Z 15 Acked 1014 Flush 0 Done Done Sent false
142942024-03-06T21:03:45.681Z 16 Acked 1015 Write 2 Done Done Sent false
142952024-03-06T21:03:45.681Z 17 Acked 1016 Write 3 Done Done Sent false
142962024-03-06T21:03:45.681Z 18 Acked 1017 Read 10 Done Done Sent false
142972024-03-06T21:03:45.681Z 19 Acked 1018 Read 7 Done Done Sent false
142982024-03-06T21:03:45.681Z 20 Acked 1019 Read 8 Done Done Sent false
142992024-03-06T21:03:45.681Z 21 Acked 1020 Write 4 Done Done Sent false
143002024-03-06T21:03:45.681Z 22 Acked 1021 Read 3 Done Done Sent false
143012024-03-06T21:03:45.681Z 23 Acked 1022 Flush 0 Done Done Sent false
143022024-03-06T21:03:45.681Z 24 Acked 1023 Write 7 Done Done Sent false
143032024-03-06T21:03:45.682Z 25 Acked 1024 Read 3 Done Done Sent false
143042024-03-06T21:03:45.682Z 26 Acked 1025 Read 10 Done Done Sent false
143052024-03-06T21:03:45.682Z 27 Acked 1026 Read 7 Done Done Sent false
143062024-03-06T21:03:45.682Z 28 Acked 1027 Read 4 Done Done Sent false
143072024-03-06T21:03:45.682Z 29 Acked 1028 Read 2 Done Done Sent false
143082024-03-06T21:03:45.682Z 30 Acked 1029 Write 6 Sent Sent Sent false
143092024-03-06T21:03:45.682Z STATES DS:0 DS:1 DS:2 TOTAL
143102024-03-06T21:03:45.682Z New 0 0 0 0
143112024-03-06T21:03:45.682Z Sent 1 1 30 32
143122024-03-06T21:03:45.682Z Done 29 29 0 58
143132024-03-06T21:03:45.682Z Skipped 0 0 0 0
143142024-03-06T21:03:45.682Z Error 0 0 0 0
143152024-03-06T21:03:45.682ZLast Flush: 1022 1022 0
143162024-03-06T21:03:45.682ZDownstairs last five completed:
143172024-03-06T21:03:45.682ZUpstairs last five completed: 30 29 28 27 26
143182024-03-06T21:03:45.683Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:03:44.416641934Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1667}
143192024-03-06T21:03:45.683ZWrote out file "/tmp/test_repair_verify.data"
143202024-03-06T21:03:45.683Z
143212024-03-06T21:03:45.699ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
143222024-03-06T21:03:45.710ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
143232024-03-06T21:03:45.710Z 0 000-019 131 131 131 228 228 228 F F F
143242024-03-06T21:03:45.710Z 1 020-039 131 131 131 228 228 228 F F F
143252024-03-06T21:03:45.710Z 2 040-059 132 132 124 229 229 215 F F F <---
143262024-03-06T21:03:45.710Z 3 060-079 130 130 130 227 227 227 F F F
143272024-03-06T21:03:45.710Z 4 080-099 127 127 127 223 223 223 F F F
143282024-03-06T21:03:45.710Z 5 100-119 129 129 129 226 226 226 F F F
143292024-03-06T21:03:45.711Z 6 120-139 129 129 129 226 226 226 F F F
143302024-03-06T21:03:45.711Z 7 140-159 132 132 131 231 231 228 T T F <---
143312024-03-06T21:03:45.711Z 8 160-179 132 132 131 230 230 228 T T F <---
143322024-03-06T21:03:45.711Z 9 180-199 132 132 130 229 229 227 F F F <---
143332024-03-06T21:03:45.711Z 10 200-219 130 130 130 227 227 227 F F F
143342024-03-06T21:03:45.711Z 11 220-239 132 132 130 229 229 227 F F F <---
143352024-03-06T21:03:45.711Z 12 240-259 132 132 130 229 229 227 F F F <---
143362024-03-06T21:03:45.711Z 13 260-279 129 129 129 226 226 226 F F F
143372024-03-06T21:03:45.711Z 14 280-299 129 129 129 226 226 226 F F F
143382024-03-06T21:03:45.711Z 15 300-319 125 125 125 217 217 217 T T F <---
143392024-03-06T21:03:45.711Z 16 320-339 132 132 127 229 229 223 F F F <---
143402024-03-06T21:03:45.711Z 17 340-359 132 132 127 229 229 223 F F F <---
143412024-03-06T21:03:45.711Z 18 360-379 130 130 130 227 227 227 F F F
143422024-03-06T21:03:45.712Z 19 380-399 128 128 128 225 225 225 F F F
143432024-03-06T21:03:45.712Z 20 400-419 127 127 127 223 223 223 F F F
143442024-03-06T21:03:45.712Z 21 420-439 132 132 131 231 231 228 F F F <---
143452024-03-06T21:03:45.712Z 22 440-459 131 131 131 228 228 228 F F F
143462024-03-06T21:03:45.712Z 23 460-479 132 132 127 231 231 223 F F F <---
143472024-03-06T21:03:45.712Z 24 480-499 129 129 129 226 226 226 F F F
143482024-03-06T21:03:45.712Z 25 500-519 129 129 129 226 226 226 F F F
143492024-03-06T21:03:45.712Z 26 520-539 132 132 129 229 229 226 F F F <---
143502024-03-06T21:03:45.712Z 27 540-559 130 130 130 227 227 227 F F F
143512024-03-06T21:03:45.712Z 28 560-579 132 132 130 229 229 227 F F F <---
143522024-03-06T21:03:45.713Z 29 580-599 130 130 130 227 227 227 F F F
143532024-03-06T21:03:45.713ZMax gen: 132, Max flush: 231
143542024-03-06T21:03:45.713ZError: Difference in extent metadata found!
143552024-03-06T21:03:45.713ZA Difference in extent metadata is expected here
143562024-03-06T21:03:45.713ZOn loop 66
143572024-03-06T21:03:45.713Z
143582024-03-06T21:03:45.718ZVerifying data now
143592024-03-06T21:03:50.399ZNo change after verify
143602024-03-06T21:03:50.400ZLoop: 66 Downstairs dump after verify (and repair):
143612024-03-06T21:03:50.416ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
143622024-03-06T21:03:50.416Z 0 000-019 131 131 131 228 228 228 F F F
143632024-03-06T21:03:50.416Z 1 020-039 131 131 131 228 228 228 F F F
143642024-03-06T21:03:50.416Z 2 040-059 132 132 132 229 229 229 F F F
143652024-03-06T21:03:50.416Z 3 060-079 130 130 130 227 227 227 F F F
143662024-03-06T21:03:50.417Z 4 080-099 127 127 127 223 223 223 F F F
143672024-03-06T21:03:50.417Z 5 100-119 129 129 129 226 226 226 F F F
143682024-03-06T21:03:50.417Z 6 120-139 129 129 129 226 226 226 F F F
143692024-03-06T21:03:50.417Z 7 140-159 133 133 133 232 232 232 F F F
143702024-03-06T21:03:50.417Z 8 160-179 133 133 133 232 232 232 F F F
143712024-03-06T21:03:50.417Z 9 180-199 132 132 132 229 229 229 F F F
143722024-03-06T21:03:50.417Z 10 200-219 130 130 130 227 227 227 F F F
143732024-03-06T21:03:50.417Z 11 220-239 132 132 132 229 229 229 F F F
143742024-03-06T21:03:50.417Z 12 240-259 132 132 132 229 229 229 F F F
143752024-03-06T21:03:50.417Z 13 260-279 129 129 129 226 226 226 F F F
143762024-03-06T21:03:50.417Z 14 280-299 129 129 129 226 226 226 F F F
143772024-03-06T21:03:50.417Z 15 300-319 133 133 133 232 232 232 F F F
143782024-03-06T21:03:50.417Z 16 320-339 132 132 132 229 229 229 F F F
143792024-03-06T21:03:50.417Z 17 340-359 132 132 132 229 229 229 F F F
143802024-03-06T21:03:50.417Z 18 360-379 130 130 130 227 227 227 F F F
143812024-03-06T21:03:50.417Z 19 380-399 128 128 128 225 225 225 F F F
143822024-03-06T21:03:50.417Z 20 400-419 127 127 127 223 223 223 F F F
143832024-03-06T21:03:50.417Z 21 420-439 132 132 132 231 231 231 F F F
143842024-03-06T21:03:50.417Z 22 440-459 131 131 131 228 228 228 F F F
143852024-03-06T21:03:50.418Z 23 460-479 132 132 132 231 231 231 F F F
143862024-03-06T21:03:50.418Z 24 480-499 129 129 129 226 226 226 F F F
143872024-03-06T21:03:50.418Z 25 500-519 129 129 129 226 226 226 F F F
143882024-03-06T21:03:50.418Z 26 520-539 132 132 132 229 229 229 F F F
143892024-03-06T21:03:50.422Z 27 540-559 130 130 130 227 227 227 F F F
143902024-03-06T21:03:50.422Z 28 560-579 132 132 132 229 229 229 F F F
143912024-03-06T21:03:50.422Z 29 580-599 130 130 130 227 227 227 F F F
143922024-03-06T21:03:50.422ZMax gen: 133, Max flush: 232
143932024-03-06T21:03:50.422Z
143942024-03-06T21:03:50.422ZBegin loop 67
143952024-03-06T21:03:50.422ZDownstairs to restart: 0
143962024-03-06T21:03:50.445Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.191056485Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675}
143972024-03-06T21:03:50.445Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.191343476Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675}
143982024-03-06T21:03:50.445Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.191376723Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675}
143992024-03-06T21:03:50.445Z{"msg":"Crucible runtime is spawned
144002024-03-06T21:03:50.452ZCrucible 49f4d3a1-1482-4677-96b2-83f31f3fb32d has session id: 048ebcf8-19c8-4513-b31a-ad913f0fee5b","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.19142683Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144012024-03-06T21:03:50.452Z{"msg":"Upstairs opts: Upstairs UUID: 49f4d3a1-1482-4677-96b2-83f31f3fb32d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.191476647Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144022024-03-06T21:03:50.452Z{"msg":"Crucible stats registered with UUID: 49f4d3a1-1482-4677-96b2-83f31f3fb32d","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.191525803Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144032024-03-06T21:03:50.452Z{"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.191580809Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675}
144042024-03-06T21:03:50.452Z{"msg":"49f4d3a1-1482-4677-96b2-83f31f3fb32d active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.191656994Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144052024-03-06T21:03:50.452Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.191685602Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"client":"0","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144062024-03-06T21:03:50.452Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.191704711Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"client":"0","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144072024-03-06T21:03:50.452Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.191738338Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"client":"1","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144082024-03-06T21:03:50.452Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.191782426Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"client":"1","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144092024-03-06T21:03:50.452Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.191816443Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"client":"2","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144102024-03-06T21:03:50.452Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.19186192Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"client":"2","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144112024-03-06T21:03:50.452Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.191889348Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144122024-03-06T21:03:50.452Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.191933645Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144132024-03-06T21:03:50.452Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.191976512Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144142024-03-06T21:03:50.452Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.192018069Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144152024-03-06T21:03:50.452Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.192045947Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144162024-03-06T21:03:50.453Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.192213846Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144172024-03-06T21:03:50.453Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.192927707Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"client":"1","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144182024-03-06T21:03:50.453Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.192965754Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"client":"1","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144192024-03-06T21:03:50.453Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.193061647Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"client":"2","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144202024-03-06T21:03:50.453Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.193117254Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"client":"2","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144212024-03-06T21:03:50.453Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.193571432Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"client":"2","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144222024-03-06T21:03:50.453Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.193613439Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"client":"1","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144232024-03-06T21:03:50.453Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.194191489Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"client":"2","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144242024-03-06T21:03:50.453Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.194235166Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"client":"2","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144252024-03-06T21:03:50.453Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.194261085Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144262024-03-06T21:03:50.453Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.194284623Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"client":"1","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144272024-03-06T21:03:50.453Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.194298802Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"client":"1","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144282024-03-06T21:03:50.453Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.194319111Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144292024-03-06T21:03:50.453Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.194580723Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"client":"0","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144302024-03-06T21:03:50.453Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.19461182Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"client":"0","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144312024-03-06T21:03:50.453Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.195115626Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"client":"0","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144322024-03-06T21:03:50.453Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.195523388Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"client":"0","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144332024-03-06T21:03:50.453Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.195568934Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"client":"0","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144342024-03-06T21:03:50.453Z{"msg":"[0]R flush_numbers[0..12]: [228, 228, 229, 227, 223, 226, 226, 232, 232, 229, 227, 229]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.195608382Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"":"downstairs","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144352024-03-06T21:03:50.454Z{"msg":"[0]R generation[0..12]: [131, 131, 132, 130, 127, 129, 129, 133, 133, 132, 130, 132]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.195643839Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"":"downstairs","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144362024-03-06T21:03:50.454Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.195677697Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"":"downstairs","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144372024-03-06T21:03:50.454Z{"msg":"[1]R flush_numbers[0..12]: [228, 228, 229, 227, 223, 226, 226, 232, 232, 229, 227, 229]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.195710865Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"":"downstairs","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144382024-03-06T21:03:50.454Z{"msg":"[1]R generation[0..12]: [131, 131, 132, 130, 127, 129, 129, 133, 133, 132, 130, 132]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
144392024-03-06T21:03:50.454Z,"time":"2024-03-06T21:03:49.195731293Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"":"downstairs","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144402024-03-06T21:03:50.454Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.195761201Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"":"downstairs","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144412024-03-06T21:03:50.454Z{"msg":"[2]R flush_numbers[0..12]: [228, 228, 229, 227, 223, 226, 226, 232, 232, 229, 227, 229]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.195809048Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"":"downstairs","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144422024-03-06T21:03:50.454Z{"msg":"[2]R generation[0..12]: [131, 131, 132, 130, 127, 129, 129, 133, 133, 132, 130, 132]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.195832386Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"":"downstairs","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144432024-03-06T21:03:50.454Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.195864294Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"":"downstairs","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144442024-03-06T21:03:50.454Z{"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.195887013Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"":"downstairs","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144452024-03-06T21:03:50.454Z{"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.195910371Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"":"downstairs","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144462024-03-06T21:03:50.454Z{"msg":"Next flush: 233","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.195941669Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"":"downstairs","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144472024-03-06T21:03:50.454Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.195974706Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"":"downstairs","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144482024-03-06T21:03:50.454Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.195997345Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144492024-03-06T21:03:50.454Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.196018943Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"client":"0","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144502024-03-06T21:03:50.454Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.196041382Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"client":"1","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144512024-03-06T21:03:50.454Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.19606607Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"client":"2","":"downstairs","session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144522024-03-06T21:03:50.454Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.196093269Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144532024-03-06T21:03:50.455Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.196134906Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144542024-03-06T21:03:50.455Z{"msg":"49f4d3a1-1482-4677-96b2-83f31f3fb32d is now active with session: 048ebcf8-19c8-4513-b31a-ad913f0fee5b","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.196161544Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144552024-03-06T21:03:50.455Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.196192391Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675,"session_id":"048ebcf8-19c8-4513-b31a-ad913f0fee5b"}
144562024-03-06T21:03:50.455Z{"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:49.19621372Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675}
144572024-03-06T21:03:50.455ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
144582024-03-06T21:03:50.460ZLoading write count information from file "/tmp/test_repair_verify.data"
144592024-03-06T21:03:50.460ZRun Repair workload
144602024-03-06T21:03:50.460Z01/30 Flush
144612024-03-06T21:03:50.460Z02/30 Write block 574 len 36864 data: 7 9 11 8 6 7 7 10 9
144622024-03-06T21:03:50.460Z03/30 Read block 57 len 12288
144632024-03-06T21:03:50.460Z04/30 Write block 414 len 20480 data: 9 8 10 11 11
144642024-03-06T21:03:50.460Z05/30 Write block 29 len 4096 data: 11
144652024-03-06T21:03:50.460Z06/30 Write block 307 len 20480 data: 15 14 13 12 13
144662024-03-06T21:03:50.460Z07/30 Write block 401 len 40960 data: 6 6 8 7 10 10 13 14 12 13
144672024-03-06T21:03:50.460Z08/30 Read block 536 len 16384
144682024-03-06T21:03:50.460Z09/30 Read block 86 len 16384
144692024-03-06T21:03:50.460Z10/30 Write block 261 len 12288 data: 7 6 9
144702024-03-06T21:03:50.464Z11/30 Write block 240 len 20480 data: 14 14 14 12 11
144712024-03-06T21:03:50.464Z12/30 Read block 371 len 24576
144722024-03-06T21:03:50.464Z13/30 Write block 462 len 24576 data: 13 16 16 15 11 10
144732024-03-06T21:03:50.464Z14/30 Write block 87 len 4096 data: 7
144742024-03-06T21:03:50.464Z15/30 Read block 98 len 28672
144752024-03-06T21:03:50.468Z16/30 Write block 331 len 16384 data: 6 5 7 7
144762024-03-06T21:03:50.468Z17/30 Read block 100 len 8192
144772024-03-06T21:03:50.468Z18/30 Write block 149 len 12288 data: 11 12 11
144782024-03-06T21:03:50.468Z19/30 Flush
144792024-03-06T21:03:50.471Z20/30 Write block 379 len 12288 data: 14 14 13
144802024-03-06T21:03:50.475Z21/30 Read block 411 len 24576
144812024-03-06T21:03:50.475Z22/30 Read block 436 len 16384
144822024-03-06T21:03:50.475Z23/30 Write block 256 len 12288 data: 5 5 6
144832024-03-06T21:03:50.479Z24/30 Write block 7 len 36864 data: 11 9 9 8 12 11 12 11 13
144842024-03-06T21:03:50.479Z25/30 Read block 240 len 4096
144852024-03-06T21:03:50.479Z26/30 Write block 487 len 16384 data: 10 10 11 8
144862024-03-06T21:03:50.479Z27/30 Read block 576 len 40960
144872024-03-06T21:03:50.483Z28/30 Read block 356 len 40960
144882024-03-06T21:03:50.484Z29/30 Read block 49 len 32768
144892024-03-06T21:03:50.484Z30/30 Write block 559 len 28672 data: 7 11 11 12 11 12 11
144902024-03-06T21:03:50.484Z----------------------------------------------------------------
144912024-03-06T21:03:50.484Z Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30
144922024-03-06T21:03:50.484ZStates: Active Active Active
144932024-03-06T21:03:50.484ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
144942024-03-06T21:03:50.484Z 1 Acked 1000 Flush 0 Sent Done Done false
144952024-03-06T21:03:50.484Z 2 Acked 1001 Write 9 Sent Done Done false
144962024-03-06T21:03:50.484Z 3 Acked 1002 Read 3 Sent Done Done false
144972024-03-06T21:03:50.485Z 4 Acked 1003 Write 5 Sent Done Done false
144982024-03-06T21:03:50.485Z 5 Acked 1004 Write 1 Sent Done Done false
144992024-03-06T21:03:50.485Z 6 Acked 1005 Write 5 Sent Done Done false
145002024-03-06T21:03:50.485Z 7 Acked 1006 Write 10 Sent Done Done false
145012024-03-06T21:03:50.485Z 8 Acked 1007 Read 4 Sent Done Done false
145022024-03-06T21:03:50.485Z 9 Acked 1008 Read 4 Sent Done Done false
145032024-03-06T21:03:50.485Z 10 Acked 1009 Write 3 Sent Done Done false
145042024-03-06T21:03:50.485Z 11 Acked 1010 Write 5 Sent Done Done false
145052024-03-06T21:03:50.485Z 12 Acked 1011 Read 6 Sent Done Done false
145062024-03-06T21:03:50.485Z 13 Acked 1012 Write 6 Sent Done Done false
145072024-03-06T21:03:50.485Z 14 Acked 1013 Write 1 Sent Done Done false
145082024-03-06T21:03:50.486Z 15 Acked 1014 Read 7 Sent Done Done false
145092024-03-06T21:03:50.486Z 16 Acked 1015 Write 4 Sent Done Done false
145102024-03-06T21:03:50.486Z 17 Acked 1016 Read 2 Sent Done Done false
145112024-03-06T21:03:50.486Z 18 Acked 1017 Write 3 Sent Done Done false
145122024-03-06T21:03:50.486Z 19 Acked 1018 Flush 0 Sent Done Done false
145132024-03-06T21:03:50.486Z 20 Acked 1019 Write 3 Sent Done Done false
145142024-03-06T21:03:50.486Z 21 Acked 1020 Read 6 Sent Done Done false
145152024-03-06T21:03:50.487Z 22 Acked 1021 Read 4 Sent Done Done false
145162024-03-06T21:03:50.487Z 23 Acked 1022 Write 3 Sent Done Done false
145172024-03-06T21:03:50.488Z 24 Acked 1023 Write 9 Sent Done Done false
145182024-03-06T21:03:50.488Z 25 Acked 1024 Read 1 Sent Done Done false
145192024-03-06T21:03:50.488Z 26 Acked 1025 Write 4 Sent Done Done false
145202024-03-06T21:03:50.488Z 27 Acked 1026 Read 10 Sent Done Done false
145212024-03-06T21:03:50.488Z 28 Acked 1027 Read 10 Sent Done Done false
145222024-03-06T21:03:50.488Z 29 Acked 1028 Read 8 Sent Done Done false
145232024-03-06T21:03:50.488Z 30 Acked 1029 Write 7 Sent Sent Sent false
145242024-03-06T21:03:50.488Z STATES DS:0 DS:1 DS:2 TOTAL
145252024-03-06T21:03:50.488Z New 0 0 0 0
145262024-03-06T21:03:50.488Z Sent 30 1 1 32
145272024-03-06T21:03:50.488Z Done 0 29 29 58
145282024-03-06T21:03:50.489Z Skipped 0 0 0 0
145292024-03-06T21:03:50.489Z Error 0 0 0 0
145302024-03-06T21:03:50.489ZLast Flush: 0 1018 1018
145312024-03-06T21:03:50.489ZDownstairs last five completed:
145322024-03-06T21:03:50.489ZUpstairs last five completed: 30 29 28 27 26
145332024-03-06T21:03:50.506Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:03:49.230373856Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1675}
145342024-03-06T21:03:50.507ZWrote out file "/tmp/test_repair_verify.data"
145352024-03-06T21:03:50.507Z
145362024-03-06T21:03:50.507ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
145372024-03-06T21:03:50.512ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
145382024-03-06T21:03:50.513Z 0 000-019 131 131 131 228 228 228 F T T <---
145392024-03-06T21:03:50.513Z 1 020-039 131 134 134 228 234 234 F F F <---
145402024-03-06T21:03:50.513Z 2 040-059 132 132 132 229 229 229 F F F
145412024-03-06T21:03:50.513Z 3 060-079 130 130 130 227 227 227 F F F
145422024-03-06T21:03:50.513Z 4 080-099 127 134 134 223 234 234 F F F <---
145432024-03-06T21:03:50.513Z 5 100-119 129 129 129 226 226 226 F F F
145442024-03-06T21:03:50.513Z 6 120-139 129 129 129 226 226 226 F F F
145452024-03-06T21:03:50.513Z 7 140-159 133 134 134 232 234 234 F F F <---
145462024-03-06T21:03:50.513Z 8 160-179 133 133 133 232 232 232 F F F
145472024-03-06T21:03:50.513Z 9 180-199 132 132 132 229 229 229 F F F
145482024-03-06T21:03:50.513Z 10 200-219 130 130 130 227 227 227 F F F
145492024-03-06T21:03:50.513Z 11 220-239 132 132 132 229 229 229 F F F
145502024-03-06T21:03:50.514Z 12 240-259 132 134 134 229 234 234 F T T <---
145512024-03-06T21:03:50.514Z 13 260-279 129 134 134 226 234 234 F F F <---
145522024-03-06T21:03:50.514Z 14 280-299 129 129 129 226 226 226 F F F
145532024-03-06T21:03:50.514Z 15 300-319 133 134 134 232 234 234 F F F <---
145542024-03-06T21:03:50.514Z 16 320-339 132 134 134 229 234 234 F F F <---
145552024-03-06T21:03:50.514Z 17 340-359 132 132 132 229 229 229 F F F
145562024-03-06T21:03:50.519Z 18 360-379 130 130 130 227 227 227 F T T <---
145572024-03-06T21:03:50.519Z 19 380-399 128 128 128 225 225 225 F T T <---
145582024-03-06T21:03:50.519Z 20 400-419 127 134 134 223 234 234 F F F <---
145592024-03-06T21:03:50.519Z 21 420-439 132 132 132 231 231 231 F F F
145602024-03-06T21:03:50.519Z 22 440-459 131 131 131 228 228 228 F F F
145612024-03-06T21:03:50.519Z 23 460-479 132 134 134 231 234 234 F F F <---
145622024-03-06T21:03:50.519Z 24 480-499 129 129 129 226 226 226 F T T <---
145632024-03-06T21:03:50.519Z 25 500-519 129 129 129 226 226 226 F F F
145642024-03-06T21:03:50.519Z 26 520-539 132 132 132 229 229 229 F F F
145652024-03-06T21:03:50.519Z 27 540-559 130 130 130 227 227 227 F T T <---
145662024-03-06T21:03:50.519Z 28 560-579 132 134 134 229 234 234 F T T <---
145672024-03-06T21:03:50.519Z 29 580-599 130 134 134 227 234 234 F F F <---
145682024-03-06T21:03:50.519ZMax gen: 134, Max flush: 234
145692024-03-06T21:03:50.519ZError: Difference in extent metadata found!
145702024-03-06T21:03:50.519ZA Difference in extent metadata is expected here
145712024-03-06T21:03:50.519ZOn loop 67
145722024-03-06T21:03:50.520Z
145732024-03-06T21:03:50.525ZVerifying data now
145742024-03-06T21:03:55.327ZNo change after verify
145752024-03-06T21:03:55.327ZLoop: 67 Downstairs dump after verify (and repair):
145762024-03-06T21:03:55.342ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
145772024-03-06T21:03:55.342Z 0 000-019 135 135 135 235 235 235 F F F
145782024-03-06T21:03:55.342Z 1 020-039 134 134 134 234 234 234 F F F
145792024-03-06T21:03:55.342Z 2 040-059 132 132 132 229 229 229 F F F
145802024-03-06T21:03:55.342Z 3 060-079 130 130 130 227 227 227 F F F
145812024-03-06T21:03:55.342Z 4 080-099 134 134 134 234 234 234 F F F
145822024-03-06T21:03:55.343Z 5 100-119 129 129 129 226 226 226 F F F
145832024-03-06T21:03:55.343Z 6 120-139 129 129 129 226 226 226 F F F
145842024-03-06T21:03:55.343Z 7 140-159 134 134 134 234 234 234 F F F
145852024-03-06T21:03:55.343Z 8 160-179 133 133 133 232 232 232 F F F
145862024-03-06T21:03:55.343Z 9 180-199 132 132 132 229 229 229 F F F
145872024-03-06T21:03:55.343Z 10 200-219 130 130 130 227 227 227 F F F
145882024-03-06T21:03:55.343Z 11 220-239 132 132 132 229 229 229 F F F
145892024-03-06T21:03:55.343Z 12 240-259 135 135 135 235 235 235 F F F
145902024-03-06T21:03:55.343Z 13 260-279 134 134 134 234 234 234 F F F
145912024-03-06T21:03:55.343Z 14 280-299 129 129 129 226 226 226 F F F
145922024-03-06T21:03:55.343Z 15 300-319 134 134 134 234 234 234 F F F
145932024-03-06T21:03:55.343Z 16 320-339 134 134 134 234 234 234 F F F
145942024-03-06T21:03:55.343Z 17 340-359 132 132 132 229 229 229 F F F
145952024-03-06T21:03:55.343Z 18 360-379 135 135 135 235 235 235 F F F
145962024-03-06T21:03:55.343Z 19 380-399 135 135 135 235 235 235 F F F
145972024-03-06T21:03:55.346Z 20 400-419 134 134 134 234 234 234 F F F
145982024-03-06T21:03:55.347Z 21 420-439 132 132 132 231 231 231 F F F
145992024-03-06T21:03:55.347Z 22 440-459 131 131 131 228 228 228 F F F
146002024-03-06T21:03:55.347Z 23 460-479 134 134 134 234 234 234 F F F
146012024-03-06T21:03:55.347Z 24 480-499 135 135 135 235 235 235 F F F
146022024-03-06T21:03:55.347Z 25 500-519 129 129 129 226 226 226 F F F
146032024-03-06T21:03:55.347Z 26 520-539 132 132 132 229 229 229 F F F
146042024-03-06T21:03:55.347Z 27 540-559 135 135 135 235 235 235 F F F
146052024-03-06T21:03:55.347Z 28 560-579 135 135 135 235 235 235 F F F
146062024-03-06T21:03:55.347Z 29 580-599 134 134 134 234 234 234 F F F
146072024-03-06T21:03:55.347ZMax gen: 135, Max flush: 235
146082024-03-06T21:03:55.347Z
146092024-03-06T21:03:55.347ZBegin loop 68
146102024-03-06T21:03:55.347ZDownstairs to restart: 1
146112024-03-06T21:03:55.368Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.115448451Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683}
146122024-03-06T21:03:55.368Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.11575013Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683}
146132024-03-06T21:03:55.369Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
146142024-03-06T21:03:55.369Z,"time":"2024-03-06T21:03:54.115783548Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683}
146152024-03-06T21:03:55.376Z{"msg":"Crucible 614dbdf7-e8e5-4769-bb6c-0720a6299ee6 has session id: 98a21f17-1ea8-4f93-a7c8-e5371512b9b4","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.115832175Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146162024-03-06T21:03:55.376Z{"msg":"Upstairs opts: Upstairs UUID: 614dbdf7-e8e5-4769-bb6c-0720a6299ee6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.115883991Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146172024-03-06T21:03:55.376Z{"msg":"Crucible stats registered with UUID: 614dbdf7-e8e5-4769-bb6c-0720a6299ee6","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.115915279Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146182024-03-06T21:03:55.376Z{"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.115951966Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683}
146192024-03-06T21:03:55.376Z{"msg":"614dbdf7-e8e5-4769-bb6c-0720a6299ee6 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.115977865Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146202024-03-06T21:03:55.376Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.116013072Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"client":"0","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146212024-03-06T21:03:55.376Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.116061119Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"client":"0","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146222024-03-06T21:03:55.376Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.116100596Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"client":"1","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146232024-03-06T21:03:55.376Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.116146543Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"client":"1","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146242024-03-06T21:03:55.377Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.11618792Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"client":"2","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146252024-03-06T21:03:55.377Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.116232447Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"client":"2","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146262024-03-06T21:03:55.377Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.116277254Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146272024-03-06T21:03:55.377Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.11633383Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146282024-03-06T21:03:55.377Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.116369298Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146292024-03-06T21:03:55.377Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.116437383Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146302024-03-06T21:03:55.377Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.11647751Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146312024-03-06T21:03:55.377Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.116501518Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146322024-03-06T21:03:55.377Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.117259846Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"client":"0","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146332024-03-06T21:03:55.377Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.117337731Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"client":"0","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146342024-03-06T21:03:55.377Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.117379668Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"client":"2","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146352024-03-06T21:03:55.377Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.117398777Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"client":"2","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146362024-03-06T21:03:55.377Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.117981307Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"client":"2","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146372024-03-06T21:03:55.377Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.118202271Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"client":"0","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146382024-03-06T21:03:55.377Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.11850836Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"client":"1","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146392024-03-06T21:03:55.377Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.118552647Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"client":"1","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146402024-03-06T21:03:55.377Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.118619212Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"client":"2","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146412024-03-06T21:03:55.377Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.118674259Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"client":"2","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146422024-03-06T21:03:55.377Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.118701787Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146432024-03-06T21:03:55.378Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.11879358Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"client":"0","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146442024-03-06T21:03:55.378Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.118831618Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"client":"0","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146452024-03-06T21:03:55.378Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.118847867Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146462024-03-06T21:03:55.378Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.119212962Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"client":"1","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146472024-03-06T21:03:55.378Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.11966516Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"client":"1","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146482024-03-06T21:03:55.378Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.119730666Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"client":"1","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146492024-03-06T21:03:55.378Z{"msg":"[0]R flush_numbers[0..12]: [235, 234, 229, 227, 234, 226, 226, 234, 232, 229, 227, 229]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.119769053Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"":"downstairs","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146502024-03-06T21:03:55.378Z{"msg":"[0]R generation[0..12]: [135, 134, 132, 130, 134, 129, 129, 134, 133, 132, 130, 132]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.119785132Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"":"downstairs","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146512024-03-06T21:03:55.378Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.119806851Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"":"downstairs","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146522024-03-06T21:03:55.378Z{"msg":"[1]R flush_numbers[0..12]: [235, 234, 229, 227, 234, 226, 226, 234, 232, 229, 227, 229]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.11982201Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"":"downstairs","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146532024-03-06T21:03:55.378Z{"msg":"[1]R generation[0..12]: [135, 134, 132, 130, 134, 129, 129, 134, 133, 132, 130, 132]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.119848038Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"":"downstairs","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146542024-03-06T21:03:55.378Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.119862467Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"":"downstairs","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146552024-03-06T21:03:55.378Z{"msg":"[2]R flush_numbers[0..12]: [235, 234, 229, 227, 234, 226, 226, 234, 232, 229, 227, 229]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.119877646Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO
146562024-03-06T21:03:55.378Zdownstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146572024-03-06T21:03:55.378Z{"msg":"[2]R generation[0..12]: [135, 134, 132, 130, 134, 129, 129, 134, 133, 132, 130, 132]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.119895075Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"":"downstairs","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146582024-03-06T21:03:55.378Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.119910294Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"":"downstairs","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146592024-03-06T21:03:55.378Z{"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.119931782Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"":"downstairs","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146602024-03-06T21:03:55.378Z{"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.119945561Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"":"downstairs","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146612024-03-06T21:03:55.378Z{"msg":"Next flush: 236","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.11995869Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"":"downstairs","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146622024-03-06T21:03:55.378Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.119973159Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"":"downstairs","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146632024-03-06T21:03:55.379Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.119993498Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146642024-03-06T21:03:55.379Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.120005897Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"client":"0","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146652024-03-06T21:03:55.379Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.120019446Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"client":"1","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146662024-03-06T21:03:55.379Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.120039685Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"client":"2","":"downstairs","session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146672024-03-06T21:03:55.379Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.120053434Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146682024-03-06T21:03:55.379Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.120066033Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146692024-03-06T21:03:55.379Z{"msg":"614dbdf7-e8e5-4769-bb6c-0720a6299ee6 is now active with session: 98a21f17-1ea8-4f93-a7c8-e5371512b9b4","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.120089521Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146702024-03-06T21:03:55.379Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.12010978Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683,"session_id":"98a21f17-1ea8-4f93-a7c8-e5371512b9b4"}
146712024-03-06T21:03:55.379Z{"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:54.120139498Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683}
146722024-03-06T21:03:55.389ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
146732024-03-06T21:03:55.389ZLoading write count information from file "/tmp/test_repair_verify.data"
146742024-03-06T21:03:55.389ZRun Repair workload
146752024-03-06T21:03:55.389Z01/30 Write block 565 len 40960 data: 12 11 7 5 5 7 8 8 9 8
146762024-03-06T21:03:55.389Z02/30 Write block 416 len 4096 data: 11
146772024-03-06T21:03:55.389Z03/30 Read block 464 len 8192
146782024-03-06T21:03:55.389Z04/30 Write block 116 len 28672 data: 5 7 8 9 8 8 8
146792024-03-06T21:03:55.389Z05/30 Write block 282 len 24576 data: 11 9 11 13 14 14
146802024-03-06T21:03:55.389Z06/30 Read block 180 len 32768
146812024-03-06T21:03:55.389Z07/30 Read block 121 len 36864
146822024-03-06T21:03:55.393Z08/30 Flush
146832024-03-06T21:03:55.393Z09/30 Write block 207 len 40960 data: 7 10 9 11 8 9 8 9 12 14
146842024-03-06T21:03:55.393Z10/30 Read block 279 len 36864
146852024-03-06T21:03:55.393Z11/30 Read block 333 len 8192
146862024-03-06T21:03:55.393Z12/30 Read block 403 len 16384
146872024-03-06T21:03:55.393Z13/30 Read block 4 len 16384
146882024-03-06T21:03:55.393Z14/30 Read block 316 len 24576
146892024-03-06T21:03:55.393Z15/30 Flush
146902024-03-06T21:03:55.397Z16/30 Write block 63 len 28672 data: 21 18 18 17 16 14 12
146912024-03-06T21:03:55.397Z17/30 Read block 235 len 12288
146922024-03-06T21:03:55.404Z18/30 Read block 178 len 32768
146932024-03-06T21:03:55.404Z19/30 Write block 125 len 32768 data: 13 12 14 13 13 10 8 8
146942024-03-06T21:03:55.404Z20/30 Read block 435 len 20480
146952024-03-06T21:03:55.407Z21/30 Read block 227 len 12288
146962024-03-06T21:03:55.407Z22/30 Read block 461 len 36864
146972024-03-06T21:03:55.407Z23/30 Read block 492 len 32768
146982024-03-06T21:03:55.407Z24/30 Read block 21 len 40960
146992024-03-06T21:03:55.407Z25/30 Read block 397 len 16384
147002024-03-06T21:03:55.408Z26/30 Read block 247 len 8192
147012024-03-06T21:03:55.408Z27/30 Write block 301 len 36864 data: 7 10 10 12 13 15 16 15 14
147022024-03-06T21:03:55.408Z28/30 Read block 273 len 32768
147032024-03-06T21:03:55.412Z29/30 Read block 404 len 40960
147042024-03-06T21:03:55.412Z30/30 Read block 97 len 20480
147052024-03-06T21:03:55.412Z----------------------------------------------------------------
147062024-03-06T21:03:55.412Z Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:22
147072024-03-06T21:03:55.412ZStates: Active Active Active
147082024-03-06T21:03:55.412ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
147092024-03-06T21:03:55.412Z 9 Acked 1008 Write 10 Done Sent Done false
147102024-03-06T21:03:55.412Z 10 Acked 1009 Read 9 Done Sent Done false
147112024-03-06T21:03:55.412Z 11 Acked 1010 Read 2 Done Sent Done false
147122024-03-06T21:03:55.412Z 12 Acked 1011 Read 4 Done Sent Done false
147132024-03-06T21:03:55.412Z 13 Acked 1012 Read 4 Done Sent Done false
147142024-03-06T21:03:55.412Z 14 Acked 1013 Read 6 Done Sent Done false
147152024-03-06T21:03:55.412Z 15 Acked 1014 Flush 0 Done Sent Done false
147162024-03-06T21:03:55.412Z 16 Acked 1015 Write 7 Done Sent Done false
147172024-03-06T21:03:55.412Z 17 Acked 1016 Read 3 Done Sent Done false
147182024-03-06T21:03:55.412Z 18 Acked 1017 Read 8 Done Sent Done false
147192024-03-06T21:03:55.412Z 19 Acked 1018 Write 8 Done Sent Done false
147202024-03-06T21:03:55.413Z 20 Acked 1019 Read 5 Done Sent Done false
147212024-03-06T21:03:55.413Z 21 Acked 1020 Read 3 Done Sent Done false
147222024-03-06T21:03:55.413Z 22 Acked 1021 Read 9 Done Sent Done false
147232024-03-06T21:03:55.413Z 23 Acked 1022 Read 8 Done Sent Done false
147242024-03-06T21:03:55.413Z 24 Acked 1023 Read 10 Done Sent Done false
147252024-03-06T21:03:55.413Z 25 Acked 1024 Read 4 Done Sent Done false
147262024-03-06T21:03:55.413Z 26 Acked 1025 Read 2 Done Sent Done false
147272024-03-06T21:03:55.413Z 27 Acked 1026 Write 9 Done Sent Done false
147282024-03-06T21:03:55.413Z 28 Acked 1027 Read 8 Done Sent Done false
147292024-03-06T21:03:55.413Z 29 Acked 1028 Read 10 Done Sent Done false
147302024-03-06T21:03:55.413Z 30 Acked 1029 Read 5 Done Sent Sent false
147312024-03-06T21:03:55.413Z STATES DS:0 DS:1 DS:2 TOTAL
147322024-03-06T21:03:55.413Z New 0 0 0 0
147332024-03-06T21:03:55.413Z Sent 0 22 1 23
147342024-03-06T21:03:55.413Z Done 22 0 21 43
147352024-03-06T21:03:55.413Z Skipped 0 0 0 0
147362024-03-06T21:03:55.413Z Error 0 0 0 0
147372024-03-06T21:03:55.413ZLast Flush: 1014 1007 1014
147382024-03-06T21:03:55.413ZDownstairs last five completed: 1007 1006 1005 1004 1003
147392024-03-06T21:03:55.413ZUpstairs last five completed: 30 29 28 27 26
147402024-03-06T21:03:55.414Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:03:54.158315697Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1683}
147412024-03-06T21:03:55.414ZWrote out file "/tmp/test_repair_verify.data"
147422024-03-06T21:03:55.417Z
147432024-03-06T21:03:55.424ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
147442024-03-06T21:03:55.444ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
147452024-03-06T21:03:55.444Z 0 000-019 135 135 135 235 235 235 F F F
147462024-03-06T21:03:55.444Z 1 020-039 134 134 134 234 234 234 F F F
147472024-03-06T21:03:55.444Z 2 040-059 132 132 132 229 229 229 F F F
147482024-03-06T21:03:55.444Z 3 060-079 130 130 130 227 227 227 T F T <---
147492024-03-06T21:03:55.444Z 4 080-099 134 134 134 234 234 234 F F F
147502024-03-06T21:03:55.445Z 5 100-119 136 136 136 236 236 236 F F F
147512024-03-06T21:03:55.445Z 6 120-139 136 136 136 236 236 236 T F T <---
147522024-03-06T21:03:55.445Z 7 140-159 134 134 134 234 234 234 F F F
147532024-03-06T21:03:55.445Z 8 160-179 133 133 133 232 232 232 F F F
147542024-03-06T21:03:55.445Z 9 180-199 132 132 132 229 229 229 F F F
147552024-03-06T21:03:55.445Z 10 200-219 136 130 136 237 227 237 F F F <---
147562024-03-06T21:03:55.445Z 11 220-239 132 132 132 229 229 229 F F F
147572024-03-06T21:03:55.874Z 12 240-259 135 135 135 235 235 235 F F F
147582024-03-06T21:03:55.875Z 13 260-279 134 134 134 234 234 234 F F F
147592024-03-06T21:03:55.875Z 14 280-299 136 136 136 236 236 236 F F F
147602024-03-06T21:03:55.875Z 15 300-319 134 134 134 234 234 234 T F T <---
147612024-03-06T21:03:55.875Z 16 320-339 134 134 134 234 234 234 F F F
147622024-03-06T21:03:55.875Z 17 340-359 132 132 132 229 229 229 F F F
147632024-03-06T21:03:55.875Z 18 360-379 135 135 135 235 235 235 F F F
147642024-03-06T21:03:55.875Z 19 380-399 135 135 135 235 235 235 F F F
147652024-03-06T21:03:55.875Z 20 400-419 136 136 136 236 236 236 F F F
147662024-03-06T21:03:55.875Z 21 420-439 132 132 132 231 231 231 F F F
147672024-03-06T21:03:55.875Z 22 440-459 131 131 131 228 228 228 F F F
147682024-03-06T21:03:55.875Z 23 460-479 134 134 134 234 234 234 F F F
147692024-03-06T21:03:55.875Z 24 480-499 135 135 135 235 235 235 F F F
147702024-03-06T21:03:55.875Z 25 500-519 129 129 129 226 226 226 F F F
147712024-03-06T21:03:55.875Z 26 520-539 132 132 132 229 229 229 F F F
147722024-03-06T21:03:55.875Z 27 540-559 135 135 135 235 235 235 F F F
147732024-03-06T21:03:55.875Z 28 560-579 136 136 136 236 236 236 F F F
147742024-03-06T21:03:55.875Z 29 580-599 134 134 134 234 234 234 F F F
147752024-03-06T21:03:55.876ZMax gen: 136, Max flush: 237
147762024-03-06T21:03:55.876ZError: Difference in extent metadata found!
147772024-03-06T21:03:55.876ZA Difference in extent metadata is expected here
147782024-03-06T21:03:55.876ZOn loop 68
147792024-03-06T21:03:55.876Z
147802024-03-06T21:03:55.876ZVerifying data now
147812024-03-06T21:03:59.732ZNo change after verify
147822024-03-06T21:03:59.732ZLoop: 68 Downstairs dump after verify (and repair):
147832024-03-06T21:03:59.747ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
147842024-03-06T21:03:59.747Z 0 000-019 135 135 135 235 235 235 F F F
147852024-03-06T21:03:59.747Z 1 020-039 134 134 134 234 234 234 F F F
147862024-03-06T21:03:59.747Z 2 040-059 132 132 132 229 229 229 F F F
147872024-03-06T21:03:59.747Z 3 060-079 137 137 137 238 238 238 F F F
147882024-03-06T21:03:59.747Z 4 080-099 134 134 134 234 234 234 F F F
147892024-03-06T21:03:59.747Z 5 100-119 136 136 136 236 236 236 F F F
147902024-03-06T21:03:59.748Z 6 120-139 137 137 137 238 238 238 F F F
147912024-03-06T21:03:59.748Z 7 140-159 134 134 134 234 234 234 F F F
147922024-03-06T21:03:59.748Z 8 160-179 133 133 133 232 232 232 F F F
147932024-03-06T21:03:59.748Z 9 180-199 132 132 132 229 229 229 F F F
147942024-03-06T21:03:59.748Z 10 200-219 136 136 136 237 237 237 F F F
147952024-03-06T21:03:59.748Z 11 220-239 132 132 132 229 229 229 F F F
147962024-03-06T21:03:59.751Z 12 240-259 135 135 135 235 235 235 F F F
147972024-03-06T21:03:59.751Z 13 260-279 134 134 134 234 234 234 F F F
147982024-03-06T21:03:59.752Z 14 280-299 136 136 136 236 236 236 F F F
147992024-03-06T21:03:59.752Z 15 300-319 137 137 137 238 238 238 F F F
148002024-03-06T21:03:59.752Z 16 320-339 134 134 134 234 234 234 F F F
148012024-03-06T21:03:59.752Z 17 340-359 132 132 132 229 229 229 F F F
148022024-03-06T21:03:59.752Z 18 360-379 135 135 135 235 235 235 F F F
148032024-03-06T21:03:59.752Z 19 380-399 135 135 135 235 235 235 F F F
148042024-03-06T21:03:59.752Z 20 400-419 136 136 136 236 236 236 F F F
148052024-03-06T21:03:59.752Z 21 420-439 132 132 132 231 231 231 F F F
148062024-03-06T21:03:59.752Z 22 440-459 131 131 131 228 228 228 F F F
148072024-03-06T21:03:59.752Z 23 460-479 134 134 134 234 234 234 F F F
148082024-03-06T21:03:59.752Z 24 480-499 135 135 135 235 235 235 F F F
148092024-03-06T21:03:59.752Z 25 500-519 129 129 129 226 226 226 F F F
148102024-03-06T21:03:59.752Z 26 520-539 132 132 132 229 229 229 F F F
148112024-03-06T21:03:59.753Z 27 540-559 135 135 135 235 235 235 F F F
148122024-03-06T21:03:59.753Z 28 560-579 136 136 136 236 236 236 F F F
148132024-03-06T21:03:59.753Z 29 580-599 134 134 134 234 234 234 F F F
148142024-03-06T21:03:59.753ZMax gen: 137, Max flush: 238
148152024-03-06T21:03:59.760Z
148162024-03-06T21:03:59.760ZBegin loop 69
148172024-03-06T21:03:59.760ZDownstairs to restart: 2
148182024-03-06T21:03:59.775Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.521420375Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691}
148192024-03-06T21:03:59.775Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.521723774Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691}
148202024-03-06T21:03:59.775Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
148212024-03-06T21:03:59.775Z,"time":"2024-03-06T21:03:58.521744803Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691}
148222024-03-06T21:03:59.781Z{"msg":"Crucible 8641029d-4c34-4946-8081-1e0c85562a96 has session id: f5db0ed5-bb25-4141-b934-d1e209dd4ca7","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.521794229Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148232024-03-06T21:03:59.781Z{"msg":"Upstairs opts: Upstairs UUID: 8641029d-4c34-4946-8081-1e0c85562a96, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.521853775Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148242024-03-06T21:03:59.781Z{"msg":"Crucible stats registered with UUID: 8641029d-4c34-4946-8081-1e0c85562a96","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.521894412Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148252024-03-06T21:03:59.781Z{"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.52192133Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691}
148262024-03-06T21:03:59.781Z{"msg":"8641029d-4c34-4946-8081-1e0c85562a96 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.521955418Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148272024-03-06T21:03:59.781Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.521999415Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"client":"0","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148282024-03-06T21:03:59.781Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.522030413Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"client":"0","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148292024-03-06T21:03:59.781Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.52207001Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"client":"1","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148302024-03-06T21:03:59.781Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.522097318Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"client":"1","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148312024-03-06T21:03:59.781Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.522122097Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"client":"2","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148322024-03-06T21:03:59.781Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.522155524Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"client":"2","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148332024-03-06T21:03:59.781Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.522171473Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148342024-03-06T21:03:59.781Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.522196791Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148352024-03-06T21:03:59.781Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.522263707Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148362024-03-06T21:03:59.781Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.522428895Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148372024-03-06T21:03:59.781Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.522472563Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148382024-03-06T21:03:59.781Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.522550497Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148392024-03-06T21:03:59.782Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.523335133Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"client":"0","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148402024-03-06T21:03:59.782Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.523393489Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"client":"0","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148412024-03-06T21:03:59.782Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.523424927Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"client":"1","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148422024-03-06T21:03:59.782Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.523460834Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"client":"1","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148432024-03-06T21:03:59.782Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.52409948Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"client":"1","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148442024-03-06T21:03:59.782Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.524158656Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"client":"0","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148452024-03-06T21:03:59.782Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.524483294Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"client":"2","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148462024-03-06T21:03:59.782Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.524521281Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"client":"2","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148472024-03-06T21:03:59.782Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.524698909Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"client":"0","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148482024-03-06T21:03:59.782Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.524739676Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"client":"0","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148492024-03-06T21:03:59.782Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.524764614Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148502024-03-06T21:03:59.782Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.524789543Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"client":"1","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148512024-03-06T21:03:59.782Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.524809051Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"client":"1","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148522024-03-06T21:03:59.782Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.524840719Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148532024-03-06T21:03:59.782Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.5251197Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"client":"2","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148542024-03-06T21:03:59.782Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.525527332Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"client":"2","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148552024-03-06T21:03:59.782Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.525588558Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"client":"2","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148562024-03-06T21:03:59.782Z{"msg":"[0]R flush_numbers[0..12]: [235, 234, 229, 238, 234, 236, 238, 234, 232, 229, 237, 229]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.525623755Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"":"downstairs","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148572024-03-06T21:03:59.782Z{"msg":"[0]R generation[0..12]: [135, 134, 132, 137, 134, 136, 137, 134, 133, 132, 136, 132]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.525650493Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"":"downstairs","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148582024-03-06T21:03:59.782Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.525681541Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"":"downstairs","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148592024-03-06T21:03:59.782Z{"msg":"[1]R flush_numbers[0..12]: [235, 234, 229, 238, 234, 236, 238, 234, 232, 229, 237, 229]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
148602024-03-06T21:03:59.782Z,"time":"2024-03-06T21:03:58.52570471Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"":"downstairs","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148612024-03-06T21:03:59.782Z{"msg":"[1]R generation[0..12]: [135, 134, 132, 137, 134, 136, 137, 134, 133, 132, 136, 132]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.525730338Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"":"downstairs","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148622024-03-06T21:03:59.782Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.525760846Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"":"downstairs","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148632024-03-06T21:03:59.783Z{"msg":"[2]R flush_numbers[0..12]: [235, 234, 229, 238, 234, 236, 238, 234, 232, 229, 237, 229]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.525786414Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"":"downstairs","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148642024-03-06T21:03:59.783Z{"msg":"[2]R generation[0..12]: [135, 134, 132, 137, 134, 136, 137, 134, 133, 132, 136, 132]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.525814402Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"":"downstairs","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148652024-03-06T21:03:59.787Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.525839831Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"":"downstairs","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148662024-03-06T21:03:59.787Z{"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.525877238Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"":"downstairs","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148672024-03-06T21:03:59.787Z{"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.525901416Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"":"downstairs","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148682024-03-06T21:03:59.787Z{"msg":"Next flush: 239","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.525934074Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"":"downstairs","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148692024-03-06T21:03:59.787Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.525958082Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"":"downstairs","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148702024-03-06T21:03:59.787Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.525981841Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148712024-03-06T21:03:59.787Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.526003119Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"client":"0","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148722024-03-06T21:03:59.787Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.526033667Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"client":"1","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148732024-03-06T21:03:59.787Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.526048876Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"client":"2","":"downstairs","session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148742024-03-06T21:03:59.787Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.526070635Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148752024-03-06T21:03:59.787Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.526093113Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148762024-03-06T21:03:59.787Z{"msg":"8641029d-4c34-4946-8081-1e0c85562a96 is now active with session: f5db0ed5-bb25-4141-b934-d1e209dd4ca7","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.526116001Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
148772024-03-06T21:03:59.787Zsession_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148782024-03-06T21:03:59.787Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.526150249Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691,"session_id":"f5db0ed5-bb25-4141-b934-d1e209dd4ca7"}
148792024-03-06T21:03:59.787Z{"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:03:58.526163858Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691}
148802024-03-06T21:03:59.787ZLoading write count information from file "/tmp/test_repair_verify.data"
148812024-03-06T21:03:59.787ZRun Repair workload
148822024-03-06T21:03:59.787Z01/30 Write block 326 len 16384 data: 14 11 11 10
148832024-03-06T21:03:59.787Z02/30 Write block 251 len 8192 data: 5 6
148842024-03-06T21:03:59.787Z03/30 Write block 528 len 4096 data: 12
148852024-03-06T21:03:59.787Z04/30 Flush
148862024-03-06T21:03:59.788Z05/30 Write block 85 len 36864 data: 7 9 8 7 7 4 7 7 7
148872024-03-06T21:03:59.788Z06/30 Read block 167 len 28672
148882024-03-06T21:03:59.791Z07/30 Read block 233 len 28672
148892024-03-06T21:03:59.791Z08/30 Read block 93 len 32768
148902024-03-06T21:03:59.791Z09/30 Read block 8 len 28672
148912024-03-06T21:03:59.791Z10/30 Flush
148922024-03-06T21:03:59.795Z11/30 Flush
148932024-03-06T21:03:59.795Z12/30 Write block 473 len 24576 data: 5 6 6 7 7 9
148942024-03-06T21:03:59.795Z13/30 Read block 417 len 36864
148952024-03-06T21:03:59.798Z14/30 Read block 268 len 32768
148962024-03-06T21:03:59.798Z15/30 Write block 168 len 32768 data: 9 7 6 7 9 13 12 12
148972024-03-06T21:03:59.798Z16/30 Write block 178 len 8192 data: 8 12
148982024-03-06T21:03:59.801Z17/30 Write block 261 len 4096 data: 8
148992024-03-06T21:03:59.801Z18/30 Write block 564 len 36864 data: 13 13 12 8 6 6 8 9 9
149002024-03-06T21:03:59.801Z19/30 Flush
149012024-03-06T21:03:59.805Z20/30 Write block 28 len 8192 data: 9 12
149022024-03-06T21:03:59.805Z21/30 Read block 35 len 4096
149032024-03-06T21:03:59.805Z22/30 Read block 543 len 32768
149042024-03-06T21:03:59.809Z23/30 Read block 438 len 16384
149052024-03-06T21:03:59.809Z24/30 Read block 250 len 12288
149062024-03-06T21:03:59.809Z25/30 Flush
149072024-03-06T21:03:59.809Z26/30 Write block 336 len 20480 data: 8 9 12 10 12
149082024-03-06T21:03:59.812Z27/30 Read block 374 len 16384
149092024-03-06T21:03:59.812Z28/30 Write block 512 len 20480 data: 12 11 12 14 13
149102024-03-06T21:03:59.812Z29/30 Read block 72 len 20480
149112024-03-06T21:03:59.816Z30/30 Write block 190 len 28672 data: 15 12 14 11 10 12 14
149122024-03-06T21:03:59.816Z----------------------------------------------------------------
149132024-03-06T21:03:59.816Z Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:30
149142024-03-06T21:03:59.816ZStates: Active Active Active
149152024-03-06T21:03:59.816ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
149162024-03-06T21:03:59.816Z 1 Acked 1000 Write 4 Done Done Done false
149172024-03-06T21:03:59.816Z 2 Acked 1001 Write 2 Done Done Done false
149182024-03-06T21:03:59.816Z 3 Acked 1002 Write 1 Done Done Sent false
149192024-03-06T21:03:59.816Z 4 Acked 1003 Flush 0 Done Done Sent false
149202024-03-06T21:03:59.816Z 5 Acked 1004 Write 9 Done Done Sent false
149212024-03-06T21:03:59.816Z 6 Acked 1005 Read 7 Done Done Sent false
149222024-03-06T21:03:59.817Z 7 Acked 1006 Read 7 Done Done Sent false
149232024-03-06T21:03:59.817Z 8 Acked 1007 Read 8 Done Done Sent false
149242024-03-06T21:03:59.817Z 9 Acked 1008 Read 7 Done Done Sent false
149252024-03-06T21:03:59.817Z 10 Acked 1009 Flush 0 Done Done Sent false
149262024-03-06T21:03:59.817Z 11 Acked 1010 Flush 0 Done Done Sent false
149272024-03-06T21:03:59.817Z 12 Acked 1011 Write 6 Done Done Sent false
149282024-03-06T21:03:59.817Z 13 Acked 1012 Read 9 Done Done Sent false
149292024-03-06T21:03:59.817Z 14 Acked 1013 Read 8 Done Done Sent false
149302024-03-06T21:03:59.817Z 15 Acked 1014 Write 8 Done Done Sent false
149312024-03-06T21:03:59.817Z 16 Acked 1015 Write 2 Done Done Sent false
149322024-03-06T21:03:59.817Z 17 Acked 1016 Write 1 Done Done Sent false
149332024-03-06T21:03:59.817Z 18 Acked 1017 Write 9 Done Done Sent false
149342024-03-06T21:03:59.817Z 19 Acked 1018 Flush 0 Done Done Sent false
149352024-03-06T21:03:59.817Z 20 Acked 1019 Write 2 Done Done Sent false
149362024-03-06T21:03:59.817Z 21 Acked 1020 Read 1 Done Done Sent false
149372024-03-06T21:03:59.817Z 22 Acked 1021 Read 8 Done Done Sent false
149382024-03-06T21:03:59.817Z 23 Acked 1022 Read 4 Done Done Sent false
149392024-03-06T21:03:59.817Z 24 Acked 1023 Read 3 Done Done Sent false
149402024-03-06T21:03:59.817Z 25 Acked 1024 Flush 0 Done Done Sent false
149412024-03-06T21:03:59.817Z 26 Acked 1025 Write 5 Done Done Sent false
149422024-03-06T21:03:59.817Z 27 Acked 1026 Read 4 Done Done Sent false
149432024-03-06T21:03:59.817Z 28 Acked 1027 Write 5 Done Done Sent false
149442024-03-06T21:03:59.817Z 29 Acked 1028 Read 5 Done Done Sent false
149452024-03-06T21:03:59.818Z 30 Acked 1029 Write 7 Sent Sent Sent false
149462024-03-06T21:03:59.818Z STATES DS:0 DS:1 DS:2 TOTAL
149472024-03-06T21:03:59.818Z New 0 0 0 0
149482024-03-06T21:03:59.818Z Sent 1 1 28 30
149492024-03-06T21:03:59.818Z Done 29 29 2 60
149502024-03-06T21:03:59.818Z Skipped 0 0 0 0
149512024-03-06T21:03:59.818Z Error 0 0 0 0
149522024-03-06T21:03:59.818ZLast Flush: 1024 1024 0
149532024-03-06T21:03:59.818ZDownstairs last five completed:
149542024-03-06T21:03:59.818ZUpstairs last five completed: 30 29 28 27 26
149552024-03-06T21:03:59.818Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:03:58.560704798Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1691}
149562024-03-06T21:03:59.818ZWrote out file "/tmp/test_repair_verify.data"
149572024-03-06T21:03:59.822Z
149582024-03-06T21:03:59.825ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
149592024-03-06T21:03:59.844ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
149602024-03-06T21:03:59.844Z 0 000-019 135 135 135 235 235 235 F F F
149612024-03-06T21:03:59.844Z 1 020-039 138 138 134 243 243 234 F F F <---
149622024-03-06T21:03:59.844Z 2 040-059 132 132 132 229 229 229 F F F
149632024-03-06T21:03:59.844Z 3 060-079 137 137 137 238 238 238 F F F
149642024-03-06T21:03:59.844Z 4 080-099 138 138 134 240 240 234 F F F <---
149652024-03-06T21:03:59.844Z 5 100-119 136 136 136 236 236 236 F F F
149662024-03-06T21:03:59.844Z 6 120-139 137 137 137 238 238 238 F F F
149672024-03-06T21:03:59.844Z 7 140-159 134 134 134 234 234 234 F F F
149682024-03-06T21:03:59.844Z 8 160-179 138 138 133 242 242 232 F F F <---
149692024-03-06T21:03:59.844Z 9 180-199 132 132 132 229 229 229 T T F <---
149702024-03-06T21:03:59.844Z 10 200-219 136 136 136 237 237 237 F F F
149712024-03-06T21:03:59.844Z 11 220-239 132 132 132 229 229 229 F F F
149722024-03-06T21:03:59.844Z 12 240-259 138 138 135 239 239 235 F F T <---
149732024-03-06T21:03:59.845Z 13 260-279 138 138 134 242 242 234 F F F <---
149742024-03-06T21:03:59.845Z 14 280-299 136 136 136 236 236 236 F F F
149752024-03-06T21:03:59.845Z 15 300-319 137 137 137 238 238 238 F F F
149762024-03-06T21:03:59.845Z 16 320-339 138 138 134 239 239 234 T T T <---
149772024-03-06T21:03:59.845Z 17 340-359 132 132 132 229 229 229 T T F <---
149782024-03-06T21:03:59.845Z 18 360-379 135 135 135 235 235 235 F F F
149792024-03-06T21:03:59.849Z 19 380-399 135 135 135 235 235 235 F F F
149802024-03-06T21:03:59.849Z 20 400-419 136 136 136 236 236 236 F F F
149812024-03-06T21:03:59.849Z 21 420-439 132 132 132 231 231 231 F F F
149822024-03-06T21:03:59.849Z 22 440-459 131 131 131 228 228 228 F F F
149832024-03-06T21:03:59.849Z 23 460-479 138 138 134 242 242 234 F F F <---
149842024-03-06T21:03:59.849Z 24 480-499 135 135 135 235 235 235 F F F
149852024-03-06T21:03:59.849Z 25 500-519 129 129 129 226 226 226 T T F <---
149862024-03-06T21:03:59.849Z 26 520-539 138 138 132 239 239 229 F F F <---
149872024-03-06T21:03:59.849Z 27 540-559 135 135 135 235 235 235 F F F
149882024-03-06T21:03:59.849Z 28 560-579 138 138 136 242 242 236 F F F <---
149892024-03-06T21:03:59.849Z 29 580-599 134 134 134 234 234 234 F F F
149902024-03-06T21:03:59.849ZMax gen: 138, Max flush: 243
149912024-03-06T21:03:59.849ZError: Difference in extent metadata found!
149922024-03-06T21:03:59.849ZA Difference in extent metadata is expected here
149932024-03-06T21:03:59.849ZOn loop 69
149942024-03-06T21:03:59.849Z
149952024-03-06T21:03:59.852ZVerifying data now
149962024-03-06T21:04:05.488ZNo change after verify
149972024-03-06T21:04:05.488ZLoop: 69 Downstairs dump after verify (and repair):
149982024-03-06T21:04:05.931ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
149992024-03-06T21:04:05.931Z 0 000-019 135 135 135 235 235 235 F F F
150002024-03-06T21:04:05.931Z 1 020-039 138 138 138 243 243 243 F F F
150012024-03-06T21:04:05.932Z 2 040-059 132 132 132 229 229 229 F F F
150022024-03-06T21:04:05.932Z 3 060-079 137 137 137 238 238 238 F F F
150032024-03-06T21:04:05.932Z 4 080-099 138 138 138 240 240 240 F F F
150042024-03-06T21:04:05.932Z 5 100-119 136 136 136 236 236 236 F F F
150052024-03-06T21:04:05.932Z 6 120-139 137 137 137 238 238 238 F F F
150062024-03-06T21:04:05.933Z 7 140-159 134 134 134 234 234 234 F F F
150072024-03-06T21:04:05.933Z 8 160-179 138 138 138 242 242 242 F F F
150082024-03-06T21:04:05.933Z 9 180-199 139 139 139 244 244 244 F F F
150092024-03-06T21:04:05.933Z 10 200-219 136 136 136 237 237 237 F F F
150102024-03-06T21:04:05.933Z 11 220-239 132 132 132 229 229 229 F F F
150112024-03-06T21:04:05.934Z 12 240-259 138 138 138 239 239 239 F F F
150122024-03-06T21:04:05.934Z 13 260-279 138 138 138 242 242 242 F F F
150132024-03-06T21:04:05.934Z 14 280-299 136 136 136 236 236 236 F F F
150142024-03-06T21:04:05.934Z 15 300-319 137 137 137 238 238 238 F F F
150152024-03-06T21:04:05.943Z 16 320-339 139 139 139 244 244 244 F F F
150162024-03-06T21:04:05.943Z 17 340-359 139 139 139 244 244 244 F F F
150172024-03-06T21:04:05.943Z 18 360-379 135 135 135 235 235 235 F F F
150182024-03-06T21:04:05.943Z 19 380-399 135 135 135 235 235 235 F F F
150192024-03-06T21:04:05.943Z 20 400-419 136 136 136 236 236 236 F F F
150202024-03-06T21:04:05.943Z 21 420-439 132 132 132 231 231 231 F F F
150212024-03-06T21:04:05.943Z 22 440-459 131 131 131 228 228 228 F F F
150222024-03-06T21:04:05.944Z 23 460-479 138 138 138 242 242 242 F F F
150232024-03-06T21:04:05.944Z 24 480-499 135 135 135 235 235 235 F F F
150242024-03-06T21:04:05.944Z 25 500-519 139 139 139 244 244 244 F F F
150252024-03-06T21:04:05.944Z 26 520-539 138 138 138 239 239 239 F F F
150262024-03-06T21:04:05.944Z 27 540-559 135 135 135 235 235 235 F F F
150272024-03-06T21:04:05.944Z 28 560-579 138 138 138 242 242 242 F F F
150282024-03-06T21:04:05.944Z 29 580-599 134 134 134 234 234 234 F F F
150292024-03-06T21:04:05.944ZMax gen: 139, Max flush: 244
150302024-03-06T21:04:05.944Z
150312024-03-06T21:04:05.944ZBegin loop 70
150322024-03-06T21:04:05.944ZDownstairs to restart: 0
150332024-03-06T21:04:05.944Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.276155157Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699}
150342024-03-06T21:04:05.945Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.276438677Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699}
150352024-03-06T21:04:05.945Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.276462215Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699}
150362024-03-06T21:04:05.945Z{"msg":"Crucible 7d762e70-6128-4985-9c94-33e97eb6caa1 has session id: 613906d7-a8f9-4ea9-acd7-53aaecb1cffc","v":Crucible runtime is spawned
150372024-03-06T21:04:05.945Z0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.27654165Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150382024-03-06T21:04:05.945Z{"msg":"Upstairs opts: Upstairs UUID: 7d762e70-6128-4985-9c94-33e97eb6caa1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.276602906Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150392024-03-06T21:04:05.945Z{"msg":"Crucible stats registered with UUID: 7d762e70-6128-4985-9c94-33e97eb6caa1","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.276626154Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150402024-03-06T21:04:05.945Z{"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.276649403Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699}
150412024-03-06T21:04:05.945Z{"msg":"7d762e70-6128-4985-9c94-33e97eb6caa1 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.276661512Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150422024-03-06T21:04:05.946Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.276674061Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"client":"0","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150432024-03-06T21:04:05.946Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.27668867Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"client":"0","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150442024-03-06T21:04:05.946Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.276703479Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"client":"1","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150452024-03-06T21:04:05.946Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.276717618Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"client":"1","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150462024-03-06T21:04:05.946Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.276738856Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"client":"2","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150472024-03-06T21:04:05.946Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.276754765Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"client":"2","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150482024-03-06T21:04:05.946Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.276818681Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150492024-03-06T21:04:05.946Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.276851778Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150502024-03-06T21:04:05.946Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.277059564Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150512024-03-06T21:04:05.946Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.277104511Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150522024-03-06T21:04:05.946Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.277130839Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150532024-03-06T21:04:05.946Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.277156937Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150542024-03-06T21:04:05.946Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.277915945Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"client":"2","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150552024-03-06T21:04:05.946Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.277962652Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"client":"2","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150562024-03-06T21:04:05.947Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.27799135Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"client":"1","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150572024-03-06T21:04:05.947Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.278036377Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"client":"1","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150582024-03-06T21:04:05.947Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.278615787Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"client":"2","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150592024-03-06T21:04:05.947Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.278670173Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"client":"1","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150602024-03-06T21:04:05.947Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.278808794Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"client":"0","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150612024-03-06T21:04:05.947Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.2788584Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"client":"0","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150622024-03-06T21:04:05.947Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.279272582Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"client":"2","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150632024-03-06T21:04:05.947Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.279308839Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"client":"2","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150642024-03-06T21:04:05.947Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.279325258Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150652024-03-06T21:04:05.947Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.279399623Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"client":"1","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150662024-03-06T21:04:05.947Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.279428261Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"client":"1","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150672024-03-06T21:04:05.947Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.27944318Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150682024-03-06T21:04:05.947Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.279657685Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"client":"0","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150692024-03-06T21:04:05.947Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.280069807Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"client":"0","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150702024-03-06T21:04:05.948Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.280097485Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"client":"0","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150712024-03-06T21:04:05.948Z{"msg":"[0]R flush_numbers[0..12]: [235, 243, 229, 238, 240, 236, 238, 234, 242, 244, 237, 229]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.280112704Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"":"downstairs","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150722024-03-06T21:04:05.948Z{"msg":"[0]R generation[0..12]: [135, 138, 132, 137, 138, 136, 137, 134, 138, 139, 136, 132]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.280127303Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"":"downstairs","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150732024-03-06T21:04:05.948Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.280149361Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"":"downstairs","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150742024-03-06T21:04:05.948Z{"msg":"[1]R flush_numbers[0..12]: [235, 243, 229, 238, 240, 236, 238, 234, 242, 244, 237, 229]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.28016461Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"":"downstairs","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150752024-03-06T21:04:05.948Z{"msg":"[1]R generation[0..12]: [135, 138, 132, 137, 138, 136, 137, 134, 138, 139, 136, 132]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.280185359Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"":"downstairs","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150762024-03-06T21:04:05.948Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.280200048Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"":"downstairs","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150772024-03-06T21:04:05.948Z{"msg":"[2]R flush_numbers[0..12]: [235, 243, 229, 238, 240, 236, 238, 234, 242, 244, 237, 229]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.280220206Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"":"downstairs","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150782024-03-06T21:04:05.948Z{"msg":"[2]R generation[0..12]: [135, 138, 132, 137, 138, 136, 137, 134, 138, 139, 136, 132]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
150792024-03-06T21:04:05.948Z,"time":"2024-03-06T21:04:04.280234356Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"":"downstairs","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150802024-03-06T21:04:05.948Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.280261494Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"":"downstairs","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150812024-03-06T21:04:05.948Z{"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.280281242Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"":"downstairs","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150822024-03-06T21:04:05.948Z{"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.280295191Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"":"downstairs","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150832024-03-06T21:04:05.948Z{"msg":"Next flush: 245","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.28031455Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"":"downstairs","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150842024-03-06T21:04:05.948Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.280328089Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"":"downstairs","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150852024-03-06T21:04:05.948Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.280348688Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150862024-03-06T21:04:05.948Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.280361147Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"client":"0","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150872024-03-06T21:04:05.948Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.280374726Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"client":"1","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150882024-03-06T21:04:05.948Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.280395074Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"client":"2","":"downstairs","session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150892024-03-06T21:04:05.948Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.280409033Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150902024-03-06T21:04:05.948Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.280427822Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150912024-03-06T21:04:05.948Z{"msg":"7d762e70-6128-4985-9c94-33e97eb6caa1 is now active with session: 613906d7-a8f9-4ea9-acd7-53aaecb1cffc","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.280441011Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150922024-03-06T21:04:05.948Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.28045292Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699,"session_id":"613906d7-a8f9-4ea9-acd7-53aaecb1cffc"}
150932024-03-06T21:04:05.949Z{"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:04.28046626Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699}
150942024-03-06T21:04:05.949ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
150952024-03-06T21:04:05.949ZLoading write count information from file "/tmp/test_repair_verify.data"
150962024-03-06T21:04:05.949ZRun Repair workload
150972024-03-06T21:04:05.949Z01/30 Write block 60 len 16384 data: 16 19 18 22
150982024-03-06T21:04:05.949Z02/30 Write block 189 len 8192 data: 14 16
150992024-03-06T21:04:05.949Z03/30 Read block 501 len 28672
151002024-03-06T21:04:05.949Z04/30 Write block 304 len 20480 data: 13 14 16 17 16
151012024-03-06T21:04:05.949Z05/30 Read block 387 len 36864
151022024-03-06T21:04:05.949Z06/30 Read block 285 len 36864
151032024-03-06T21:04:05.949Z07/30 Write block 170 len 40960 data: 7 8 10 14 13 13 11 9 9 13
151042024-03-06T21:04:05.949Z08/30 Flush
151052024-03-06T21:04:05.949Z09/30 Write block 544 len 20480 data: 14 15 14 13 11
151062024-03-06T21:04:05.949Z10/30 Read block 457 len 24576
151072024-03-06T21:04:05.949Z11/30 Read block 349 len 32768
151082024-03-06T21:04:05.949Z12/30 Flush
151092024-03-06T21:04:05.949Z13/30 Write block 335 len 40960 data: 6 9 10 13 11 13 12 11 12 13
151102024-03-06T21:04:05.949Z14/30 Read block 33 len 4096
151112024-03-06T21:04:05.949Z15/30 Flush
151122024-03-06T21:04:05.949Z16/30 Flush
151132024-03-06T21:04:05.949Z17/30 Write block 302 len 8192 data: 11 11
151142024-03-06T21:04:05.949Z18/30 Read block 96 len 24576
151152024-03-06T21:04:05.963Z19/30 Write block 40 len 8192 data: 9 12
151162024-03-06T21:04:05.963Z20/30 Write block 22 len 20480 data: 15 15 12 11 11
151172024-03-06T21:04:05.963Z21/30 Read block 52 len 8192
151182024-03-06T21:04:05.963Z22/30 Write block 175 len 28672 data: 14 12 10 10 14 14 16
151192024-03-06T21:04:05.963Z23/30 Read block 519 len 36864
151202024-03-06T21:04:05.963Z24/30 Read block 36 len 24576
151212024-03-06T21:04:05.963Z25/30 Read block 386 len 40960
151222024-03-06T21:04:05.963Z26/30 Write block 310 len 8192 data: 13 14
151232024-03-06T21:04:05.963Z27/30 Read block 469 len 28672
151242024-03-06T21:04:05.963Z28/30 Write block 463 len 8192 data: 17 17
151252024-03-06T21:04:05.963Z29/30 Write block 332 len 8192 data: 6 8
151262024-03-06T21:04:05.963Z30/30 Write block 457 len 20480 data: 13 12 13 12 15
151272024-03-06T21:04:05.963Z----------------------------------------------------------------
151282024-03-06T21:04:05.963Z Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:22
151292024-03-06T21:04:05.963ZStates: Active Active Active
151302024-03-06T21:04:05.963ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
151312024-03-06T21:04:05.963Z 9 Acked 1008 Write 5 Sent Done Done false
151322024-03-06T21:04:05.963Z 10 Acked 1009 Read 6 Sent Done Done false
151332024-03-06T21:04:05.963Z 11 Acked 1010 Read 8 Sent Done Done false
151342024-03-06T21:04:05.963Z 12 Acked 1011 Flush 0 Sent Done Done false
151352024-03-06T21:04:05.964Z 13 Acked 1012 Write 10 Sent Done Done false
151362024-03-06T21:04:05.964Z 14 Acked 1013 Read 1 Sent Done Done false
151372024-03-06T21:04:05.964Z 15 Acked 1014 Flush 0 Sent Done Done false
151382024-03-06T21:04:05.964Z 16 Acked 1015 Flush 0 Sent Done Done false
151392024-03-06T21:04:05.964Z 17 Acked 1016 Write 2 Sent Done Done false
151402024-03-06T21:04:05.964Z 18 Acked 1017 Read 6 Sent Done Done false
151412024-03-06T21:04:05.964Z 19 Acked 1018 Write 2 Sent Done Done false
151422024-03-06T21:04:05.964Z 20 Acked 1019 Write 5 Sent Done Done false
151432024-03-06T21:04:05.964Z 21 Acked 1020 Read 2 Sent Done Done false
151442024-03-06T21:04:05.964Z 22 Acked 1021 Write 7 Sent Done Done false
151452024-03-06T21:04:05.964Z 23 Acked 1022 Read 9 Sent Done Done false
151462024-03-06T21:04:05.964Z 24 Acked 1023 Read 6 Sent Done Done false
151472024-03-06T21:04:05.964Z 25 Acked 1024 Read 10 Sent Done Done false
151482024-03-06T21:04:05.964Z 26 Acked 1025 Write 2 Sent Done Done false
151492024-03-06T21:04:05.964Z 27 Acked 1026 Read 7 Sent Done Done false
151502024-03-06T21:04:05.964Z 28 Acked 1027 Write 2 Sent Done Done false
151512024-03-06T21:04:05.964Z 29 Acked 1028 Write 2 Sent Done Done false
151522024-03-06T21:04:05.964Z 30 Acked 1029 Write 5 Sent Sent Sent false
151532024-03-06T21:04:05.964Z STATES DS:0 DS:1 DS:2 TOTAL
151542024-03-06T21:04:05.964Z New 0 0 0 0
151552024-03-06T21:04:05.964Z Sent 22 1 1 24
151562024-03-06T21:04:05.964Z Done 0 21 21 42
151572024-03-06T21:04:05.964Z Skipped 0 0 0 0
151582024-03-06T21:04:05.965Z Error 0 0 0 0
151592024-03-06T21:04:05.965ZLast Flush: 1007 1015 1015
151602024-03-06T21:04:05.965ZDownstairs last five completed: 1007 1006 1005 1004 1003
151612024-03-06T21:04:05.965ZUpstairs last five completed: 30 29 28 27 26
151622024-03-06T21:04:05.965Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:04:04.315548722Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1699}
151632024-03-06T21:04:05.965ZWrote out file "/tmp/test_repair_verify.data"
151642024-03-06T21:04:05.965Z
151652024-03-06T21:04:05.965ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
151662024-03-06T21:04:05.965ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
151672024-03-06T21:04:05.965Z 0 000-019 135 135 135 235 235 235 F F F
151682024-03-06T21:04:05.965Z 1 020-039 138 138 138 243 243 243 F T T <---
151692024-03-06T21:04:05.965Z 2 040-059 132 132 132 229 229 229 F T T <---
151702024-03-06T21:04:05.965Z 3 060-079 140 140 140 245 245 245 F F F
151712024-03-06T21:04:05.965Z 4 080-099 138 138 138 240 240 240 F F F
151722024-03-06T21:04:05.965Z 5 100-119 136 136 136 236 236 236 F F F
151732024-03-06T21:04:05.965Z 6 120-139 137 137 137 238 238 238 F F F
151742024-03-06T21:04:05.965Z 7 140-159 134 134 134 234 234 234 F F F
151752024-03-06T21:04:05.965Z 8 160-179 140 140 140 245 245 245 F T T <---
151762024-03-06T21:04:05.965Z 9 180-199 140 140 140 245 245 245 F T T <---
151772024-03-06T21:04:05.965Z 10 200-219 136 136 136 237 237 237 F F F
151782024-03-06T21:04:05.965Z 11 220-239 132 132 132 229 229 229 F F F
151792024-03-06T21:04:05.965Z 12 240-259 138 138 138 239 239 239 F F F
151802024-03-06T21:04:05.965Z 13 260-279 138 138 138 242 242 242 F F F
151812024-03-06T21:04:05.966Z 14 280-299 136 136 136 236 236 236 F F F
151822024-03-06T21:04:05.966Z 15 300-319 140 140 140 245 245 245 F T T <---
151832024-03-06T21:04:05.966Z 16 320-339 139 140 140 244 247 247 F T T <---
151842024-03-06T21:04:05.966Z 17 340-359 139 140 140 244 247 247 F F F <---
151852024-03-06T21:04:05.966Z 18 360-379 135 135 135 235 235 235 F F F
151862024-03-06T21:04:05.966Z 19 380-399 135 135 135 235 235 235 F F F
151872024-03-06T21:04:05.966Z 20 400-419 136 136 136 236 236 236 F F F
151882024-03-06T21:04:05.966Z 21 420-439 132 132 132 231 231 231 F F F
151892024-03-06T21:04:05.966Z 22 440-459 131 131 131 228 228 228 F T T <---
151902024-03-06T21:04:05.966Z 23 460-479 138 138 138 242 242 242 F T T <---
151912024-03-06T21:04:05.966Z 24 480-499 135 135 135 235 235 235 F F F
151922024-03-06T21:04:05.966Z 25 500-519 139 139 139 244 244 244 F F F
151932024-03-06T21:04:05.966Z 26 520-539 138 138 138 239 239 239 F F F
151942024-03-06T21:04:05.966Z 27 540-559 135 140 140 235 246 246 F F F <---
151952024-03-06T21:04:05.966Z 28 560-579 138 138 138 242 242 242 F F F
151962024-03-06T21:04:05.966Z 29 580-599 134 134 134 234 234 234 F F F
151972024-03-06T21:04:05.966ZMax gen: 140, Max flush: 247
151982024-03-06T21:04:05.966ZError: Difference in extent metadata found!
151992024-03-06T21:04:05.966ZA Difference in extent metadata is expected here
152002024-03-06T21:04:05.966ZOn loop 70
152012024-03-06T21:04:05.966Z
152022024-03-06T21:04:05.966ZVerifying data now
152032024-03-06T21:04:10.504ZNo change after verify
152042024-03-06T21:04:10.505ZLoop: 70 Downstairs dump after verify (and repair):
152052024-03-06T21:04:10.505ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
152062024-03-06T21:04:10.505Z 0 000-019 135 135 135 235 235 235 F F F
152072024-03-06T21:04:10.505Z 1 020-039 141 141 141 248 248 248 F F F
152082024-03-06T21:04:10.505Z 2 040-059 141 141 141 248 248 248 F F F
152092024-03-06T21:04:10.505Z 3 060-079 140 140 140 245 245 245 F F F
152102024-03-06T21:04:10.505Z 4 080-099 138 138 138 240 240 240 F F F
152112024-03-06T21:04:10.505Z 5 100-119 136 136 136 236 236 236 F F F
152122024-03-06T21:04:10.505Z 6 120-139 137 137 137 238 238 238 F F F
152132024-03-06T21:04:10.505Z 7 140-159 134 134 134 234 234 234 F F F
152142024-03-06T21:04:10.505Z 8 160-179 141 141 141 248 248 248 F F F
152152024-03-06T21:04:10.506Z 9 180-199 141 141 141 248 248 248 F F F
152162024-03-06T21:04:10.506Z 10 200-219 136 136 136 237 237 237 F F F
152172024-03-06T21:04:10.506Z 11 220-239 132 132 132 229 229 229 F F F
152182024-03-06T21:04:10.506Z 12 240-259 138 138 138 239 239 239 F F F
152192024-03-06T21:04:10.506Z 13 260-279 138 138 138 242 242 242 F F F
152202024-03-06T21:04:10.506Z 14 280-299 136 136 136 236 236 236 F F F
152212024-03-06T21:04:10.506Z 15 300-319 141 141 141 248 248 248 F F F
152222024-03-06T21:04:10.506Z 16 320-339 141 141 141 248 248 248 F F F
152232024-03-06T21:04:10.506Z 17 340-359 140 140 140 247 247 247 F F F
152242024-03-06T21:04:10.506Z 18 360-379 135 135 135 235 235 235 F F F
152252024-03-06T21:04:10.506Z 19 380-399 135 135 135 235 235 235 F F F
152262024-03-06T21:04:10.506Z 20 400-419 136 136 136 236 236 236 F F F
152272024-03-06T21:04:10.507Z 21 420-439 132 132 132 231 231 231 F F F
152282024-03-06T21:04:10.507Z 22 440-459 141 141 141 248 248 248 F F F
152292024-03-06T21:04:10.507Z 23 460-479 141 141 141 248 248 248 F F F
152302024-03-06T21:04:10.507Z 24 480-499 135 135 135 235 235 235 F F F
152312024-03-06T21:04:10.507Z 25 500-519 139 139 139 244 244 244 F F F
152322024-03-06T21:04:10.507Z 26 520-539 138 138 138 239 239 239 F F F
152332024-03-06T21:04:10.507Z 27 540-559 140 140 140 246 246 246 F F F
152342024-03-06T21:04:10.507Z 28 560-579 138 138 138 242 242 242 F F F
152352024-03-06T21:04:10.507Z 29 580-599 134 134 134 234 234 234 F F F
152362024-03-06T21:04:10.507ZMax gen: 141, Max flush: 248
152372024-03-06T21:04:10.507Z
152382024-03-06T21:04:10.507ZBegin loop 71
152392024-03-06T21:04:10.507ZDownstairs to restart: 1
152402024-03-06T21:04:10.508Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.950140229Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707}
152412024-03-06T21:04:10.508Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.950395892Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707}
152422024-03-06T21:04:10.508Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.95042167Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707}
152432024-03-06T21:04:10.508ZCrucible runtime is spawned
152442024-03-06T21:04:10.508Z{"msg":"Crucible d9354ac0-2e2d-4e65-a934-3fc525f5231c has session id: e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.950467997Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152452024-03-06T21:04:10.508Z{"msg":"Upstairs opts: Upstairs UUID: d9354ac0-2e2d-4e65-a934-3fc525f5231c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.950514603Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152462024-03-06T21:04:10.508Z{"msg":"Crucible stats registered with UUID: d9354ac0-2e2d-4e65-a934-3fc525f5231c","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.950556331Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152472024-03-06T21:04:10.508Z{"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.950577999Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707}
152482024-03-06T21:04:10.508Z{"msg":"d9354ac0-2e2d-4e65-a934-3fc525f5231c active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.950724189Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152492024-03-06T21:04:10.532Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.950750817Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"client":"0","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152502024-03-06T21:04:10.532Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.950780085Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"client":"0","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152512024-03-06T21:04:10.532Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.950816283Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"client":"1","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152522024-03-06T21:04:10.532Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.95085459Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"client":"1","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152532024-03-06T21:04:10.533Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.950887918Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"client":"2","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152542024-03-06T21:04:10.533Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.950915836Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"client":"2","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152552024-03-06T21:04:10.533Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.950949904Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152562024-03-06T21:04:10.533Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.95100002Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152572024-03-06T21:04:10.533Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.951037527Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152582024-03-06T21:04:10.533Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.951187617Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152592024-03-06T21:04:10.533Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.951271401Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152602024-03-06T21:04:10.533Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.951322468Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152612024-03-06T21:04:10.534Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.951899388Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"client":"2","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152622024-03-06T21:04:10.534Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.951927356Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"client":"2","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152632024-03-06T21:04:10.534Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.951959614Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"client":"0","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152642024-03-06T21:04:10.534Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.951984782Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"client":"0","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152652024-03-06T21:04:10.534Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.952499577Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"client":"0","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152662024-03-06T21:04:10.534Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.952617498Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"client":"2","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152672024-03-06T21:04:10.534Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.953083697Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"client":"0","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152682024-03-06T21:04:10.534Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.953120564Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"client":"0","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152692024-03-06T21:04:10.534Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.953148562Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152702024-03-06T21:04:10.535Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.953162021Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"client":"2","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152712024-03-06T21:04:10.535Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.953190289Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"client":"2","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152722024-03-06T21:04:10.535Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.953220627Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152732024-03-06T21:04:10.535Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.953253535Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"client":"1","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152742024-03-06T21:04:10.535Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.953275553Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"client":"1","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152752024-03-06T21:04:10.535Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.953777679Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"client":"1","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152762024-03-06T21:04:10.535Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.954220998Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"client":"1","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152772024-03-06T21:04:10.535Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.954265645Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"client":"1","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152782024-03-06T21:04:10.536Z{"msg":"[0]R flush_numbers[0..12]: [235, 248, 248, 245, 240, 236, 238, 234, 248, 248, 237, 229]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.954282934Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"":"downstairs","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152792024-03-06T21:04:10.536Z{"msg":"[0]R generation[0..12]: [135, 141, 141, 140, 138, 136, 137, 134, 141, 141, 136, 132]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.954304482Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"":"downstairs","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152802024-03-06T21:04:10.536Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.954320451Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"":"downstairs","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152812024-03-06T21:04:10.536Z{"msg":"[1]R flush_numbers[0..12]: [235, 248, 248, 245, 240, 236, 238, 234, 248, 248, 237, 229]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.95434185Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"":"downstairs","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152822024-03-06T21:04:10.536Z{"msg":"[1]R generation[0..12]: [135, 141, 141, 140, 138, 136, 137, 134, 141, 141, 136, 132]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.954363148Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"":"downstairs","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152832024-03-06T21:04:10.536Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
152842024-03-06T21:04:10.536Z,"time":"2024-03-06T21:04:08.954384457Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"":"downstairs","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152852024-03-06T21:04:10.536Z{"msg":"[2]R flush_numbers[0..12]: [235, 248, 248, 245, 240, 236, 238, 234, 248, 248, 237, 229]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.954407725Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"":"downstairs","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152862024-03-06T21:04:10.537Z{"msg":"[2]R generation[0..12]: [135, 141, 141, 140, 138, 136, 137, 134, 141, 141, 136, 132]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.954428394Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"":"downstairs","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152872024-03-06T21:04:10.537Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.954449672Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"":"downstairs","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152882024-03-06T21:04:10.537Z{"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.954469381Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"":"downstairs","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152892024-03-06T21:04:10.537Z{"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.954489649Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"":"downstairs","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152902024-03-06T21:04:10.537Z{"msg":"Next flush: 249","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.954510028Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"":"downstairs","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152912024-03-06T21:04:10.537Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.954529807Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"":"downstairs","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152922024-03-06T21:04:10.538Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.954550195Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152932024-03-06T21:04:10.538Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.954568694Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"client":"0","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152942024-03-06T21:04:10.538Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.954588673Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"client":"1","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152952024-03-06T21:04:10.538Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.954610351Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"client":"2","":"downstairs","session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152962024-03-06T21:04:10.538Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.95462496Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152972024-03-06T21:04:10.538Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.954644099Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152982024-03-06T21:04:10.540Z{"msg":"d9354ac0-2e2d-4e65-a934-3fc525f5231c is now active with session: e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.954663537Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
152992024-03-06T21:04:10.540Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.954682086Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707,"session_id":"e253bc15-a9b2-4a37-a8e5-1f3ab4e9e81c"}
153002024-03-06T21:04:10.541Z{"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:08.954702405Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707}
153012024-03-06T21:04:10.541ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
153022024-03-06T21:04:10.541ZLoading write count information from file "/tmp/test_repair_verify.data"
153032024-03-06T21:04:10.541ZRun Repair workload
153042024-03-06T21:04:10.541ZError: Difference in extent metadata found!
153052024-03-06T21:04:10.541Z01/30 Flush
153062024-03-06T21:04:10.542Z02/30 Write block 243 len 20480 data: 13 12 11 10 9
153072024-03-06T21:04:10.542Z03/30 Flush
153082024-03-06T21:04:10.542Z04/30 Write block 533 len 20480 data: 10 10 12 10 11
153092024-03-06T21:04:10.542Z05/30 Read block 506 len 4096
153102024-03-06T21:04:10.543Z06/30 Read block 489 len 24576
153112024-03-06T21:04:10.543Z07/30 Read block 93 len 8192
153122024-03-06T21:04:10.543Z08/30 Read block 348 len 36864
153132024-03-06T21:04:10.543Z09/30 Write block 463 len 32768 data: 18 18 16 12 11 10 9 6
153142024-03-06T21:04:10.543Z10/30 Read block 300 len 40960
153152024-03-06T21:04:10.543Z11/30 Read block 266 len 16384
153162024-03-06T21:04:10.543Z12/30 Read block 266 len 32768
153172024-03-06T21:04:10.544Z13/30 Read block 288 len 32768
153182024-03-06T21:04:10.544Z14/30 Write block 87 len 24576 data: 9 8 8 5 8 8
153192024-03-06T21:04:10.544Z15/30 Read block 569 len 28672
153202024-03-06T21:04:10.544Z16/30 Read block 447 len 36864
153212024-03-06T21:04:10.544Z17/30 Write block 265 len 40960 data: 12 10 10 11 11 10 11 9 10 14
153222024-03-06T21:04:10.544Z18/30 Read block 298 len 32768
153232024-03-06T21:04:10.544Z19/30 Write block 523 len 28672 data: 7 9 9 9 10 13 12
153242024-03-06T21:04:10.544Z20/30 Flush
153252024-03-06T21:04:10.544Z21/30 Write block 4 len 32768 data: 11 11 11 12 10 10 9 13
153262024-03-06T21:04:10.544Z22/30 Read block 138 len 36864
153272024-03-06T21:04:10.545Z23/30 Flush
153282024-03-06T21:04:10.545Z24/30 Write block 526 len 24576 data: 10 11 14 13 13 12
153292024-03-06T21:04:10.545Z25/30 Read block 40 len 16384
153302024-03-06T21:04:10.545Z26/30 Write block 454 len 32768 data: 11 11 10 14 13 14 13 16
153312024-03-06T21:04:10.545Z27/30 Read block 587 len 40960
153322024-03-06T21:04:10.545Z28/30 Read block 547 len 16384
153332024-03-06T21:04:10.545Z29/30 Write block 55 len 8192 data: 10 7
153342024-03-06T21:04:10.545Z30/30 Read block 182 len 32768
153352024-03-06T21:04:10.545Z----------------------------------------------------------------
153362024-03-06T21:04:10.545Z Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:29
153372024-03-06T21:04:10.546ZStates: Active Active Active
153382024-03-06T21:04:10.546ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
153392024-03-06T21:04:10.546Z 2 Acked 1001 Write 5 Done Sent Done false
153402024-03-06T21:04:10.546Z 3 Acked 1002 Flush 0 Done Sent Done false
153412024-03-06T21:04:10.546Z 4 Acked 1003 Write 5 Done Sent Done false
153422024-03-06T21:04:10.546Z 5 Acked 1004 Read 1 Done Sent Done false
153432024-03-06T21:04:10.546Z 6 Acked 1005 Read 6 Done Sent Done false
153442024-03-06T21:04:10.547Z 7 Acked 1006 Read 2 Done Sent Done false
153452024-03-06T21:04:10.547Z 8 Acked 1007 Read 9 Done Sent Done false
153462024-03-06T21:04:10.547Z 9 Acked 1008 Write 8 Done Sent Done false
153472024-03-06T21:04:10.547Z 10 Acked 1009 Read 10 Done Sent Done false
153482024-03-06T21:04:10.547Z 11 Acked 1010 Read 4 Done Sent Done false
153492024-03-06T21:04:10.577Z 12 Acked 1011 Read 8 Done Sent Done false
153502024-03-06T21:04:10.577Z 13 Acked 1012 Read 8 Done Sent Done false
153512024-03-06T21:04:10.577Z 14 Acked 1013 Write 6 Done Sent Done false
153522024-03-06T21:04:10.577Z 15 Acked 1014 Read 7 Done Sent Done false
153532024-03-06T21:04:10.577Z 16 Acked 1015 Read 9 Done Sent Done false
153542024-03-06T21:04:10.577Z 17 Acked 1016 Write 10 Done Sent Done false
153552024-03-06T21:04:10.577Z 18 Acked 1017 Read 8 Done Sent Done false
153562024-03-06T21:04:10.578Z 19 Acked 1018 Write 7 Done Sent Done false
153572024-03-06T21:04:10.578Z 20 Acked 1019 Flush 0 Done Sent Done false
153582024-03-06T21:04:10.578Z 21 Acked 1020 Write 8 Done Sent Done false
153592024-03-06T21:04:10.578Z 22 Acked 1021 Read 9 Done Sent Done false
153602024-03-06T21:04:10.578Z 23 Acked 1022 Flush 0 Done Sent Done false
153612024-03-06T21:04:10.578Z 24 Acked 1023 Write 6 Done Sent Done false
153622024-03-06T21:04:10.578Z 25 Acked 1024 Read 4 Done Sent Done false
153632024-03-06T21:04:10.578Z 26 Acked 1025 Write 8 Done Sent Done false
153642024-03-06T21:04:10.578Z 27 Acked 1026 Read 10 Done Sent Done false
153652024-03-06T21:04:10.578Z 28 Acked 1027 Read 4 Done Sent Done false
153662024-03-06T21:04:10.579Z 29 Acked 1028 Write 2 Done Sent Done false
153672024-03-06T21:04:10.579Z 30 Acked 1029 Read 8 Done Sent Done false
153682024-03-06T21:04:10.579Z STATES DS:0 DS:1 DS:2 TOTAL
153692024-03-06T21:04:10.579Z New 0 0 0 0
153702024-03-06T21:04:10.579Z Sent 0 29 0 29
153712024-03-06T21:04:10.579Z Done 29 0 29 58
153722024-03-06T21:04:10.579Z Skipped 0 0 0 0
153732024-03-06T21:04:10.579Z Error 0 0 0 0
153742024-03-06T21:04:10.579ZLast Flush: 1022 1000 1022
153752024-03-06T21:04:10.579ZDownstairs last five completed: 1000
153762024-03-06T21:04:10.579ZUpstairs last five completed: 30 29 28 27 26
153772024-03-06T21:04:10.580Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:04:08.993126367Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1707}
153782024-03-06T21:04:10.580ZWrote out file "/tmp/test_repair_verify.data"
153792024-03-06T21:04:10.580Z
153802024-03-06T21:04:10.580ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
153812024-03-06T21:04:10.580ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
153822024-03-06T21:04:10.580Z 0 000-019 142 135 142 252 235 252 F F F <---
153832024-03-06T21:04:10.580Z 1 020-039 141 141 141 248 248 248 F F F
153842024-03-06T21:04:10.580Z 2 040-059 141 141 141 248 248 248 T F T <---
153852024-03-06T21:04:10.580Z 3 060-079 140 140 140 245 245 245 F F F
153862024-03-06T21:04:10.580Z 4 080-099 142 138 142 251 240 251 F F F <---
153872024-03-06T21:04:10.580Z 5 100-119 136 136 136 236 236 236 F F F
153882024-03-06T21:04:10.581Z 6 120-139 137 137 137 238 238 238 F F F
153892024-03-06T21:04:10.581Z 7 140-159 134 134 134 234 234 234 F F F
153902024-03-06T21:04:10.581Z 8 160-179 141 141 141 248 248 248 F F F
153912024-03-06T21:04:10.581Z 9 180-199 141 141 141 248 248 248 F F F
153922024-03-06T21:04:10.581Z 10 200-219 136 136 136 237 237 237 F F F
153932024-03-06T21:04:10.581Z 11 220-239 132 132 132 229 229 229 F F F
153942024-03-06T21:04:10.581Z 12 240-259 142 138 142 250 239 250 F F F <---
153952024-03-06T21:04:10.581Z 13 260-279 142 138 142 251 242 251 F F F <---
153962024-03-06T21:04:10.581Z 14 280-299 136 136 136 236 236 236 F F F
153972024-03-06T21:04:10.581Z 15 300-319 141 141 141 248 248 248 F F F
153982024-03-06T21:04:10.581Z 16 320-339 141 141 141 248 248 248 F F F
153992024-03-06T21:04:10.581Z 17 340-359 140 140 140 247 247 247 F F F
154002024-03-06T21:04:10.581Z 18 360-379 135 135 135 235 235 235 F F F
154012024-03-06T21:04:10.582Z 19 380-399 135 135 135 235 235 235 F F F
154022024-03-06T21:04:10.582Z 20 400-419 136 136 136 236 236 236 F F F
154032024-03-06T21:04:10.582Z 21 420-439 132 132 132 231 231 231 F F F
154042024-03-06T21:04:10.582Z 22 440-459 141 141 141 248 248 248 T F T <---
154052024-03-06T21:04:10.582Z 23 460-479 142 141 142 251 248 251 T F T <---
154062024-03-06T21:04:10.582Z 24 480-499 135 135 135 235 235 235 F F F
154072024-03-06T21:04:10.582Z 25 500-519 139 139 139 244 244 244 F F F
154082024-03-06T21:04:10.582Z 26 520-539 142 138 142 251 239 251 T F T <---
154092024-03-06T21:04:10.583Z 27 540-559 140 140 140 246 246 246 F F F
154102024-03-06T21:04:10.583Z 28 560-579 138 138 138 242 242 242 F F F
154112024-03-06T21:04:10.583Z 29 580-599 134 134 134 234 234 234 F F F
154122024-03-06T21:04:10.583ZMax gen: 142, Max flush: 252
154132024-03-06T21:04:10.583ZA Difference in extent metadata is expected here
154142024-03-06T21:04:10.583ZOn loop 71
154152024-03-06T21:04:10.583Z
154162024-03-06T21:04:10.583ZVerifying data now
154172024-03-06T21:04:14.752ZNo change after verify
154182024-03-06T21:04:14.752ZLoop: 71 Downstairs dump after verify (and repair):
154192024-03-06T21:04:14.767ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
154202024-03-06T21:04:14.767Z 0 000-019 142 142 142 252 252 252 F F F
154212024-03-06T21:04:14.767Z 1 020-039 141 141 141 248 248 248 F F F
154222024-03-06T21:04:14.767Z 2 040-059 143 143 143 253 253 253 F F F
154232024-03-06T21:04:14.767Z 3 060-079 140 140 140 245 245 245 F F F
154242024-03-06T21:04:14.767Z 4 080-099 142 142 142 251 251 251 F F F
154252024-03-06T21:04:14.767Z 5 100-119 136 136 136 236 236 236 F F F
154262024-03-06T21:04:14.767Z 6 120-139 137 137 137 238 238 238 F F F
154272024-03-06T21:04:14.767Z 7 140-159 134 134 134 234 234 234 F F F
154282024-03-06T21:04:14.767Z 8 160-179 141 141 141 248 248 248 F F F
154292024-03-06T21:04:14.767Z 9 180-199 141 141 141 248 248 248 F F F
154302024-03-06T21:04:14.767Z 10 200-219 136 136 136 237 237 237 F F F
154312024-03-06T21:04:14.767Z 11 220-239 132 132 132 229 229 229 F F F
154322024-03-06T21:04:14.767Z 12 240-259 142 142 142 250 250 250 F F F
154332024-03-06T21:04:14.767Z 13 260-279 142 142 142 251 251 251 F F F
154342024-03-06T21:04:14.767Z 14 280-299 136 136 136 236 236 236 F F F
154352024-03-06T21:04:14.767Z 15 300-319 141 141 141 248 248 248 F F F
154362024-03-06T21:04:14.767Z 16 320-339 141 141 141 248 248 248 F F F
154372024-03-06T21:04:14.768Z 17 340-359 140 140 140 247 247 247 F F F
154382024-03-06T21:04:14.768Z 18 360-379 135 135 135 235 235 235 F F F
154392024-03-06T21:04:14.771Z 19 380-399 135 135 135 235 235 235 F F F
154402024-03-06T21:04:14.771Z 20 400-419 136 136 136 236 236 236 F F F
154412024-03-06T21:04:14.771Z 21 420-439 132 132 132 231 231 231 F F F
154422024-03-06T21:04:14.771Z 22 440-459 143 143 143 253 253 253 F F F
154432024-03-06T21:04:14.771Z 23 460-479 143 143 143 253 253 253 F F F
154442024-03-06T21:04:14.771Z 24 480-499 135 135 135 235 235 235 F F F
154452024-03-06T21:04:14.771Z 25 500-519 139 139 139 244 244 244 F F F
154462024-03-06T21:04:14.771Z 26 520-539 143 143 143 253 253 253 F F F
154472024-03-06T21:04:14.772Z 27 540-559 140 140 140 246 246 246 F F F
154482024-03-06T21:04:14.772Z 28 560-579 138 138 138 242 242 242 F F F
154492024-03-06T21:04:14.772Z 29 580-599 134 134 134 234 234 234 F F F
154502024-03-06T21:04:14.772ZMax gen: 143, Max flush: 253
154512024-03-06T21:04:14.772Z
154522024-03-06T21:04:14.772ZBegin loop 72
154532024-03-06T21:04:14.772ZDownstairs to restart: 1
154542024-03-06T21:04:14.799Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.541811096Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715}
154552024-03-06T21:04:14.799Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",Crucible runtime is spawned
154562024-03-06T21:04:15.216Z\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.542170071Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715}
154572024-03-06T21:04:15.216Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.542216318Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715}
154582024-03-06T21:04:15.216Z{"msg":"Crucible 2b81f564-cc34-4a28-9efd-cf63576b5730 has session id: e1c43b64-da8d-4479-8816-cb339cc2d695","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.542241966Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154592024-03-06T21:04:15.216Z{"msg":"Upstairs opts: Upstairs UUID: 2b81f564-cc34-4a28-9efd-cf63576b5730, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.542289613Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154602024-03-06T21:04:15.216Z{"msg":"Crucible stats registered with UUID: 2b81f564-cc34-4a28-9efd-cf63576b5730","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.54232201Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154612024-03-06T21:04:15.216Z{"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.542343119Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715}
154622024-03-06T21:04:15.216Z{"msg":"2b81f564-cc34-4a28-9efd-cf63576b5730 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.542381546Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154632024-03-06T21:04:15.216Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.542418184Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"client":"0","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154642024-03-06T21:04:15.217Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.542435463Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"client":"0","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154652024-03-06T21:04:15.217Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.542465141Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"client":"1","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154662024-03-06T21:04:15.217Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.542489719Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"client":"1","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154672024-03-06T21:04:15.217Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.542533556Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"client":"2","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154682024-03-06T21:04:15.217Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.542553925Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"client":"2","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154692024-03-06T21:04:15.217Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.542577383Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154702024-03-06T21:04:15.217Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.54261493Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154712024-03-06T21:04:15.217Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.542630999Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154722024-03-06T21:04:15.217Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.542842465Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154732024-03-06T21:04:15.217Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.543075409Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154742024-03-06T21:04:15.217Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.543126985Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154752024-03-06T21:04:15.217Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.543682317Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"client":"2","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154762024-03-06T21:04:15.217Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.543738453Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"client":"2","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154772024-03-06T21:04:15.218Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.543765381Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"client":"0","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154782024-03-06T21:04:15.218Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.543801488Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"client":"0","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154792024-03-06T21:04:15.218Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.544274916Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"client":"0","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154802024-03-06T21:04:15.218Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.544415056Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"client":"1","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154812024-03-06T21:04:15.218Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.544468793Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"client":"1","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154822024-03-06T21:04:15.218Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.54450757Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"client":"2","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154832024-03-06T21:04:15.218Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.544721225Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"client":"0","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154842024-03-06T21:04:15.218Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.544762672Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"client":"0","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154852024-03-06T21:04:15.218Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.544785401Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154862024-03-06T21:04:15.218Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.54508757Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"client":"2","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154872024-03-06T21:04:15.218Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.545116948Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"client":"2","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154882024-03-06T21:04:15.218Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.545132097Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154892024-03-06T21:04:15.218Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.545150106Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"client":"1","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154902024-03-06T21:04:15.218Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.545571126Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"client":"1","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154912024-03-06T21:04:15.218Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.545620913Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"client":"1","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154922024-03-06T21:04:15.218Z{"msg":"[0]R flush_numbers[0..12]: [252, 248, 253, 245, 251, 236, 238, 234, 248, 248, 237, 229]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.545644192Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"":"downstairs","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154932024-03-06T21:04:15.218Z{"msg":"[0]R generation[0..12]: [142, 141, 143, 140, 142, 136, 137, 134, 141, 141, 136, 132]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.54566894Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"":"downstairs","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154942024-03-06T21:04:15.218Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.545684259Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"":"downstairs","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154952024-03-06T21:04:15.218Z{"msg":"[1]R flush_numbers[0..12]: [252, 248, 253, 245, 251, 236, 238, 234, 248, 248, 237, 229]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.545699098Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"":"downstairs","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154962024-03-06T21:04:15.218Z{"msg":"[1]R generation[0..12]: [142, 141, 143, 140, 142, 136, 137, 134, 141, 141, 136, 132]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.545712847Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"":"downstairs","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154972024-03-06T21:04:15.219Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.545733845Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"":"downstairs","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154982024-03-06T21:04:15.219Z{"msg":"[2]R flush_numbers[0..12]: [252, 248, 253, 245, 251, 236, 238, 234, 248, 248, 237, 229]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.545747984Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"":"downstairs","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
154992024-03-06T21:04:15.219Z{"msg":"[2]R generation[0..12]: [142, 141, 143, 140, 142, 136, 137, 134, 141, 141, 136, 132]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.545761624Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"":"downstairs","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
155002024-03-06T21:04:15.219Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
155012024-03-06T21:04:15.219Z,"time":"2024-03-06T21:04:13.545802721Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"":"downstairs","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
155022024-03-06T21:04:15.219Z{"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.545822949Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"":"downstairs","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
155032024-03-06T21:04:15.219Z{"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.545850757Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"":"downstairs","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
155042024-03-06T21:04:15.219Z{"msg":"Next flush: 254","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.545869366Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"":"downstairs","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
155052024-03-06T21:04:15.219Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.545895944Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"":"downstairs","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
155062024-03-06T21:04:15.219Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.545914753Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
155072024-03-06T21:04:15.219Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.54595174Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"client":"0","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
155082024-03-06T21:04:15.219Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.545986838Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"client":"1","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
155092024-03-06T21:04:15.219Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.546009926Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"client":"2","":"downstairs","session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
155102024-03-06T21:04:15.219Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.546042034Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
155112024-03-06T21:04:15.219Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.546063523Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
155122024-03-06T21:04:15.219Z{"msg":"2b81f564-cc34-4a28-9efd-cf63576b5730 is now active with session: e1c43b64-da8d-4479-8816-cb339cc2d695","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.546093311Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
155132024-03-06T21:04:15.219Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.546115109Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715,"session_id":"e1c43b64-da8d-4479-8816-cb339cc2d695"}
155142024-03-06T21:04:15.219Z{"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:13.546141167Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715}
155152024-03-06T21:04:15.220ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
155162024-03-06T21:04:15.220ZLoading write count information from file "/tmp/test_repair_verify.data"
155172024-03-06T21:04:15.220ZRun Repair workload
155182024-03-06T21:04:15.220Z01/30 Flush
155192024-03-06T21:04:15.220Z02/30 Write block 169 len 20480 data: 8 8 9 11 15
155202024-03-06T21:04:15.232Z03/30 Read block 34 len 36864
155212024-03-06T21:04:15.232Z04/30 Write block 433 len 16384 data: 10 13 11 12
155222024-03-06T21:04:15.232Z05/30 Write block 571 len 36864 data: 10 10 10 9 10 12 9 7 8
155232024-03-06T21:04:15.232Z06/30 Read block 440 len 20480
155242024-03-06T21:04:15.232Z07/30 Read block 208 len 24576
155252024-03-06T21:04:15.232Z08/30 Write block 93 len 40960 data: 8 8 7 9 8 10 10 7 10 10
155262024-03-06T21:04:15.233Z09/30 Write block 175 len 32768 data: 15 13 11 11 15 15 17 15
155272024-03-06T21:04:15.233Z10/30 Read block 212 len 12288
155282024-03-06T21:04:15.233Z11/30 Write block 566 len 12288 data: 13 9 7
155292024-03-06T21:04:15.233Z12/30 Write block 35 len 36864 data: 9 5 3 7 9 10 13 14 13
155302024-03-06T21:04:15.233Z13/30 Read block 488 len 28672
155312024-03-06T21:04:15.233Z14/30 Write block 216 len 12288 data: 15 16 13
155322024-03-06T21:04:15.233Z15/30 Write block 245 len 8192 data: 12 11
155332024-03-06T21:04:15.233Z16/30 Flush
155342024-03-06T21:04:15.233Z17/30 Write block 467 len 8192 data: 12 11
155352024-03-06T21:04:15.234Z18/30 Write block 27 len 16384 data: 11 10 13 11
155362024-03-06T21:04:15.234Z19/30 Read block 120 len 4096
155372024-03-06T21:04:15.234Z20/30 Read block 335 len 32768
155382024-03-06T21:04:15.234Z21/30 Flush
155392024-03-06T21:04:15.234Z22/30 Write block 324 len 40960 data: 13 12 15 12 12 11 8 7 7 9
155402024-03-06T21:04:15.234Z23/30 Write block 95 len 8192 data: 8 10
155412024-03-06T21:04:15.234Z24/30 Read block 548 len 36864
155422024-03-06T21:04:15.234Z25/30 Write block 285 len 16384 data: 14 15 15 16
155432024-03-06T21:04:15.234Z26/30 Read block 228 len 8192
155442024-03-06T21:04:15.234Z27/30 Write block 383 len 32768 data: 14 15 15 14 11 11 10 12
155452024-03-06T21:04:15.235Z28/30 Write block 236 len 36864 data: 14 16 19 19 15 15 15 14 13
155462024-03-06T21:04:15.235Z29/30 Write block 459 len 40960 data: 15 14 17 14 19 19 17 13 13 12
155472024-03-06T21:04:15.235Z30/30 Read block 551 len 20480
155482024-03-06T21:04:15.235Z----------------------------------------------------------------
155492024-03-06T21:04:15.235Z Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:29
155502024-03-06T21:04:15.235ZStates: Active Active Active
155512024-03-06T21:04:15.236ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
155522024-03-06T21:04:15.236Z 2 Acked 1001 Write 5 Done Done Done false
155532024-03-06T21:04:15.236Z 3 Acked 1002 Read 9 Done Done Done false
155542024-03-06T21:04:15.236Z 4 Acked 1003 Write 4 Done Done Done false
155552024-03-06T21:04:15.236Z 5 Acked 1004 Write 9 Done Sent Done false
155562024-03-06T21:04:15.236Z 6 Acked 1005 Read 5 Done Sent Done false
155572024-03-06T21:04:15.236Z 7 Acked 1006 Read 6 Done Sent Done false
155582024-03-06T21:04:15.236Z 8 Acked 1007 Write 10 Done Sent Done false
155592024-03-06T21:04:15.236Z 9 Acked 1008 Write 8 Done Sent Done false
155602024-03-06T21:04:15.236Z 10 Acked 1009 Read 3 Done Sent Done false
155612024-03-06T21:04:15.237Z 11 Acked 1010 Write 3 Done Sent Done false
155622024-03-06T21:04:15.237Z 12 Acked 1011 Write 9 Done Sent Done false
155632024-03-06T21:04:15.237Z 13 Acked 1012 Read 7 Done Sent Done false
155642024-03-06T21:04:15.237Z 14 Acked 1013 Write 3 Done Sent Done false
155652024-03-06T21:04:15.237Z 15 Acked 1014 Write 2 Done Sent Done false
155662024-03-06T21:04:15.237Z 16 Acked 1015 Flush 0 Done Sent Done false
155672024-03-06T21:04:15.237Z 17 Acked 1016 Write 2 Done Sent Done false
155682024-03-06T21:04:15.237Z 18 Acked 1017 Write 4 Done Sent Done false
155692024-03-06T21:04:15.237Z 19 Acked 1018 Read 1 Done Sent Done false
155702024-03-06T21:04:15.237Z 20 Acked 1019 Read 8 Done Sent Done false
155712024-03-06T21:04:15.238Z 21 Acked 1020 Flush 0 Done Sent Done false
155722024-03-06T21:04:15.238Z 22 Acked 1021 Write 10 Done Sent Done false
155732024-03-06T21:04:15.238Z 23 Acked 1022 Write 2 Done Sent Done false
155742024-03-06T21:04:15.238Z 24 Acked 1023 Read 9 Done Sent Done false
155752024-03-06T21:04:15.238Z 25 Acked 1024 Write 4 Done Sent Done false
155762024-03-06T21:04:15.238Z 26 Acked 1025 Read 2 Done Sent Done false
155772024-03-06T21:04:15.238Z 27 Acked 1026 Write 8 Done Sent Done false
155782024-03-06T21:04:15.238Z 28 Acked 1027 Write 9 Done Sent Done false
155792024-03-06T21:04:15.238Z 29 Acked 1028 Write 10 Done Sent Done false
155802024-03-06T21:04:15.238Z 30 Acked 1029 Read 5 Done Sent Sent false
155812024-03-06T21:04:15.238Z STATES DS:0 DS:1 DS:2 TOTAL
155822024-03-06T21:04:15.238Z New 0 0 0 0
155832024-03-06T21:04:15.239Z Sent 0 26 1 27
155842024-03-06T21:04:15.239Z Done 29 3 28 60
155852024-03-06T21:04:15.239Z Skipped 0 0 0 0
155862024-03-06T21:04:15.239Z Error 0 0 0 0
155872024-03-06T21:04:15.239ZLast Flush: 1020 1000 1020
155882024-03-06T21:04:15.239ZDownstairs last five completed: 1000
155892024-03-06T21:04:15.239ZUpstairs last five completed: 30 29 28 27 26
155902024-03-06T21:04:15.239Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:04:13.583024145Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1715}
155912024-03-06T21:04:15.239ZWrote out file "/tmp/test_repair_verify.data"
155922024-03-06T21:04:15.239Z
155932024-03-06T21:04:15.239ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
155942024-03-06T21:04:15.240ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
155952024-03-06T21:04:15.240Z 0 000-019 142 142 142 252 252 252 F F F
155962024-03-06T21:04:15.240Z 1 020-039 144 141 144 256 248 256 F F F <---
155972024-03-06T21:04:15.240Z 2 040-059 144 143 144 255 253 255 F F F <---
155982024-03-06T21:04:15.240Z 3 060-079 140 140 140 245 245 245 F F F
155992024-03-06T21:04:15.240Z 4 080-099 144 142 144 255 251 255 T F T <---
156002024-03-06T21:04:15.240Z 5 100-119 144 136 144 255 236 255 F F F <---
156012024-03-06T21:04:15.240Z 6 120-139 137 137 137 238 238 238 F F F
156022024-03-06T21:04:15.240Z 7 140-159 134 134 134 234 234 234 F F F
156032024-03-06T21:04:15.240Z 8 160-179 144 141 144 255 248 255 F T F <---
156042024-03-06T21:04:15.240Z 9 180-199 144 141 144 255 248 255 F F F <---
156052024-03-06T21:04:15.241Z 10 200-219 144 136 144 255 237 255 F F F <---
156062024-03-06T21:04:15.241Z 11 220-239 132 132 132 229 229 229 T F T <---
156072024-03-06T21:04:15.241Z 12 240-259 144 142 144 255 250 255 T F T <---
156082024-03-06T21:04:15.241Z 13 260-279 142 142 142 251 251 251 F F F
156092024-03-06T21:04:15.241Z 14 280-299 136 136 136 236 236 236 T F T <---
156102024-03-06T21:04:15.241Z 15 300-319 141 141 141 248 248 248 F F F
156112024-03-06T21:04:15.241Z 16 320-339 141 141 141 248 248 248 T F T <---
156122024-03-06T21:04:15.241Z 17 340-359 140 140 140 247 247 247 F F F
156132024-03-06T21:04:15.241Z 18 360-379 135 135 135 235 235 235 F F F
156142024-03-06T21:04:15.241Z 19 380-399 135 135 135 235 235 235 T F T <---
156152024-03-06T21:04:15.241Z 20 400-419 136 136 136 236 236 236 F F F
156162024-03-06T21:04:15.242Z 21 420-439 144 132 144 255 231 255 F T F <---
156172024-03-06T21:04:15.242Z 22 440-459 143 143 143 253 253 253 T F T <---
156182024-03-06T21:04:15.242Z 23 460-479 144 143 144 256 253 256 T F T <---
156192024-03-06T21:04:15.242Z 24 480-499 135 135 135 235 235 235 F F F
156202024-03-06T21:04:15.253Z 25 500-519 139 139 139 244 244 244 F F F
156212024-03-06T21:04:15.253Z 26 520-539 143 143 143 253 253 253 F F F
156222024-03-06T21:04:15.253Z 27 540-559 140 140 140 246 246 246 F F F
156232024-03-06T21:04:15.253Z 28 560-579 144 138 144 255 242 255 F F F <---
156242024-03-06T21:04:15.253Z 29 580-599 134 134 134 234 234 234 F F F
156252024-03-06T21:04:15.253ZMax gen: 144, Max flush: 256
156262024-03-06T21:04:15.253ZError: Difference in extent metadata found!
156272024-03-06T21:04:15.254ZA Difference in extent metadata is expected here
156282024-03-06T21:04:15.254ZOn loop 72
156292024-03-06T21:04:15.254Z
156302024-03-06T21:04:15.254ZVerifying data now
156312024-03-06T21:04:20.700ZNo change after verify
156322024-03-06T21:04:20.700ZLoop: 72 Downstairs dump after verify (and repair):
156332024-03-06T21:04:20.715ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
156342024-03-06T21:04:20.715Z 0 000-019 142 142 142 252 252 252 F F F
156352024-03-06T21:04:20.715Z 1 020-039 144 144 144 256 256 256 F F F
156362024-03-06T21:04:20.715Z 2 040-059 144 144 144 255 255 255 F F F
156372024-03-06T21:04:20.715Z 3 060-079 140 140 140 245 245 245 F F F
156382024-03-06T21:04:20.715Z 4 080-099 145 145 145 257 257 257 F F F
156392024-03-06T21:04:20.715Z 5 100-119 144 144 144 255 255 255 F F F
156402024-03-06T21:04:20.715Z 6 120-139 137 137 137 238 238 238 F F F
156412024-03-06T21:04:20.715Z 7 140-159 134 134 134 234 234 234 F F F
156422024-03-06T21:04:20.715Z 8 160-179 144 144 144 255 255 255 F F F
156432024-03-06T21:04:20.715Z 9 180-199 144 144 144 255 255 255 F F F
156442024-03-06T21:04:20.719Z 10 200-219 144 144 144 255 255 255 F F F
156452024-03-06T21:04:20.719Z 11 220-239 145 145 145 257 257 257 F F F
156462024-03-06T21:04:20.719Z 12 240-259 145 145 145 257 257 257 F F F
156472024-03-06T21:04:20.719Z 13 260-279 142 142 142 251 251 251 F F F
156482024-03-06T21:04:20.719Z 14 280-299 145 145 145 257 257 257 F F F
156492024-03-06T21:04:20.719Z 15 300-319 141 141 141 248 248 248 F F F
156502024-03-06T21:04:20.719Z 16 320-339 145 145 145 257 257 257 F F F
156512024-03-06T21:04:20.719Z 17 340-359 140 140 140 247 247 247 F F F
156522024-03-06T21:04:20.719Z 18 360-379 135 135 135 235 235 235 F F F
156532024-03-06T21:04:20.720Z 19 380-399 145 145 145 257 257 257 F F F
156542024-03-06T21:04:20.720Z 20 400-419 136 136 136 236 236 236 F F F
156552024-03-06T21:04:20.720Z 21 420-439 144 144 144 255 255 255 F F F
156562024-03-06T21:04:20.720Z 22 440-459 145 145 145 257 257 257 F F F
156572024-03-06T21:04:20.720Z 23 460-479 145 145 145 257 257 257 F F F
156582024-03-06T21:04:20.720Z 24 480-499 135 135 135 235 235 235 F F F
156592024-03-06T21:04:20.720Z 25 500-519 139 139 139 244 244 244 F F F
156602024-03-06T21:04:20.720Z 26 520-539 143 143 143 253 253 253 F F F
156612024-03-06T21:04:20.720Z 27 540-559 140 140 140 246 246 246 F F F
156622024-03-06T21:04:20.720Z 28 560-579 144 144 144 255 255 255 F F F
156632024-03-06T21:04:20.720Z 29 580-599 134 134 134 234 234 234 F F F
156642024-03-06T21:04:20.720ZMax gen: 145, Max flush: 257
156652024-03-06T21:04:20.724Z
156662024-03-06T21:04:20.724ZBegin loop 73
156672024-03-06T21:04:20.724ZDownstairs to restart: 1
156682024-03-06T21:04:20.744Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.488330079Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723}
156692024-03-06T21:04:20.744Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
156702024-03-06T21:04:20.751Z,"time":"2024-03-06T21:04:19.488678545Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723}
156712024-03-06T21:04:20.751Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.488806726Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723}
156722024-03-06T21:04:20.751Z{"msg":"Crucible 16e2e3ec-cb1c-4318-bb4a-8a2696689a01 has session id: 9ff48a70-4517-465a-a79c-6fdbae25a4a6","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.488868652Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
156732024-03-06T21:04:20.751Z{"msg":"Upstairs opts: Upstairs UUID: 16e2e3ec-cb1c-4318-bb4a-8a2696689a01, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.488917619Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
156742024-03-06T21:04:20.751Z{"msg":"Crucible stats registered with UUID: 16e2e3ec-cb1c-4318-bb4a-8a2696689a01","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.488956436Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
156752024-03-06T21:04:20.752Z{"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.488981324Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723}
156762024-03-06T21:04:20.752Z{"msg":"16e2e3ec-cb1c-4318-bb4a-8a2696689a01 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.48904957Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
156772024-03-06T21:04:20.752Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.489089237Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"client":"0","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
156782024-03-06T21:04:20.752Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.489111365Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"client":"0","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
156792024-03-06T21:04:20.752Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.489151343Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"client":"1","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
156802024-03-06T21:04:20.752Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.489199669Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"client":"1","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
156812024-03-06T21:04:20.752Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.489239506Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"client":"2","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
156822024-03-06T21:04:20.752Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.489373087Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"client":"2","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
156832024-03-06T21:04:20.752Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.489410795Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
156842024-03-06T21:04:20.752Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.489459831Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
156852024-03-06T21:04:20.752Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.489490359Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
156862024-03-06T21:04:20.752Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.489516538Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
156872024-03-06T21:04:20.752Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.489558924Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
156882024-03-06T21:04:20.752Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.489592442Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
156892024-03-06T21:04:20.752Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.490267046Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"client":"2","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
156902024-03-06T21:04:20.752Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.490309143Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"client":"2","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
156912024-03-06T21:04:20.752Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.490343221Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"client":"0","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
156922024-03-06T21:04:20.752Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.490368449Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"client":"0","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
156932024-03-06T21:04:20.752Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.49093798Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"client":"2","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
156942024-03-06T21:04:20.753Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.490980177Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"client":"0","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
156952024-03-06T21:04:20.753Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.491532379Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"client":"2","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
156962024-03-06T21:04:20.753Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.491580515Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"client":"2","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
156972024-03-06T21:04:20.753Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.491610033Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
156982024-03-06T21:04:20.753Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.491647081Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"client":"0","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
156992024-03-06T21:04:20.753Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.491685778Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"client":"0","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
157002024-03-06T21:04:20.753Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.491721695Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
157012024-03-06T21:04:20.753Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.491745174Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"client":"1","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
157022024-03-06T21:04:20.753Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.491760193Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"client":"1","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
157032024-03-06T21:04:20.753Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.492158215Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"client":"1","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
157042024-03-06T21:04:20.753Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.492554918Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"client":"1","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
157052024-03-06T21:04:20.753Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.492587666Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"client":"1","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
157062024-03-06T21:04:20.753Z{"msg":"[0]R flush_numbers[0..12]: [252, 256, 255, 245, 257, 255, 238, 234, 255, 255, 255, 257]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.492609944Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"":"downstairs","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
157072024-03-06T21:04:20.753Z{"msg":"[0]R generation[0..12]: [142, 144, 144, 140, 145, 144, 137, 134, 144, 144, 144, 145]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.492643132Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"":"downstairs","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
157082024-03-06T21:04:20.753Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.49266695Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"":"downstairs","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
157092024-03-06T21:04:20.753Z{"msg":"[1]R flush_numbers[0..12]: [252, 256, 255, 245, 257, 255, 238, 234, 255, 255, 255, 257]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.492682509Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"":"downstairs","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
157102024-03-06T21:04:20.753Z{"msg":"[1]R generation[0..12]: [142, 144, 144, 140, 145, 144, 137, 134, 144, 144, 144, 145]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.492704948Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"":"downstairs","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
157112024-03-06T21:04:20.754Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.492720527Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"":"downstairs","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
157122024-03-06T21:04:20.754Z{"msg":"[2]R flush_numbers[0..12]: [252, 256, 255, 245, 257, 255, 238, 234, 255, 255, 255, 257]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.492742295Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"":"downstairs","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
157132024-03-06T21:04:20.754Z{"msg":"[2]R generation[0..12]: [142, 144, 144, 140, 145, 144, 137, 134, 144, 144, 144, 145]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
157142024-03-06T21:04:20.754Z2024-03-06T21:04:19.492756394Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"":"downstairs","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
157152024-03-06T21:04:20.754Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.492784212Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"":"downstairs","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
157162024-03-06T21:04:20.754Z{"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.492797821Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"":"downstairs","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
157172024-03-06T21:04:20.754Z{"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.49281879Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"":"downstairs","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
157182024-03-06T21:04:20.754Z{"msg":"Next flush: 258","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.492833219Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"":"downstairs","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
157192024-03-06T21:04:20.754Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.492853457Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"":"downstairs","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
157202024-03-06T21:04:20.754Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.492866976Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
157212024-03-06T21:04:20.754Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.492891285Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"client":"0","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
157222024-03-06T21:04:20.754Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.492922133Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"client":"1","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
157232024-03-06T21:04:20.754Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.49295521Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"client":"2","":"downstairs","session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
157242024-03-06T21:04:20.754Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.492982679Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
157252024-03-06T21:04:20.755Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.493003747Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
157262024-03-06T21:04:20.755Z{"msg":"16e2e3ec-cb1c-4318-bb4a-8a2696689a01 is now active with session: 9ff48a70-4517-465a-a79c-6fdbae25a4a6","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.493032935Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
157272024-03-06T21:04:20.755Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.493050394Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723,"session_id":"9ff48a70-4517-465a-a79c-6fdbae25a4a6"}
157282024-03-06T21:04:20.755Z{"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:19.493081452Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723}
157292024-03-06T21:04:20.755ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
157302024-03-06T21:04:20.761ZLoading write count information from file "/tmp/test_repair_verify.data"
157312024-03-06T21:04:20.761ZRun Repair workload
157322024-03-06T21:04:20.761Z01/30 Write block 524 len 20480 data: 10 10 11 12 15
157332024-03-06T21:04:20.761Z02/30 Read block 44 len 36864
157342024-03-06T21:04:20.762Z03/30 Flush
157352024-03-06T21:04:20.762Z04/30 Write block 96 len 12288 data: 11 9 11
157362024-03-06T21:04:20.762Z05/30 Read block 133 len 8192
157372024-03-06T21:04:20.762Z06/30 Write block 144 len 28672 data: 5 5 5 7 10 12 13
157382024-03-06T21:04:20.762Z07/30 Read block 578 len 32768
157392024-03-06T21:04:20.762Z08/30 Read block 188 len 24576
157402024-03-06T21:04:20.766Z09/30 Read block 302 len 4096
157412024-03-06T21:04:20.766Z10/30 Read block 436 len 28672
157422024-03-06T21:04:20.766Z11/30 Write block 178 len 16384 data: 12 16 16 18
157432024-03-06T21:04:20.766Z12/30 Flush
157442024-03-06T21:04:20.766Z13/30 Flush
157452024-03-06T21:04:20.766Z14/30 Write block 330 len 4096 data: 9
157462024-03-06T21:04:20.766Z15/30 Write block 90 len 12288 data: 6 9 9
157472024-03-06T21:04:20.769Z16/30 Read block 179 len 4096
157482024-03-06T21:04:20.769Z17/30 Read block 74 len 40960
157492024-03-06T21:04:20.770Z18/30 Write block 214 len 28672 data: 10 13 16 17 14 9 6
157502024-03-06T21:04:20.770Z19/30 Write block 567 len 20480 data: 10 8 7 9 11
157512024-03-06T21:04:20.773Z20/30 Read block 531 len 24576
157522024-03-06T21:04:20.773Z21/30 Read block 582 len 4096
157532024-03-06T21:04:20.773Z22/30 Read block 101 len 24576
157542024-03-06T21:04:20.776Z23/30 Write block 250 len 40960 data: 5 6 7 7 6 6 6 6 7 6
157552024-03-06T21:04:20.776Z24/30 Write block 375 len 32768 data: 15 11 9 10 15 15 14 14
157562024-03-06T21:04:20.776Z25/30 Read block 467 len 8192
157572024-03-06T21:04:20.779Z26/30 Write block 517 len 32768 data: 11 10 9 6 6 6 8 11
157582024-03-06T21:04:20.779Z27/30 Write block 529 len 36864 data: 14 14 13 11 11 11 13 11 12
157592024-03-06T21:04:20.780Z28/30 Write block 39 len 24576 data: 10 11 14 15 14 12
157602024-03-06T21:04:20.780Z29/30 Read block 345 len 24576
157612024-03-06T21:04:20.783Z30/30 Read block 250 len 28672
157622024-03-06T21:04:20.783Z----------------------------------------------------------------
157632024-03-06T21:04:20.783Z Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:18
157642024-03-06T21:04:20.783ZStates: Active Active Active
157652024-03-06T21:04:20.784ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
157662024-03-06T21:04:20.784Z 13 Acked 1012 Flush 0 Done Sent Done false
157672024-03-06T21:04:20.784Z 14 Acked 1013 Write 1 Done Sent Done false
157682024-03-06T21:04:20.784Z 15 Acked 1014 Write 3 Done Sent Done false
157692024-03-06T21:04:20.784Z 16 Acked 1015 Read 1 Done Sent Done false
157702024-03-06T21:04:20.784Z 17 Acked 1016 Read 10 Done Sent Done false
157712024-03-06T21:04:20.784Z 18 Acked 1017 Write 7 Done Sent Done false
157722024-03-06T21:04:20.784Z 19 Acked 1018 Write 5 Done Sent Done false
157732024-03-06T21:04:20.784Z 20 Acked 1019 Read 6 Done Sent Done false
157742024-03-06T21:04:20.784Z 21 Acked 1020 Read 1 Done Sent Done false
157752024-03-06T21:04:20.784Z 22 Acked 1021 Read 6 Done Sent Done false
157762024-03-06T21:04:20.784Z 23 Acked 1022 Write 10 Done Sent Done false
157772024-03-06T21:04:20.784Z 24 Acked 1023 Write 8 Done Sent Done false
157782024-03-06T21:04:20.784Z 25 Acked 1024 Read 2 Done Sent Done false
157792024-03-06T21:04:20.784Z 26 Acked 1025 Write 8 Done Sent Done false
157802024-03-06T21:04:20.784Z 27 Acked 1026 Write 9 Done Sent Done false
157812024-03-06T21:04:20.784Z 28 Acked 1027 Write 6 Done Sent Done false
157822024-03-06T21:04:20.784Z 29 Acked 1028 Read 6 Done Sent Done false
157832024-03-06T21:04:20.784Z 30 Acked 1029 Read 7 Sent Sent Done false
157842024-03-06T21:04:20.785Z STATES DS:0 DS:1 DS:2 TOTAL
157852024-03-06T21:04:20.785Z New 0 0 0 0
157862024-03-06T21:04:20.785Z Sent 1 18 0 19
157872024-03-06T21:04:20.785Z Done 17 0 18 35
157882024-03-06T21:04:20.785Z Skipped 0 0 0 0
157892024-03-06T21:04:20.785Z Error 0 0 0 0
157902024-03-06T21:04:20.785ZLast Flush: 1012 1011 1012
157912024-03-06T21:04:20.785ZDownstairs last five completed: 1011 1010 1009 1008 1007
157922024-03-06T21:04:20.785ZUpstairs last five completed: 30 29 28 27 26
157932024-03-06T21:04:20.785Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:04:19.527235098Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1723}
157942024-03-06T21:04:20.785ZWrote out file "/tmp/test_repair_verify.data"
157952024-03-06T21:04:20.805Z
157962024-03-06T21:04:20.805ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
157972024-03-06T21:04:20.814ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
157982024-03-06T21:04:20.814Z 0 000-019 142 142 142 252 252 252 F F F
157992024-03-06T21:04:20.814Z 1 020-039 144 144 144 256 256 256 T F T <---
158002024-03-06T21:04:20.814Z 2 040-059 144 144 144 255 255 255 T F T <---
158012024-03-06T21:04:20.814Z 3 060-079 140 140 140 245 245 245 F F F
158022024-03-06T21:04:20.814Z 4 080-099 146 146 146 259 259 259 T F T <---
158032024-03-06T21:04:20.815Z 5 100-119 144 144 144 255 255 255 F F F
158042024-03-06T21:04:20.815Z 6 120-139 137 137 137 238 238 238 F F F
158052024-03-06T21:04:20.815Z 7 140-159 146 146 146 259 259 259 F F F
158062024-03-06T21:04:20.815Z 8 160-179 146 146 146 259 259 259 F F F
158072024-03-06T21:04:20.815Z 9 180-199 146 146 146 259 259 259 F F F
158082024-03-06T21:04:20.815Z 10 200-219 144 144 144 255 255 255 T F T <---
158092024-03-06T21:04:20.815Z 11 220-239 145 145 145 257 257 257 T F T <---
158102024-03-06T21:04:20.815Z 12 240-259 145 145 145 257 257 257 T F T <---
158112024-03-06T21:04:20.815Z 13 260-279 142 142 142 251 251 251 F F F
158122024-03-06T21:04:20.815Z 14 280-299 145 145 145 257 257 257 F F F
158132024-03-06T21:04:20.815Z 15 300-319 141 141 141 248 248 248 F F F
158142024-03-06T21:04:20.819Z 16 320-339 145 145 145 257 257 257 T F T <---
158152024-03-06T21:04:20.819Z 17 340-359 140 140 140 247 247 247 F F F
158162024-03-06T21:04:20.819Z 18 360-379 135 135 135 235 235 235 T F T <---
158172024-03-06T21:04:20.819Z 19 380-399 145 145 145 257 257 257 T F T <---
158182024-03-06T21:04:20.819Z 20 400-419 136 136 136 236 236 236 F F F
158192024-03-06T21:04:20.819Z 21 420-439 144 144 144 255 255 255 F F F
158202024-03-06T21:04:20.819Z 22 440-459 145 145 145 257 257 257 F F F
158212024-03-06T21:04:20.819Z 23 460-479 145 145 145 257 257 257 F F F
158222024-03-06T21:04:20.819Z 24 480-499 135 135 135 235 235 235 F F F
158232024-03-06T21:04:20.819Z 25 500-519 139 139 139 244 244 244 T F T <---
158242024-03-06T21:04:20.819Z 26 520-539 146 146 146 258 258 258 T F T <---
158252024-03-06T21:04:20.819Z 27 540-559 140 140 140 246 246 246 F F F
158262024-03-06T21:04:20.819Z 28 560-579 144 144 144 255 255 255 T F T <---
158272024-03-06T21:04:20.819Z 29 580-599 134 134 134 234 234 234 F F F
158282024-03-06T21:04:20.819ZMax gen: 146, Max flush: 259
158292024-03-06T21:04:20.819ZError: Difference in extent metadata found!
158302024-03-06T21:04:20.819ZA Difference in extent metadata is expected here
158312024-03-06T21:04:20.820ZOn loop 73
158322024-03-06T21:04:20.820Z
158332024-03-06T21:04:20.823ZVerifying data now
158342024-03-06T21:04:30.743ZNo change after verify
158352024-03-06T21:04:30.743ZLoop: 73 Downstairs dump after verify (and repair):
158362024-03-06T21:04:30.744ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
158372024-03-06T21:04:30.744Z 0 000-019 142 142 142 252 252 252 F F F
158382024-03-06T21:04:30.744Z 1 020-039 147 147 147 260 260 260 F F F
158392024-03-06T21:04:30.744Z 2 040-059 147 147 147 260 260 260 F F F
158402024-03-06T21:04:30.744Z 3 060-079 140 140 140 245 245 245 F F F
158412024-03-06T21:04:30.744Z 4 080-099 147 147 147 260 260 260 F F F
158422024-03-06T21:04:30.745Z 5 100-119 144 144 144 255 255 255 F F F
158432024-03-06T21:04:30.745Z 6 120-139 137 137 137 238 238 238 F F F
158442024-03-06T21:04:30.745Z 7 140-159 146 146 146 259 259 259 F F F
158452024-03-06T21:04:30.745Z 8 160-179 146 146 146 259 259 259 F F F
158462024-03-06T21:04:30.745Z 9 180-199 146 146 146 259 259 259 F F F
158472024-03-06T21:04:30.745Z 10 200-219 147 147 147 260 260 260 F F F
158482024-03-06T21:04:30.745Z 11 220-239 147 147 147 260 260 260 F F F
158492024-03-06T21:04:30.745Z 12 240-259 147 147 147 260 260 260 F F F
158502024-03-06T21:04:30.745Z 13 260-279 142 142 142 251 251 251 F F F
158512024-03-06T21:04:30.745Z 14 280-299 145 145 145 257 257 257 F F F
158522024-03-06T21:04:30.746Z 15 300-319 141 141 141 248 248 248 F F F
158532024-03-06T21:04:30.746Z 16 320-339 147 147 147 260 260 260 F F F
158542024-03-06T21:04:30.746Z 17 340-359 140 140 140 247 247 247 F F F
158552024-03-06T21:04:30.746Z 18 360-379 147 147 147 260 260 260 F F F
158562024-03-06T21:04:30.746Z 19 380-399 147 147 147 260 260 260 F F F
158572024-03-06T21:04:30.746Z 20 400-419 136 136 136 236 236 236 F F F
158582024-03-06T21:04:30.746Z 21 420-439 144 144 144 255 255 255 F F F
158592024-03-06T21:04:30.746Z 22 440-459 145 145 145 257 257 257 F F F
158602024-03-06T21:04:30.746Z 23 460-479 145 145 145 257 257 257 F F F
158612024-03-06T21:04:30.746Z 24 480-499 135 135 135 235 235 235 F F F
158622024-03-06T21:04:30.746Z 25 500-519 147 147 147 260 260 260 F F F
158632024-03-06T21:04:30.746Z 26 520-539 147 147 147 260 260 260 F F F
158642024-03-06T21:04:30.746Z 27 540-559 140 140 140 246 246 246 F F F
158652024-03-06T21:04:30.746Z 28 560-579 147 147 147 260 260 260 F F F
158662024-03-06T21:04:30.746Z 29 580-599 134 134 134 234 234 234 F F F
158672024-03-06T21:04:30.746ZMax gen: 147, Max flush: 260
158682024-03-06T21:04:30.747Z
158692024-03-06T21:04:30.747ZBegin loop 74
158702024-03-06T21:04:30.747ZDownstairs to restart: 2
158712024-03-06T21:04:30.747Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.267630034Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731}
158722024-03-06T21:04:30.747Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\"Crucible runtime is spawned
158732024-03-06T21:04:30.747Z,\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.268055725Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731}
158742024-03-06T21:04:30.747Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.268114401Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731}
158752024-03-06T21:04:30.747Z{"msg":"Crucible f434aeb6-985c-4993-bcfe-cef01d45819b has session id: 18080ab1-3784-4cb1-a160-ea0f450da3d7","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.268163557Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
158762024-03-06T21:04:30.747Z{"msg":"Upstairs opts: Upstairs UUID: f434aeb6-985c-4993-bcfe-cef01d45819b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.268213764Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
158772024-03-06T21:04:30.747Z{"msg":"Crucible stats registered with UUID: f434aeb6-985c-4993-bcfe-cef01d45819b","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.268259051Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
158782024-03-06T21:04:30.747Z{"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.268292488Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731}
158792024-03-06T21:04:31.313Z{"msg":"f434aeb6-985c-4993-bcfe-cef01d45819b active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.268331506Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
158802024-03-06T21:04:31.313Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.268349755Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"client":"0","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
158812024-03-06T21:04:31.313Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.268381352Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"client":"0","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
158822024-03-06T21:04:31.314Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.268423649Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"client":"1","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
158832024-03-06T21:04:31.314Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.268468186Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"client":"1","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
158842024-03-06T21:04:31.314Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.268504644Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"client":"2","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
158852024-03-06T21:04:31.314Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.268538091Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"client":"2","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
158862024-03-06T21:04:31.314Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.26856413Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"":"io task","client":"2","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
158872024-03-06T21:04:31.314Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.268597638Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"":"io task","client":"0","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
158882024-03-06T21:04:31.314Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.268624636Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"":"io task","client":"1","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
158892024-03-06T21:04:31.314Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.268652784Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"":"io task","client":"2","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
158902024-03-06T21:04:31.314Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.26869662Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"":"io task","client":"0","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
158912024-03-06T21:04:31.314Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.268734218Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"":"io task","client":"1","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
158922024-03-06T21:04:31.314Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.269091663Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"client":"0","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
158932024-03-06T21:04:31.314Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.269128201Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"client":"0","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
158942024-03-06T21:04:31.314Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.269238383Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"client":"1","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
158952024-03-06T21:04:31.315Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.26928834Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"client":"1","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
158962024-03-06T21:04:31.315Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.269622237Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"client":"0","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
158972024-03-06T21:04:31.315Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.269747858Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"client":"1","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
158982024-03-06T21:04:31.315Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.270134701Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"client":"0","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
158992024-03-06T21:04:31.315Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.270196827Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"client":"0","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159002024-03-06T21:04:31.315Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.270248124Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159012024-03-06T21:04:31.315Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.270294721Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"client":"1","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159022024-03-06T21:04:31.315Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.270332538Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"client":"1","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159032024-03-06T21:04:31.316Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.270347117Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159042024-03-06T21:04:31.316Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.270713501Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"client":"2","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159052024-03-06T21:04:31.316Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.270760898Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"client":"2","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159062024-03-06T21:04:31.316Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.271211877Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"client":"2","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159072024-03-06T21:04:31.316Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.2716086Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"client":"2","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159082024-03-06T21:04:31.316Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.271642337Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"client":"2","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159092024-03-06T21:04:31.316Z{"msg":"[0]R flush_numbers[0..12]: [252, 260, 260, 245, 260, 255, 238, 259, 259, 259, 260, 260]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.271658456Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"":"downstairs","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159102024-03-06T21:04:31.317Z{"msg":"[0]R generation[0..12]: [142, 147, 147, 140, 147, 144, 137, 146, 146, 146, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.271685155Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"":"downstairs","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159112024-03-06T21:04:31.317Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.271707913Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"":"downstairs","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159122024-03-06T21:04:31.317Z{"msg":"[1]R flush_numbers[0..12]: [252, 260, 260, 245, 260, 255, 238, 259, 259, 259, 260, 260]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.271730242Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"":"downstairs","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159132024-03-06T21:04:31.317Z{"msg":"[1]R generation[0..12]: [142, 147, 147, 140, 147, 144, 137, 146, 146, 146, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.27175149Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"":"downstairs","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159142024-03-06T21:04:31.317Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.271767139Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"":"downstairs","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159152024-03-06T21:04:31.317Z{"msg":"[2]R flush_numbers[0..12]: [252, 260, 260, 245, 260, 255, 238, 259, 259, 259, 260, 260]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
159162024-03-06T21:04:31.318Z,"time":"2024-03-06T21:04:24.271787847Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"":"downstairs","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159172024-03-06T21:04:31.318Z{"msg":"[2]R generation[0..12]: [142, 147, 147, 140, 147, 144, 137, 146, 146, 146, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.271802326Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"":"downstairs","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159182024-03-06T21:04:31.318Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.271822905Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"":"downstairs","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159192024-03-06T21:04:31.319Z{"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.271842524Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"":"downstairs","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159202024-03-06T21:04:31.319Z{"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.271856653Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"":"downstairs","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159212024-03-06T21:04:31.319Z{"msg":"Next flush: 261","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.271880001Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"":"downstairs","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159222024-03-06T21:04:31.319Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.271913409Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"":"downstairs","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159232024-03-06T21:04:31.320Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.271947636Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159242024-03-06T21:04:31.320Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.271979294Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"client":"0","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159252024-03-06T21:04:31.320Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.272031301Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"client":"1","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159262024-03-06T21:04:31.320ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
159272024-03-06T21:04:31.321Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.272076028Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"client":"2","":"downstairs","session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159282024-03-06T21:04:31.321Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.272110315Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159292024-03-06T21:04:31.321Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.272131284Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159302024-03-06T21:04:31.321Z{"msg":"f434aeb6-985c-4993-bcfe-cef01d45819b is now active with session: 18080ab1-3784-4cb1-a160-ea0f450da3d7","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.272150753Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159312024-03-06T21:04:31.321Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.272169291Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731,"session_id":"18080ab1-3784-4cb1-a160-ea0f450da3d7"}
159322024-03-06T21:04:31.321Z{"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:24.27218789Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731}
159332024-03-06T21:04:31.322ZLoading write count information from file "/tmp/test_repair_verify.data"
159342024-03-06T21:04:31.322ZRun Repair workload
159352024-03-06T21:04:31.322ZError: Difference in extent metadata found!
159362024-03-06T21:04:31.322Z01/30 Write block 88 len 12288 data: 9 9 7
159372024-03-06T21:04:31.323Z02/30 Write block 235 len 16384 data: 14 15 17 20
159382024-03-06T21:04:31.323Z03/30 Write block 350 len 32768 data: 11 11 10 11 12 12 16 14
159392024-03-06T21:04:31.323Z04/30 Read block 267 len 8192
159402024-03-06T21:04:31.323Z05/30 Write block 588 len 8192 data: 13 13
159412024-03-06T21:04:31.323Z06/30 Read block 160 len 16384
159422024-03-06T21:04:31.323Z07/30 Write block 234 len 4096 data: 12
159432024-03-06T21:04:31.323Z08/30 Write block 495 len 12288 data: 12 12 12
159442024-03-06T21:04:31.323Z09/30 Read block 547 len 16384
159452024-03-06T21:04:31.324Z10/30 Write block 10 len 8192 data: 10 14
159462024-03-06T21:04:31.324Z11/30 Write block 94 len 28672 data: 9 9 12 10 12 11 8
159472024-03-06T21:04:31.324Z12/30 Write block 524 len 8192 data: 12 11
159482024-03-06T21:04:31.324Z13/30 Read block 121 len 28672
159492024-03-06T21:04:31.325Z14/30 Write block 489 len 36864 data: 12 9 10 10 12 10 13 13 13
159502024-03-06T21:04:31.325Z15/30 Write block 382 len 36864 data: 15 15 16 16 15 12 12 11 13
159512024-03-06T21:04:31.325Z16/30 Read block 569 len 8192
159522024-03-06T21:04:31.325Z17/30 Read block 102 len 36864
159532024-03-06T21:04:31.325Z18/30 Write block 161 len 40960 data: 12 12 10 10 10 8 9 10 9 9
159542024-03-06T21:04:31.325Z19/30 Write block 526 len 20480 data: 12 13 16 15 15
159552024-03-06T21:04:31.326Z20/30 Write block 322 len 8192 data: 13 11
159562024-03-06T21:04:31.326Z21/30 Read block 309 len 32768
159572024-03-06T21:04:31.326Z22/30 Read block 0 len 4096
159582024-03-06T21:04:31.326Z23/30 Read block 227 len 28672
159592024-03-06T21:04:31.326Z24/30 Write block 488 len 8192 data: 11 13
159602024-03-06T21:04:31.326Z25/30 Write block 68 len 16384 data: 15 13 9 9
159612024-03-06T21:04:31.326Z26/30 Read block 337 len 24576
159622024-03-06T21:04:31.326Z27/30 Write block 393 len 40960 data: 8 7 6 5 9 9 6 8 7 7
159632024-03-06T21:04:31.326Z28/30 Read block 534 len 16384
159642024-03-06T21:04:31.326Z29/30 Write block 434 len 40960 data: 14 12 13 14 14 15 13 11 10 9
159652024-03-06T21:04:31.326Z30/30 Read block 576 len 32768
159662024-03-06T21:04:31.327Z----------------------------------------------------------------
159672024-03-06T21:04:31.327Z Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30
159682024-03-06T21:04:31.327ZStates: Active Active Active
159692024-03-06T21:04:31.327ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
159702024-03-06T21:04:31.327Z 1 Acked 1000 Write 3 Done Done Sent false
159712024-03-06T21:04:31.327Z 2 Acked 1001 Write 4 Done Done Sent false
159722024-03-06T21:04:31.327Z 3 Acked 1002 Write 8 Done Done Sent false
159732024-03-06T21:04:31.327Z 4 Acked 1003 Read 2 Done Done Sent false
159742024-03-06T21:04:31.327Z 5 Acked 1004 Write 2 Done Done Sent false
159752024-03-06T21:04:31.327Z 6 Acked 1005 Read 4 Done Done Sent false
159762024-03-06T21:04:31.327Z 7 Acked 1006 Write 1 Done Done Sent false
159772024-03-06T21:04:31.328Z 8 Acked 1007 Write 3 Done Done Sent false
159782024-03-06T21:04:31.328Z 9 Acked 1008 Read 4 Done Done Sent false
159792024-03-06T21:04:31.346Z 10 Acked 1009 Write 2 Done Done Sent false
159802024-03-06T21:04:31.346Z 11 Acked 1010 Write 7 Done Done Sent false
159812024-03-06T21:04:31.346Z 12 Acked 1011 Write 2 Done Done Sent false
159822024-03-06T21:04:31.346Z 13 Acked 1012 Read 7 Done Done Sent false
159832024-03-06T21:04:31.346Z 14 Acked 1013 Write 9 Done Done Sent false
159842024-03-06T21:04:31.346Z 15 Acked 1014 Write 9 Done Done Sent false
159852024-03-06T21:04:31.346Z 16 Acked 1015 Read 2 Done Done Sent false
159862024-03-06T21:04:31.346Z 17 Acked 1016 Read 9 Done Done Sent false
159872024-03-06T21:04:31.346Z 18 Acked 1017 Write 10 Done Done Sent false
159882024-03-06T21:04:31.347Z 19 Acked 1018 Write 5 Done Done Sent false
159892024-03-06T21:04:31.347Z 20 Acked 1019 Write 2 Done Done Sent false
159902024-03-06T21:04:31.347Z 21 Acked 1020 Read 8 Done Done Sent false
159912024-03-06T21:04:31.347Z 22 Acked 1021 Read 1 Done Done Sent false
159922024-03-06T21:04:31.347Z 23 Acked 1022 Read 7 Done Done Sent false
159932024-03-06T21:04:31.347Z 24 Acked 1023 Write 2 Done Done Sent false
159942024-03-06T21:04:31.347Z 25 Acked 1024 Write 4 Done Done Sent false
159952024-03-06T21:04:31.347Z 26 Acked 1025 Read 6 Done Done Sent false
159962024-03-06T21:04:31.347Z 27 Acked 1026 Write 10 Done Done Sent false
159972024-03-06T21:04:31.347Z 28 Acked 1027 Read 4 Done Done Sent false
159982024-03-06T21:04:31.347Z 29 Acked 1028 Write 10 Done Done Sent false
159992024-03-06T21:04:31.347Z 30 Acked 1029 Read 8 Sent Done Sent false
160002024-03-06T21:04:31.347Z STATES DS:0 DS:1 DS:2 TOTAL
160012024-03-06T21:04:31.347Z New 0 0 0 0
160022024-03-06T21:04:31.348Z Sent 1 0 30 31
160032024-03-06T21:04:31.348Z Done 29 30 0 59
160042024-03-06T21:04:31.348Z Skipped 0 0 0 0
160052024-03-06T21:04:31.348Z Error 0 0 0 0
160062024-03-06T21:04:31.348ZLast Flush: 0 0 0
160072024-03-06T21:04:31.348ZDownstairs last five completed:
160082024-03-06T21:04:31.348ZUpstairs last five completed: 30 29 28 27 26
160092024-03-06T21:04:31.348Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:04:24.302306264Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1731}
160102024-03-06T21:04:31.348ZWrote out file "/tmp/test_repair_verify.data"
160112024-03-06T21:04:31.348Z
160122024-03-06T21:04:31.348ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
160132024-03-06T21:04:31.348ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
160142024-03-06T21:04:31.348Z 0 000-019 142 142 142 252 252 252 T T F <---
160152024-03-06T21:04:31.348Z 1 020-039 147 147 147 260 260 260 F F F
160162024-03-06T21:04:31.348Z 2 040-059 147 147 147 260 260 260 F F F
160172024-03-06T21:04:31.352Z 3 060-079 140 140 140 245 245 245 T T F <---
160182024-03-06T21:04:31.352Z 4 080-099 147 147 147 260 260 260 T T F <---
160192024-03-06T21:04:31.352Z 5 100-119 144 144 144 255 255 255 T T F <---
160202024-03-06T21:04:31.352Z 6 120-139 137 137 137 238 238 238 F F F
160212024-03-06T21:04:31.352Z 7 140-159 146 146 146 259 259 259 F F F
160222024-03-06T21:04:31.352Z 8 160-179 146 146 146 259 259 259 T T F <---
160232024-03-06T21:04:31.352Z 9 180-199 146 146 146 259 259 259 F F F
160242024-03-06T21:04:31.352Z 10 200-219 147 147 147 260 260 260 F F F
160252024-03-06T21:04:31.352Z 11 220-239 147 147 147 260 260 260 T T F <---
160262024-03-06T21:04:31.353Z 12 240-259 147 147 147 260 260 260 F F F
160272024-03-06T21:04:31.353Z 13 260-279 142 142 142 251 251 251 F F F
160282024-03-06T21:04:31.353Z 14 280-299 145 145 145 257 257 257 F F F
160292024-03-06T21:04:31.353Z 15 300-319 141 141 141 248 248 248 F F F
160302024-03-06T21:04:31.353Z 16 320-339 147 147 147 260 260 260 T T F <---
160312024-03-06T21:04:31.353Z 17 340-359 140 140 140 247 247 247 T T F <---
160322024-03-06T21:04:31.353Z 18 360-379 147 147 147 260 260 260 F F F
160332024-03-06T21:04:31.353Z 19 380-399 147 147 147 260 260 260 T T F <---
160342024-03-06T21:04:31.353Z 20 400-419 136 136 136 236 236 236 T T F <---
160352024-03-06T21:04:31.354Z 21 420-439 144 144 144 255 255 255 T T F <---
160362024-03-06T21:04:31.354Z 22 440-459 145 145 145 257 257 257 T T F <---
160372024-03-06T21:04:31.354Z 23 460-479 145 145 145 257 257 257 F F F
160382024-03-06T21:04:31.354Z 24 480-499 135 135 135 235 235 235 T T F <---
160392024-03-06T21:04:31.354Z 25 500-519 147 147 147 260 260 260 F F F
160402024-03-06T21:04:31.354Z 26 520-539 147 147 147 260 260 260 T T F <---
160412024-03-06T21:04:31.354Z 27 540-559 140 140 140 246 246 246 F F F
160422024-03-06T21:04:31.354Z 28 560-579 147 147 147 260 260 260 F F F
160432024-03-06T21:04:31.354Z 29 580-599 134 134 134 234 234 234 T T F <---
160442024-03-06T21:04:31.354ZMax gen: 147, Max flush: 260
160452024-03-06T21:04:31.354ZA Difference in extent metadata is expected here
160462024-03-06T21:04:31.354ZOn loop 74
160472024-03-06T21:04:31.354Z
160482024-03-06T21:04:31.355ZVerifying data now
160492024-03-06T21:04:31.355ZNo change after verify
160502024-03-06T21:04:31.355ZLoop: 74 Downstairs dump after verify (and repair):
160512024-03-06T21:04:31.355ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
160522024-03-06T21:04:31.355Z 0 000-019 148 148 148 261 261 261 F F F
160532024-03-06T21:04:31.355Z 1 020-039 147 147 147 260 260 260 F F F
160542024-03-06T21:04:31.355Z 2 040-059 147 147 147 260 260 260 F F F
160552024-03-06T21:04:31.355Z 3 060-079 148 148 148 261 261 261 F F F
160562024-03-06T21:04:31.355Z 4 080-099 148 148 148 261 261 261 F F F
160572024-03-06T21:04:31.356Z 5 100-119 148 148 148 261 261 261 F F F
160582024-03-06T21:04:31.356Z 6 120-139 137 137 137 238 238 238 F F F
160592024-03-06T21:04:31.356Z 7 140-159 146 146 146 259 259 259 F F F
160602024-03-06T21:04:31.356Z 8 160-179 148 148 148 261 261 261 F F F
160612024-03-06T21:04:31.356Z 9 180-199 146 146 146 259 259 259 F F F
160622024-03-06T21:04:31.356Z 10 200-219 147 147 147 260 260 260 F F F
160632024-03-06T21:04:31.356Z 11 220-239 148 148 148 261 261 261 F F F
160642024-03-06T21:04:31.356Z 12 240-259 147 147 147 260 260 260 F F F
160652024-03-06T21:04:31.356Z 13 260-279 142 142 142 251 251 251 F F F
160662024-03-06T21:04:31.356Z 14 280-299 145 145 145 257 257 257 F F F
160672024-03-06T21:04:31.356Z 15 300-319 141 141 141 248 248 248 F F F
160682024-03-06T21:04:31.356Z 16 320-339 148 148 148 261 261 261 F F F
160692024-03-06T21:04:31.356Z 17 340-359 148 148 148 261 261 261 F F F
160702024-03-06T21:04:31.357Z 18 360-379 147 147 147 260 260 260 F F F
160712024-03-06T21:04:31.357Z 19 380-399 148 148 148 261 261 261 F F F
160722024-03-06T21:04:31.357Z 20 400-419 148 148 148 261 261 261 F F F
160732024-03-06T21:04:31.357Z 21 420-439 148 148 148 261 261 261 F F F
160742024-03-06T21:04:31.357Z 22 440-459 148 148 148 261 261 261 F F F
160752024-03-06T21:04:31.357Z 23 460-479 145 145 145 257 257 257 F F F
160762024-03-06T21:04:31.357Z 24 480-499 148 148 148 261 261 261 F F F
160772024-03-06T21:04:31.357Z 25 500-519 147 147 147 260 260 260 F F F
160782024-03-06T21:04:31.357Z 26 520-539 148 148 148 261 261 261 F F F
160792024-03-06T21:04:31.369Z 27 540-559 140 140 140 246 246 246 F F F
160802024-03-06T21:04:31.369Z 28 560-579 147 147 147 260 260 260 F F F
160812024-03-06T21:04:31.369Z 29 580-599 148 148 148 261 261 261 F F F
160822024-03-06T21:04:31.369ZMax gen: 148, Max flush: 261
160832024-03-06T21:04:31.369Z
160842024-03-06T21:04:31.369ZBegin loop 75
160852024-03-06T21:04:31.369ZDownstairs to restart: 1
160862024-03-06T21:04:31.369Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.173781727Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739}
160872024-03-06T21:04:31.369Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.174062518Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739}
160882024-03-06T21:04:31.369Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.174080846Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739}
160892024-03-06T21:04:31.369Z{"msg":"Crucible 514a238c-8bcc-4c04-bd07-82864925d6f3 has session id: cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.174095285Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
160902024-03-06T21:04:31.369Z{"msg":"Upstairs opts: Upstairs UUID: 514a238c-8bcc-4c04-bd07-82864925d6f3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.174119844Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
160912024-03-06T21:04:31.370Z{"msg":"Crucible stats registered with UUID: 514a238c-8bcc-4c04-bd07-82864925d6f3","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
160922024-03-06T21:04:31.370Z2024-03-06T21:04:29.174135023Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
160932024-03-06T21:04:31.370Z{"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.174236696Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739}
160942024-03-06T21:04:31.370Z{"msg":"514a238c-8bcc-4c04-bd07-82864925d6f3 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.174365407Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
160952024-03-06T21:04:31.370Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.174383966Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"client":"0","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
160962024-03-06T21:04:31.370Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.174400884Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"client":"0","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
160972024-03-06T21:04:31.370Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.174416623Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"client":"1","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
160982024-03-06T21:04:31.370Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.174431842Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"client":"1","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
160992024-03-06T21:04:31.370Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.174446901Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"client":"2","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161002024-03-06T21:04:31.370Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.1744629Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"client":"2","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161012024-03-06T21:04:31.371Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.174528796Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161022024-03-06T21:04:31.371Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.174561423Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161032024-03-06T21:04:31.371Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.174587281Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161042024-03-06T21:04:31.371Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.174731462Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161052024-03-06T21:04:31.371Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.174769089Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161062024-03-06T21:04:31.371Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.174797347Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161072024-03-06T21:04:31.371Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.175569764Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"client":"2","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161082024-03-06T21:04:31.371Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.175607371Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"client":"2","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161092024-03-06T21:04:31.371Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.175636179Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"client":"0","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161102024-03-06T21:04:31.371Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.175664447Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"client":"0","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161112024-03-06T21:04:31.371Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.176262046Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"client":"2","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161122024-03-06T21:04:31.372Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.176303313Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"client":"0","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161132024-03-06T21:04:31.372Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.176366719Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"client":"1","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161142024-03-06T21:04:31.372Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.176399936Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"client":"1","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161152024-03-06T21:04:31.372Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.176970817Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"client":"2","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161162024-03-06T21:04:31.372Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.176998035Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"client":"2","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161172024-03-06T21:04:31.372Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.177012874Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161182024-03-06T21:04:31.372Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.177026323Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"client":"1","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161192024-03-06T21:04:31.372Z{Wait for a query_work_queue command to finish before sending IO
161202024-03-06T21:04:31.372Z"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.466605697Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"client":"0","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161212024-03-06T21:04:31.372Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.466687341Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"client":"0","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161222024-03-06T21:04:31.372Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.466721209Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161232024-03-06T21:04:31.372Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.466746457Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"client":"1","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161242024-03-06T21:04:31.373Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.466775825Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"client":"1","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161252024-03-06T21:04:31.373Z{"msg":"[0]R flush_numbers[0..12]: [261, 260, 260, 261, 261, 261, 238, 259, 261, 259, 260, 261]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.466823801Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"":"downstairs","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161262024-03-06T21:04:31.373Z{"msg":"[0]R generation[0..12]: [148, 147, 147, 148, 148, 148, 137, 146, 148, 146, 147, 148]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.466856369Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"":"downstairs","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161272024-03-06T21:04:31.373Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.466889047Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"":"downstairs","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161282024-03-06T21:04:31.373Z{"msg":"[1]R flush_numbers[0..12]: [261, 260, 260, 261, 261, 261, 238, 259, 261, 259, 260, 261]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.466921075Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"":"downstairs","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161292024-03-06T21:04:31.373Z{"msg":"[1]R generation[0..12]: [148, 147, 147, 148, 148, 148, 137, 146, 148, 146, 147, 148]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.466952563Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"":"downstairs","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161302024-03-06T21:04:31.373Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.46699299Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"":"downstairs","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161312024-03-06T21:04:31.373Z{"msg":"[2]R flush_numbers[0..12]: [261, 260, 260, 261, 261, 261, 238, 259, 261, 259, 260, 261]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.467026258Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"":"downstairs","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161322024-03-06T21:04:31.373Z{"msg":"[2]R generation[0..12]: [148, 147, 147, 148, 148, 148, 137, 146, 148, 146, 147, 148]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.467064815Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"":"downstairs","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161332024-03-06T21:04:31.373Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.467097263Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"":"downstairs","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161342024-03-06T21:04:31.373Z{"msg":"Max found gen is 149","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.46713072Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"":"downstairs","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161352024-03-06T21:04:31.373Z{"msg":"Generation requested: 150 >= found:149","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.467161178Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"":"downstairs","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161362024-03-06T21:04:31.373Z{"msg":"Next flush: 262","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.467195576Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"":"downstairs","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
161372024-03-06T21:04:31.374Z}
161382024-03-06T21:04:31.374Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.467233943Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"":"downstairs","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161392024-03-06T21:04:31.374Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.467268651Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161402024-03-06T21:04:31.374Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.467298539Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"client":"0","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161412024-03-06T21:04:31.374Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.467330767Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"client":"1","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161422024-03-06T21:04:31.374Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.467362775Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"client":"2","":"downstairs","session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161432024-03-06T21:04:31.374Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.467393712Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161442024-03-06T21:04:31.374Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.4674238Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161452024-03-06T21:04:31.374Z{"msg":"514a238c-8bcc-4c04-bd07-82864925d6f3 is now active with session: cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.467455328Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161462024-03-06T21:04:31.374Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.467483746Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739,"session_id":"cb64a1bd-76ae-438f-b9ea-3d7caba1d9ce"}
161472024-03-06T21:04:31.375Z{"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:29.467504125Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739}
161482024-03-06T21:04:31.375ZLoading write count information from file "/tmp/test_repair_verify.data"
161492024-03-06T21:04:31.375ZRun Repair workload
161502024-03-06T21:04:31.375Z01/30 Write block 205 len 16384 data: 10 11 8 11
161512024-03-06T21:04:31.375Z02/30 Write block 142 len 32768 data: 6 6 6 6 6 8 11 13
161522024-03-06T21:04:31.375Z03/30 Write block 124 len 12288 data: 9 14 13
161532024-03-06T21:04:31.375Z04/30 Read block 376 len 16384
161542024-03-06T21:04:31.375Z05/30 Read block 525 len 28672
161552024-03-06T21:04:31.375Z06/30 Write block 40 len 40960 data: 12 15 16 15 13 8 9 11 10 8
161562024-03-06T21:04:31.375Z07/30 Read block 209 len 24576
161572024-03-06T21:04:31.375Z08/30 Flush
161582024-03-06T21:04:31.375Z09/30 Write block 235 len 4096 data: 15
161592024-03-06T21:04:31.375Z10/30 Read block 519 len 32768
161602024-03-06T21:04:31.376Z11/30 Write block 13 len 20480 data: 13 12 14 12 14
161612024-03-06T21:04:31.376Z12/30 Read block 440 len 24576
161622024-03-06T21:04:31.376Z13/30 Read block 76 len 16384
161632024-03-06T21:04:31.376Z14/30 Read block 362 len 28672
161642024-03-06T21:04:31.376Z15/30 Read block 270 len 36864
161652024-03-06T21:04:31.376Z16/30 Flush
161662024-03-06T21:04:31.376Z17/30 Write block 375 len 4096 data: 16
161672024-03-06T21:04:31.376Z18/30 Read block 502 len 40960
161682024-03-06T21:04:31.376Z19/30 Read block 335 len 32768
161692024-03-06T21:04:31.376Z20/30 Write block 214 len 36864 data: 11 14 17 18 15 10 7 7 5
161702024-03-06T21:04:31.376Z21/30 Write block 226 len 32768 data: 9 12 12 11 13 14 11 10
161712024-03-06T21:04:31.376Z22/30 Read block 400 len 40960
161722024-03-06T21:04:31.376Z23/30 Write block 276 len 36864 data: 12 10 10 10 9 10 12 10 12
161732024-03-06T21:04:31.377Z24/30 Flush
161742024-03-06T21:04:31.377Z25/30 Write block 59 len 40960 data: 15 17 20 19 23 19 19 18 17 16
161752024-03-06T21:04:31.377Z26/30 Write block 543 len 16384 data: 14 15 16 15
161762024-03-06T21:04:31.377Z27/30 Read block 382 len 40960
161772024-03-06T21:04:31.377Z28/30 Read block 537 len 32768
161782024-03-06T21:04:31.377Z29/30 Write block 510 len 40960 data: 13 13 13 12 13 15 14 12 11 10
161792024-03-06T21:04:31.389Z30/30 Read block 478 len 8192
161802024-03-06T21:04:31.389Z----------------------------------------------------------------
161812024-03-06T21:04:31.389Z Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:30
161822024-03-06T21:04:31.389ZStates: Active Active Active
161832024-03-06T21:04:31.389ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
161842024-03-06T21:04:31.390Z 1 Acked 1000 Write 4 Done Sent Done false
161852024-03-06T21:04:31.390Z 2 Acked 1001 Write 8 Done Sent Done false
161862024-03-06T21:04:31.390Z 3 Acked 1002 Write 3 Done Sent Done false
161872024-03-06T21:04:31.390Z 4 Acked 1003 Read 4 Done Sent Done false
161882024-03-06T21:04:31.390Z 5 Acked 1004 Read 7 Done Sent Done false
161892024-03-06T21:04:31.390Z 6 Acked 1005 Write 10 Done Sent Done false
161902024-03-06T21:04:31.390Z 7 Acked 1006 Read 6 Done Sent Done false
161912024-03-06T21:04:31.390Z 8 Acked 1007 Flush 0 Done Sent Done false
161922024-03-06T21:04:31.390Z 9 Acked 1008 Write 1 Done Sent Done false
161932024-03-06T21:04:31.390Z 10 Acked 1009 Read 8 Done Sent Done false
161942024-03-06T21:04:31.390Z 11 Acked 1010 Write 5 Done Sent Done false
161952024-03-06T21:04:31.390Z 12 Acked 1011 Read 6 Done Sent Done false
161962024-03-06T21:04:31.390Z 13 Acked 1012 Read 4 Done Sent Done false
161972024-03-06T21:04:31.390Z 14 Acked 1013 Read 7 Done Sent Done false
161982024-03-06T21:04:31.390Z 15 Acked 1014 Read 9 Done Sent Done false
161992024-03-06T21:04:31.391Z 16 Acked 1015 Flush 0 Done Sent Done false
162002024-03-06T21:04:31.391Z 17 Acked 1016 Write 1 Done Sent Done false
162012024-03-06T21:04:31.391Z 18 Acked 1017 Read 10 Done Sent Done false
162022024-03-06T21:04:31.391Z 19 Acked 1018 Read 8 Done Sent Done false
162032024-03-06T21:04:31.391Z 20 Acked 1019 Write 9 Done Sent Done false
162042024-03-06T21:04:31.391Z 21 Acked 1020 Write 8 Done Sent Done false
162052024-03-06T21:04:31.391Z 22 Acked 1021 Read 10 Done Sent Done false
162062024-03-06T21:04:31.391Z 23 Acked 1022 Write 9 Done Sent Done false
162072024-03-06T21:04:31.391Z 24 Acked 1023 Flush 0 Done Sent Done false
162082024-03-06T21:04:31.391Z 25 Acked 1024 Write 10 Done Sent Done false
162092024-03-06T21:04:31.391Z 26 Acked 1025 Write 4 Done Sent Done false
162102024-03-06T21:04:31.391Z 27 Acked 1026 Read 10 Done Sent Done false
162112024-03-06T21:04:31.391Z 28 Acked 1027 Read 8 Done Sent Done false
162122024-03-06T21:04:31.391Z 29 Acked 1028 Write 10 Done Sent Done false
162132024-03-06T21:04:31.391Z 30 Acked 1029 Read 2 Done Sent Done false
162142024-03-06T21:04:31.391Z STATES DS:0 DS:1 DS:2 TOTAL
162152024-03-06T21:04:31.392Z New 0 0 0 0
162162024-03-06T21:04:31.392Z Sent 0 30 0 30
162172024-03-06T21:04:31.392Z Done 30 0 30 60
162182024-03-06T21:04:31.392Z Skipped 0 0 0 0
162192024-03-06T21:04:31.392Z Error 0 0 0 0
162202024-03-06T21:04:31.392ZLast Flush: 1023 0 1023
162212024-03-06T21:04:31.392ZDownstairs last five completed:
162222024-03-06T21:04:31.392ZUpstairs last five completed: 30 29 28 27 26
162232024-03-06T21:04:31.392Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:04:29.505389084Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1739}
162242024-03-06T21:04:31.392ZWrote out file "/tmp/test_repair_verify.data"
162252024-03-06T21:04:31.392Z
162262024-03-06T21:04:31.392ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
162272024-03-06T21:04:31.392ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
162282024-03-06T21:04:31.392Z 0 000-019 150 148 150 263 261 263 F F F <---
162292024-03-06T21:04:31.392Z 1 020-039 147 147 147 260 260 260 F F F
162302024-03-06T21:04:31.392Z 2 040-059 150 147 150 262 260 262 T F T <---
162312024-03-06T21:04:31.393Z 3 060-079 148 148 148 261 261 261 T F T <---
162322024-03-06T21:04:31.393Z 4 080-099 148 148 148 261 261 261 F F F
162332024-03-06T21:04:31.393Z 5 100-119 148 148 148 261 261 261 F F F
162342024-03-06T21:04:31.393Z 6 120-139 150 137 150 262 238 262 F F F <---
162352024-03-06T21:04:31.393Z 7 140-159 150 146 150 262 259 262 F F F <---
162362024-03-06T21:04:31.393Z 8 160-179 148 148 148 261 261 261 F F F
162372024-03-06T21:04:31.393Z 9 180-199 146 146 146 259 259 259 F F F
162382024-03-06T21:04:31.393Z 10 200-219 150 147 150 264 260 264 F F F <---
162392024-03-06T21:04:31.393Z 11 220-239 150 148 150 264 261 264 F F F <---
162402024-03-06T21:04:31.393Z 12 240-259 147 147 147 260 260 260 F F F
162412024-03-06T21:04:31.393Z 13 260-279 150 142 150 264 251 264 F F F <---
162422024-03-06T21:04:31.393Z 14 280-299 150 145 150 264 257 264 F F F <---
162432024-03-06T21:04:31.393Z 15 300-319 141 141 141 248 248 248 F F F
162442024-03-06T21:04:31.393Z 16 320-339 148 148 148 261 261 261 F F F
162452024-03-06T21:04:31.393Z 17 340-359 148 148 148 261 261 261 F F F
162462024-03-06T21:04:31.393Z 18 360-379 150 147 150 264 260 264 F F F <---
162472024-03-06T21:04:31.393Z 19 380-399 148 148 148 261 261 261 F F F
162482024-03-06T21:04:31.393Z 20 400-419 148 148 148 261 261 261 F F F
162492024-03-06T21:04:31.394Z 21 420-439 148 148 148 261 261 261 F F F
162502024-03-06T21:04:31.394Z 22 440-459 148 148 148 261 261 261 F F F
162512024-03-06T21:04:31.394Z 23 460-479 145 145 145 257 257 257 F F F
162522024-03-06T21:04:31.394Z 24 480-499 148 148 148 261 261 261 F F F
162532024-03-06T21:04:31.394Z 25 500-519 147 147 147 260 260 260 T F T <---
162542024-03-06T21:04:31.394Z 26 520-539 148 148 148 261 261 261 F F F
162552024-03-06T21:04:31.394Z 27 540-559 140 140 140 246 246 246 T F T <---
162562024-03-06T21:04:31.394Z 28 560-579 147 147 147 260 260 260 F F F
162572024-03-06T21:04:31.394Z 29 580-599 148 148 148 261 261 261 F F F
162582024-03-06T21:04:31.394ZMax gen: 150, Max flush: 264
162592024-03-06T21:04:31.394ZError: Difference in extent metadata found!
162602024-03-06T21:04:31.394ZA Difference in extent metadata is expected here
162612024-03-06T21:04:31.394ZOn loop 75
162622024-03-06T21:04:31.394Z
162632024-03-06T21:04:31.394ZVerifying data now
162642024-03-06T21:04:36.337ZNo change after verify
162652024-03-06T21:04:36.337ZLoop: 75 Downstairs dump after verify (and repair):
162662024-03-06T21:04:36.338ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
162672024-03-06T21:04:36.338Z 0 000-019 150 150 150 263 263 263 F F F
162682024-03-06T21:04:36.338Z 1 020-039 147 147 147 260 260 260 F F F
162692024-03-06T21:04:36.338Z 2 040-059 151 151 151 265 265 265 F F F
162702024-03-06T21:04:36.338Z 3 060-079 151 151 151 265 265 265 F F F
162712024-03-06T21:04:36.339Z 4 080-099 148 148 148 261 261 261 F F F
162722024-03-06T21:04:36.339Z 5 100-119 148 148 148 261 261 261 F F F
162732024-03-06T21:04:36.339Z 6 120-139 150 150 150 262 262 262 F F F
162742024-03-06T21:04:36.339Z 7 140-159 150 150 150 262 262 262 F F F
162752024-03-06T21:04:36.339Z 8 160-179 148 148 148 261 261 261 F F F
162762024-03-06T21:04:36.339Z 9 180-199 146 146 146 259 259 259 F F F
162772024-03-06T21:04:36.339Z 10 200-219 150 150 150 264 264 264 F F F
162782024-03-06T21:04:36.339Z 11 220-239 150 150 150 264 264 264 F F F
162792024-03-06T21:04:36.340Z 12 240-259 147 147 147 260 260 260 F F F
162802024-03-06T21:04:36.340Z 13 260-279 150 150 150 264 264 264 F F F
162812024-03-06T21:04:36.340Z 14 280-299 150 150 150 264 264 264 F F F
162822024-03-06T21:04:36.340Z 15 300-319 141 141 141 248 248 248 F F F
162832024-03-06T21:04:36.340Z 16 320-339 148 148 148 261 261 261 F F F
162842024-03-06T21:04:36.340Z 17 340-359 148 148 148 261 261 261 F F F
162852024-03-06T21:04:36.341Z 18 360-379 150 150 150 264 264 264 F F F
162862024-03-06T21:04:36.341Z 19 380-399 148 148 148 261 261 261 F F F
162872024-03-06T21:04:36.341Z 20 400-419 148 148 148 261 261 261 F F F
162882024-03-06T21:04:36.341Z 21 420-439 148 148 148 261 261 261 F F F
162892024-03-06T21:04:36.341Z 22 440-459 148 148 148 261 261 261 F F F
162902024-03-06T21:04:36.341Z 23 460-479 145 145 145 257 257 257 F F F
162912024-03-06T21:04:36.341Z 24 480-499 148 148 148 261 261 261 F F F
162922024-03-06T21:04:36.342Z 25 500-519 151 151 151 265 265 265 F F F
162932024-03-06T21:04:36.342Z 26 520-539 148 148 148 261 261 261 F F F
162942024-03-06T21:04:36.342Z 27 540-559 151 151 151 265 265 265 F F F
162952024-03-06T21:04:36.342Z 28 560-579 147 147 147 260 260 260 F F F
162962024-03-06T21:04:36.342Z 29 580-599 148 148 148 261 261 261 F F F
162972024-03-06T21:04:36.342ZMax gen: 151, Max flush: 265
162982024-03-06T21:04:36.342Z
162992024-03-06T21:04:36.342ZBegin loop 76
163002024-03-06T21:04:36.342ZDownstairs to restart: 1
163012024-03-06T21:04:36.343Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.225905871Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747}
163022024-03-06T21:04:36.343Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.22620366Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747}
163032024-03-06T21:04:36.343Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
163042024-03-06T21:04:36.343Z,"time":"2024-03-06T21:04:34.226239958Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747}
163052024-03-06T21:04:36.343Z{"msg":"Crucible 413de074-0e05-4a51-8517-886df826d4e1 has session id: 4a422686-00bf-4a24-86c1-b0bf70643b72","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.226265296Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163062024-03-06T21:04:36.343Z{"msg":"Upstairs opts: Upstairs UUID: 413de074-0e05-4a51-8517-886df826d4e1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.226317312Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163072024-03-06T21:04:36.343Z{"msg":"Crucible stats registered with UUID: 413de074-0e05-4a51-8517-886df826d4e1","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.22635413Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163082024-03-06T21:04:36.344Z{"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.226381178Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747}
163092024-03-06T21:04:36.344Z{"msg":"413de074-0e05-4a51-8517-886df826d4e1 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.22649325Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163102024-03-06T21:04:36.344Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.226529698Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"client":"0","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163112024-03-06T21:04:36.344Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.226573145Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"client":"0","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163122024-03-06T21:04:36.344Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.226615512Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"client":"1","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163132024-03-06T21:04:36.344Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.226662119Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"client":"1","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163142024-03-06T21:04:36.344Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.226738043Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"client":"2","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163152024-03-06T21:04:36.344Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.226826257Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"client":"2","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163162024-03-06T21:04:36.344Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.226867665Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163172024-03-06T21:04:36.344Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.226918101Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163182024-03-06T21:04:36.345Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.226965528Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163192024-03-06T21:04:36.345Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.227017214Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163202024-03-06T21:04:36.345Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.22707574Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163212024-03-06T21:04:36.345Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.227101848Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163222024-03-06T21:04:36.345Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.22780889Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"client":"0","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163232024-03-06T21:04:36.345Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.227861446Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"client":"0","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163242024-03-06T21:04:36.345Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.227905473Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"client":"2","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163252024-03-06T21:04:36.345Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.227938691Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"client":"2","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163262024-03-06T21:04:36.345Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.228334044Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"client":"1","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163272024-03-06T21:04:36.345Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.22838613Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"client":"1","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163282024-03-06T21:04:36.345Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.228434377Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"client":"2","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163292024-03-06T21:04:36.346Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.228656691Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"client":"0","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163302024-03-06T21:04:36.346Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.228976839Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"client":"2","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163312024-03-06T21:04:36.346Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.229005717Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"client":"2","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163322024-03-06T21:04:36.346Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.229021866Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163332024-03-06T21:04:36.346Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.229046984Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"client":"1","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163342024-03-06T21:04:36.346Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.229217763Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"client":"0","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163352024-03-06T21:04:36.346Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.229247041Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"client":"0","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163362024-03-06T21:04:36.347Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.229261349Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163372024-03-06T21:04:36.347Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.229501453Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"client":"1","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163382024-03-06T21:04:36.347Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.229530371Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"client":"1","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163392024-03-06T21:04:36.347Z{"msg":"[0]R flush_numbers[0..12]: [263, 260, 265, 265, 261, 261, 262, 262, 261, 259, 264, 264]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.22954609Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"":"downstairs","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163402024-03-06T21:04:36.347Z{"msg":"[0]R generation[0..12]: [150, 147, 151, 151, 148, 148, 150, 150, 148, 146, 150, 150]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.229560979Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"":"downstairs","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163412024-03-06T21:04:36.347Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.229581907Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"":"downstairs","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163422024-03-06T21:04:36.347Z{"msg":"[1]R flush_numbers[0..12]: [263, 260, 265, 265, 261, 261, 262, 262, 261, 259, 264, 264]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.229602336Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"":"downstairs","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163432024-03-06T21:04:36.347Z{"msg":"[1]R generation[0..12]: [150, 147, 151, 151, 148, 148, 150, 150, 148, 146, 150, 150]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.229622795Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"":"downstairs","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163442024-03-06T21:04:36.347Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.229643883Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"":"downstairs","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163452024-03-06T21:04:36.347Z{"msg":"[2]R flush_numbers[0..12]: [263, 260, 265, 265, 261, 261, 262, 262, 261, 259, 264, 264]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.229664632Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"":"downstairs","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163462024-03-06T21:04:36.347Z{"msg":"Wait for a query_work_queue command to finish before sending IO
163472024-03-06T21:04:36.347Z[2]R generation[0..12]: [150, 147, 151, 151, 148, 148, 150, 150, 148, 146, 150, 150]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.22968788Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"":"downstairs","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163482024-03-06T21:04:36.348Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.229710139Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"":"downstairs","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163492024-03-06T21:04:36.348Z{"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.229731047Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"":"downstairs","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163502024-03-06T21:04:36.348Z{"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.229751436Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"":"downstairs","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163512024-03-06T21:04:36.348Z{"msg":"Next flush: 266","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.229771034Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"":"downstairs","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163522024-03-06T21:04:36.348Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.229790473Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"":"downstairs","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163532024-03-06T21:04:36.348Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.229810192Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163542024-03-06T21:04:36.348Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.229828741Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"client":"0","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163552024-03-06T21:04:36.348Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.229848559Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"client":"1","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163562024-03-06T21:04:36.348Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.229868148Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"client":"2","":"downstairs","session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163572024-03-06T21:04:36.348Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.229887836Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163582024-03-06T21:04:36.348Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.229906625Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163592024-03-06T21:04:36.348Z{"msg":"413de074-0e05-4a51-8517-886df826d4e1 is now active with session: 4a422686-00bf-4a24-86c1-b0bf70643b72","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.229925864Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163602024-03-06T21:04:36.348Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.229945162Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747,"session_id":"4a422686-00bf-4a24-86c1-b0bf70643b72"}
163612024-03-06T21:04:36.349Z{"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:34.22997618Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747}
163622024-03-06T21:04:36.349ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
163632024-03-06T21:04:36.349ZLoading write count information from file "/tmp/test_repair_verify.data"
163642024-03-06T21:04:36.365ZRun Repair workload
163652024-03-06T21:04:36.365ZError: Difference in extent metadata found!
163662024-03-06T21:04:36.365Z01/30 Write block 111 len 40960 data: 9 6 3 6 6 6 8 9 10 9
163672024-03-06T21:04:36.366Z02/30 Read block 587 len 16384
163682024-03-06T21:04:36.366Z03/30 Write block 286 len 12288 data: 16 16 17
163692024-03-06T21:04:36.366Z04/30 Read block 497 len 28672
163702024-03-06T21:04:36.366Z05/30 Write block 532 len 16384 data: 12 12 12 14
163712024-03-06T21:04:36.366Z06/30 Read block 583 len 40960
163722024-03-06T21:04:36.366Z07/30 Read block 162 len 4096
163732024-03-06T21:04:36.366Z08/30 Write block 208 len 4096 data: 12
163742024-03-06T21:04:36.366Z09/30 Read block 308 len 32768
163752024-03-06T21:04:36.367Z10/30 Read block 402 len 32768
163762024-03-06T21:04:36.367Z11/30 Read block 229 len 36864
163772024-03-06T21:04:36.367Z12/30 Write block 284 len 24576 data: 13 15 17 17 18 16
163782024-03-06T21:04:36.368Z13/30 Write block 316 len 24576 data: 9 7 11 11 10 13
163792024-03-06T21:04:36.368Z14/30 Read block 95 len 8192
163802024-03-06T21:04:36.368Z15/30 Read block 390 len 4096
163812024-03-06T21:04:36.368Z16/30 Read block 591 len 36864
163822024-03-06T21:04:36.368Z17/30 Write block 228 len 24576 data: 13 12 14 15 12 11
163832024-03-06T21:04:36.369Z18/30 Write block 22 len 28672 data: 16 16 13 12 12 12 11
163842024-03-06T21:04:36.369Z19/30 Flush
163852024-03-06T21:04:36.369Z20/30 Write block 168 len 28672 data: 11 10 10 10 12 16 14
163862024-03-06T21:04:36.369Z21/30 Read block 431 len 20480
163872024-03-06T21:04:36.369Z22/30 Write block 480 len 28672 data: 8 9 9 11 11 13 10
163882024-03-06T21:04:36.369Z23/30 Read block 299 len 28672
163892024-03-06T21:04:36.370Z24/30 Read block 185 len 8192
163902024-03-06T21:04:36.370Z25/30 Write block 200 len 8192 data: 12 10
163912024-03-06T21:04:36.370Z26/30 Read block 459 len 12288
163922024-03-06T21:04:36.370Z27/30 Write block 568 len 36864 data: 9 8 10 12 11 11 10 11 13
163932024-03-06T21:04:36.370Z28/30 Write block 99 len 36864 data: 12 9 11 11 11 9 7 12 13
163942024-03-06T21:04:36.370Z29/30 Read block 152 len 20480
163952024-03-06T21:04:36.371Z30/30 Write block 435 len 40960 data: 13 14 15 15 16 14 12 11 10 10
163962024-03-06T21:04:36.371Z----------------------------------------------------------------
163972024-03-06T21:04:36.371Z Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:30
163982024-03-06T21:04:36.371ZStates: Active Active Active
163992024-03-06T21:04:36.371ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
164002024-03-06T21:04:36.371Z 1 Acked 1000 Write 10 Done Done Done false
164012024-03-06T21:04:36.371Z 2 Acked 1001 Read 4 Done Done Done false
164022024-03-06T21:04:36.371Z 3 Acked 1002 Write 3 Done Done Done false
164032024-03-06T21:04:36.372Z 4 Acked 1003 Read 7 Done Done Done false
164042024-03-06T21:04:36.372Z 5 Acked 1004 Write 4 Done Sent Done false
164052024-03-06T21:04:36.372Z 6 Acked 1005 Read 10 Done Sent Done false
164062024-03-06T21:04:36.372Z 7 Acked 1006 Read 1 Done Sent Done false
164072024-03-06T21:04:36.372Z 8 Acked 1007 Write 1 Done Sent Done false
164082024-03-06T21:04:36.372Z 9 Acked 1008 Read 8 Done Sent Done false
164092024-03-06T21:04:36.372Z 10 Acked 1009 Read 8 Done Sent Done false
164102024-03-06T21:04:36.373Z 11 Acked 1010 Read 9 Done Sent Done false
164112024-03-06T21:04:36.373Z 12 Acked 1011 Write 6 Done Sent Done false
164122024-03-06T21:04:36.373Z 13 Acked 1012 Write 6 Done Sent Done false
164132024-03-06T21:04:36.373Z 14 Acked 1013 Read 2 Done Sent Done false
164142024-03-06T21:04:36.373Z 15 Acked 1014 Read 1 Done Sent Done false
164152024-03-06T21:04:36.373Z 16 Acked 1015 Read 9 Done Sent Done false
164162024-03-06T21:04:36.374Z 17 Acked 1016 Write 6 Done Sent Done false
164172024-03-06T21:04:36.374Z 18 Acked 1017 Write 7 Done Sent Done false
164182024-03-06T21:04:36.374Z 19 Acked 1018 Flush 0 Done Sent Done false
164192024-03-06T21:04:36.374Z 20 Acked 1019 Write 7 Done Sent Done false
164202024-03-06T21:04:36.374Z 21 Acked 1020 Read 5 Done Sent Done false
164212024-03-06T21:04:36.374Z 22 Acked 1021 Write 7 Done Sent Done false
164222024-03-06T21:04:36.374Z 23 Acked 1022 Read 7 Done Sent Done false
164232024-03-06T21:04:36.374Z 24 Acked 1023 Read 2 Done Sent Done false
164242024-03-06T21:04:36.375Z 25 Acked 1024 Write 2 Done Sent Done false
164252024-03-06T21:04:36.375Z 26 Acked 1025 Read 3 Done Sent Done false
164262024-03-06T21:04:36.375Z 27 Acked 1026 Write 9 Done Sent Done false
164272024-03-06T21:04:36.375Z 28 Acked 1027 Write 9 Done Sent Done false
164282024-03-06T21:04:36.375Z 29 Acked 1028 Read 5 Done Sent Done false
164292024-03-06T21:04:36.375Z 30 Acked 1029 Write 10 Sent Sent Sent false
164302024-03-06T21:04:36.375Z STATES DS:0 DS:1 DS:2 TOTAL
164312024-03-06T21:04:36.376Z New 0 0 0 0
164322024-03-06T21:04:36.376Z Sent 1 26 1 28
164332024-03-06T21:04:36.376Z Done 29 4 29 62
164342024-03-06T21:04:36.376Z Skipped 0 0 0 0
164352024-03-06T21:04:36.376Z Error 0 0 0 0
164362024-03-06T21:04:36.376ZLast Flush: 1018 0 1018
164372024-03-06T21:04:36.376ZDownstairs last five completed:
164382024-03-06T21:04:36.376ZUpstairs last five completed: 30 29 28 27 26
164392024-03-06T21:04:36.376Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:04:34.264923522Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1747}
164402024-03-06T21:04:36.376ZWrote out file "/tmp/test_repair_verify.data"
164412024-03-06T21:04:36.376Z
164422024-03-06T21:04:36.377ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
164432024-03-06T21:04:36.377ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
164442024-03-06T21:04:36.377Z 0 000-019 150 150 150 263 263 263 F F F
164452024-03-06T21:04:36.377Z 1 020-039 152 147 152 266 260 266 F F F <---
164462024-03-06T21:04:36.377Z 2 040-059 151 151 151 265 265 265 F F F
164472024-03-06T21:04:36.377Z 3 060-079 151 151 151 265 265 265 F F F
164482024-03-06T21:04:36.377Z 4 080-099 148 148 148 261 261 261 T F T <---
164492024-03-06T21:04:36.377Z 5 100-119 152 148 152 266 261 266 T T T <---
164502024-03-06T21:04:36.378Z 6 120-139 152 150 152 266 262 266 F T F <---
164512024-03-06T21:04:36.378Z 7 140-159 150 150 150 262 262 262 F F F
164522024-03-06T21:04:36.378Z 8 160-179 148 148 148 261 261 261 T F T <---
164532024-03-06T21:04:36.378Z 9 180-199 146 146 146 259 259 259 F F F
164542024-03-06T21:04:36.378Z 10 200-219 152 150 152 266 264 266 T F T <---
164552024-03-06T21:04:36.378Z 11 220-239 152 150 152 266 264 266 F F F <---
164562024-03-06T21:04:36.379Z 12 240-259 147 147 147 260 260 260 F F F
164572024-03-06T21:04:36.379Z 13 260-279 150 150 150 264 264 264 F F F
164582024-03-06T21:04:36.379Z 14 280-299 152 150 152 266 264 266 F T F <---
164592024-03-06T21:04:36.379Z 15 300-319 152 141 152 266 248 266 F F F <---
164602024-03-06T21:04:36.379Z 16 320-339 152 148 152 266 261 266 F F F <---
164612024-03-06T21:04:36.379Z 17 340-359 148 148 148 261 261 261 F F F
164622024-03-06T21:04:36.379Z 18 360-379 150 150 150 264 264 264 F F F
164632024-03-06T21:04:36.379Z 19 380-399 148 148 148 261 261 261 F F F
164642024-03-06T21:04:36.394Z 20 400-419 148 148 148 261 261 261 F F F
164652024-03-06T21:04:36.394Z 21 420-439 148 148 148 261 261 261 T F T <---
164662024-03-06T21:04:36.394Z 22 440-459 148 148 148 261 261 261 T F T <---
164672024-03-06T21:04:36.394Z 23 460-479 145 145 145 257 257 257 F F F
164682024-03-06T21:04:36.394Z 24 480-499 148 148 148 261 261 261 T F T <---
164692024-03-06T21:04:36.394Z 25 500-519 151 151 151 265 265 265 F F F
164702024-03-06T21:04:36.394Z 26 520-539 152 148 152 266 261 266 F F F <---
164712024-03-06T21:04:36.394Z 27 540-559 151 151 151 265 265 265 F F F
164722024-03-06T21:04:36.394Z 28 560-579 147 147 147 260 260 260 T F T <---
164732024-03-06T21:04:36.394Z 29 580-599 148 148 148 261 261 261 F F F
164742024-03-06T21:04:36.394ZMax gen: 152, Max flush: 266
164752024-03-06T21:04:36.394ZA Difference in extent metadata is expected here
164762024-03-06T21:04:36.394ZOn loop 76
164772024-03-06T21:04:36.395Z
164782024-03-06T21:04:36.395ZVerifying data now
164792024-03-06T21:04:45.743ZNo change after verify
164802024-03-06T21:04:45.743ZLoop: 76 Downstairs dump after verify (and repair):
164812024-03-06T21:04:46.143ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
164822024-03-06T21:04:46.143Z 0 000-019 150 150 150 263 263 263 F F F
164832024-03-06T21:04:46.143Z 1 020-039 152 152 152 266 266 266 F F F
164842024-03-06T21:04:46.143Z 2 040-059 151 151 151 265 265 265 F F F
164852024-03-06T21:04:46.143Z 3 060-079 151 151 151 265 265 265 F F F
164862024-03-06T21:04:46.143Z 4 080-099 153 153 153 267 267 267 F F F
164872024-03-06T21:04:46.143Z 5 100-119 153 153 153 267 267 267 F F F
164882024-03-06T21:04:46.143Z 6 120-139 152 152 152 266 266 266 F F F
164892024-03-06T21:04:46.143Z 7 140-159 150 150 150 262 262 262 F F F
164902024-03-06T21:04:46.144Z 8 160-179 153 153 153 267 267 267 F F F
164912024-03-06T21:04:46.144Z 9 180-199 146 146 146 259 259 259 F F F
164922024-03-06T21:04:46.144Z 10 200-219 153 153 153 267 267 267 F F F
164932024-03-06T21:04:46.144Z 11 220-239 152 152 152 266 266 266 F F F
164942024-03-06T21:04:46.144Z 12 240-259 147 147 147 260 260 260 F F F
164952024-03-06T21:04:46.144Z 13 260-279 150 150 150 264 264 264 F F F
164962024-03-06T21:04:46.144Z 14 280-299 152 152 152 266 266 266 F F F
164972024-03-06T21:04:46.144Z 15 300-319 152 152 152 266 266 266 F F F
164982024-03-06T21:04:46.144Z 16 320-339 152 152 152 266 266 266 F F F
164992024-03-06T21:04:46.144Z 17 340-359 148 148 148 261 261 261 F F F
165002024-03-06T21:04:46.144Z 18 360-379 150 150 150 264 264 264 F F F
165012024-03-06T21:04:46.144Z 19 380-399 148 148 148 261 261 261 F F F
165022024-03-06T21:04:46.144Z 20 400-419 148 148 148 261 261 261 F F F
165032024-03-06T21:04:46.144Z 21 420-439 153 153 153 267 267 267 F F F
165042024-03-06T21:04:46.144Z 22 440-459 153 153 153 267 267 267 F F F
165052024-03-06T21:04:46.144Z 23 460-479 145 145 145 257 257 257 F F F
165062024-03-06T21:04:46.144Z 24 480-499 153 153 153 267 267 267 F F F
165072024-03-06T21:04:46.145Z 25 500-519 151 151 151 265 265 265 F F F
165082024-03-06T21:04:46.145Z 26 520-539 152 152 152 266 266 266 F F F
165092024-03-06T21:04:46.145Z 27 540-559 151 151 151 265 265 265 F F F
165102024-03-06T21:04:46.145Z 28 560-579 153 153 153 267 267 267 F F F
165112024-03-06T21:04:46.145Z 29 580-599 148 148 148 261 261 261 F F F
165122024-03-06T21:04:46.145ZMax gen: 153, Max flush: 267
165132024-03-06T21:04:46.145Z
165142024-03-06T21:04:46.145ZBegin loop 77
165152024-03-06T21:04:46.145ZDownstairs to restart: 2
165162024-03-06T21:04:46.145Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.122782856Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755}
165172024-03-06T21:04:46.145Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.123134401Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755}
165182024-03-06T21:04:46.145ZCrucible runtime is spawned
165192024-03-06T21:04:46.145Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.123254413Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755}
165202024-03-06T21:04:46.145Z{"msg":"Crucible e9649e38-8d92-4452-a8a2-09f25d1e4b23 has session id: b85d7e4d-a694-43d4-99a8-4f990cc4559e","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.12329558Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165212024-03-06T21:04:46.145Z{"msg":"Upstairs opts: Upstairs UUID: e9649e38-8d92-4452-a8a2-09f25d1e4b23, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.123354406Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165222024-03-06T21:04:46.145Z{"msg":"Crucible stats registered with UUID: e9649e38-8d92-4452-a8a2-09f25d1e4b23","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.123395383Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165232024-03-06T21:04:46.145Z{"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.123424561Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755}
165242024-03-06T21:04:46.145Z{"msg":"e9649e38-8d92-4452-a8a2-09f25d1e4b23 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.123455089Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165252024-03-06T21:04:46.145Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.123481128Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"client":"0","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165262024-03-06T21:04:46.145Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.123518135Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"client":"0","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165272024-03-06T21:04:46.145Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.123547293Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"client":"1","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165282024-03-06T21:04:46.145Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.123602229Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"client":"1","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165292024-03-06T21:04:46.145Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.123635357Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"client":"2","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165302024-03-06T21:04:46.146Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.123676764Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"client":"2","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165312024-03-06T21:04:46.146Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.123707322Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"":"io task","client":"2","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165322024-03-06T21:04:46.146Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.123746389Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"":"io task","client":"0","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165332024-03-06T21:04:46.146Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.123778657Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"":"io task","client":"1","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165342024-03-06T21:04:46.146Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.123936036Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"":"io task","client":"0","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165352024-03-06T21:04:46.146Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.123978843Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"":"io task","client":"2","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165362024-03-06T21:04:46.146Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.124001412Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"":"io task","client":"1","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165372024-03-06T21:04:46.146Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.124625239Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"client":"1","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165382024-03-06T21:04:46.146Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.124675235Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"client":"1","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165392024-03-06T21:04:46.146Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.12475183Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"client":"0","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165402024-03-06T21:04:46.146Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.124806976Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"client":"0","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165412024-03-06T21:04:46.146Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.125014002Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"client":"2","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165422024-03-06T21:04:46.146Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.125077818Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"client":"2","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165432024-03-06T21:04:46.146Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.12532609Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"client":"1","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165442024-03-06T21:04:46.146Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.125462671Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"client":"0","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165452024-03-06T21:04:46.146Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.125809337Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"client":"2","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165462024-03-06T21:04:46.146Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.126030422Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"client":"1","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165472024-03-06T21:04:46.146Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.126081708Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"client":"1","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165482024-03-06T21:04:46.147Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.126121146Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165492024-03-06T21:04:46.147Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.126151813Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"client":"0","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165502024-03-06T21:04:46.147Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.126186541Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"client":"0","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165512024-03-06T21:04:46.147Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.126219789Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165522024-03-06T21:04:46.147Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.126377388Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"client":"2","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165532024-03-06T21:04:46.147Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.126435744Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"client":"2","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165542024-03-06T21:04:46.147Z{"msg":"[0]R flush_numbers[0..12]: [263, 266, 265, 265, 267, 267, 266, 262, 267, 259, 267, 266]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.126471811Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"":"downstairs","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165552024-03-06T21:04:46.147Z{"msg":"[0]R generation[0..12]: [150, 152, 151, 151, 153, 153, 152, 150, 153, 146, 153, 152]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.12648832Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"":"downstairs","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165562024-03-06T21:04:46.147Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.126510449Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"":"downstairs","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165572024-03-06T21:04:46.147Z{"msg":"[1]R flush_numbers[0..12]: [263, 266, 265, 265, 267, 267, 266, 262, 267, 259, 267, 266]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.126525218Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"":"downstairs","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165582024-03-06T21:04:46.147Z{"msg":"[1]R generation[0..12]: [150, 152, 151, 151, 153, 153, 152, 150, 153, 146, 153, 152]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
165592024-03-06T21:04:46.147Z,"time":"2024-03-06T21:04:40.126560995Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"":"downstairs","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165602024-03-06T21:04:46.147Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.126605702Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"":"downstairs","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165612024-03-06T21:04:46.147Z{"msg":"[2]R flush_numbers[0..12]: [263, 266, 265, 265, 267, 267, 266, 262, 267, 259, 267, 266]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.12663826Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"":"downstairs","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165622024-03-06T21:04:46.147Z{"msg":"[2]R generation[0..12]: [150, 152, 151, 151, 153, 153, 152, 150, 153, 146, 153, 152]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.126674257Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"":"downstairs","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165632024-03-06T21:04:46.147Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.126709865Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"":"downstairs","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165642024-03-06T21:04:46.147Z{"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.126744992Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"":"downstairs","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165652024-03-06T21:04:46.147Z{"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.12678598Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"":"downstairs","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165662024-03-06T21:04:46.147Z{"msg":"Next flush: 268","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.126803328Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"":"downstairs","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165672024-03-06T21:04:46.147Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.126824467Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"":"downstairs","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165682024-03-06T21:04:46.147Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.126852895Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165692024-03-06T21:04:46.148Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.126875803Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"client":"0","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165702024-03-06T21:04:46.148Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.126911671Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"client":"1","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165712024-03-06T21:04:46.148Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.126944859Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"client":"2","":"downstairs","session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165722024-03-06T21:04:46.148ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
165732024-03-06T21:04:46.148Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.126984646Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165742024-03-06T21:04:46.148Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.127043802Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165752024-03-06T21:04:46.148Z{"msg":"e9649e38-8d92-4452-a8a2-09f25d1e4b23 is now active with session: b85d7e4d-a694-43d4-99a8-4f990cc4559e","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.12706847Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165762024-03-06T21:04:46.148Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.127099368Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755,"session_id":"b85d7e4d-a694-43d4-99a8-4f990cc4559e"}
165772024-03-06T21:04:46.148Z{"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:40.127120437Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755}
165782024-03-06T21:04:46.148ZLoading write count information from file "/tmp/test_repair_verify.data"
165792024-03-06T21:04:46.148ZRun Repair workload
165802024-03-06T21:04:46.148Z01/30 Write block 284 len 4096 data: 14
165812024-03-06T21:04:46.171Z02/30 Write block 371 len 28672 data: 15 13 14 13 17 12 10
165822024-03-06T21:04:46.171Z03/30 Write block 397 len 8192 data: 10 10
165832024-03-06T21:04:46.171Z04/30 Write block 596 len 4096 data: 7
165842024-03-06T21:04:46.171Z05/30 Write block 585 len 40960 data: 13 15 15 14 14 14 15 15 10 11
165852024-03-06T21:04:46.171Z06/30 Write block 452 len 8192 data: 12 10
165862024-03-06T21:04:46.171Z07/30 Write block 514 len 24576 data: 14 16 15 13 12 11
165872024-03-06T21:04:46.171Z08/30 Write block 112 len 24576 data: 7 4 7 7 7 9
165882024-03-06T21:04:46.171Z09/30 Read block 129 len 36864
165892024-03-06T21:04:46.171Z10/30 Write block 534 len 8192 data: 13 15
165902024-03-06T21:04:46.171Z11/30 Write block 208 len 24576 data: 13 10 12 9 10 9
165912024-03-06T21:04:46.172Z12/30 Read block 207 len 32768
165922024-03-06T21:04:46.172Z13/30 Write block 147 len 20480 data: 9 12 14 14 12
165932024-03-06T21:04:46.172Z14/30 Write block 72 len 32768 data: 11 10 10 9 7 5 8 10
165942024-03-06T21:04:46.172Z15/30 Read block 91 len 28672
165952024-03-06T21:04:46.172Z16/30 Read block 467 len 32768
165962024-03-06T21:04:46.172Z17/30 Read block 583 len 4096
165972024-03-06T21:04:46.172Z18/30 Write block 338 len 24576 data: 14 12 14 13 12 13
165982024-03-06T21:04:46.172Z19/30 Read block 185 len 8192
165992024-03-06T21:04:46.172Z20/30 Write block 376 len 28672 data: 13 11 11 16 16 15 16
166002024-03-06T21:04:46.172Z21/30 Read block 453 len 20480
166012024-03-06T21:04:46.172Z22/30 Read block 21 len 4096
166022024-03-06T21:04:46.172Z23/30 Write block 316 len 24576 data: 10 8 12 12 11 14
166032024-03-06T21:04:46.172Z24/30 Read block 268 len 32768
166042024-03-06T21:04:46.173Z25/30 Flush
166052024-03-06T21:04:46.173Z26/30 Write block 358 len 8192 data: 14 12
166062024-03-06T21:04:46.173Z27/30 Read block 412 len 20480
166072024-03-06T21:04:46.173Z28/30 Write block 210 len 24576 data: 13 10 11 10 12 15
166082024-03-06T21:04:46.173Z29/30 Read block 564 len 36864
166092024-03-06T21:04:46.173Z30/30 Write block 290 len 12288 data: 14 12 9
166102024-03-06T21:04:46.173Z----------------------------------------------------------------
166112024-03-06T21:04:46.173Z Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30
166122024-03-06T21:04:46.173ZStates: Active Active Active
166132024-03-06T21:04:46.173ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
166142024-03-06T21:04:46.173Z 1 Acked 1000 Write 1 Done Done Done false
166152024-03-06T21:04:46.173Z 2 Acked 1001 Write 7 Done Done Done false
166162024-03-06T21:04:46.173Z 3 Acked 1002 Write 2 Done Done Done false
166172024-03-06T21:04:46.173Z 4 Acked 1003 Write 1 Done Done Done false
166182024-03-06T21:04:46.173Z 5 Acked 1004 Write 10 Done Done Done false
166192024-03-06T21:04:46.173Z 6 Acked 1005 Write 2 Done Done Done false
166202024-03-06T21:04:46.173Z 7 Acked 1006 Write 6 Done Done Done false
166212024-03-06T21:04:46.173Z 8 Acked 1007 Write 6 Done Done Sent false
166222024-03-06T21:04:46.173Z 9 Acked 1008 Read 9 Done Done Sent false
166232024-03-06T21:04:46.173Z 10 Acked 1009 Write 2 Done Done Sent false
166242024-03-06T21:04:46.173Z 11 Acked 1010 Write 6 Done Done Sent false
166252024-03-06T21:04:46.173Z 12 Acked 1011 Read 8 Done Done Sent false
166262024-03-06T21:04:46.173Z 13 Acked 1012 Write 5 Done Done Sent false
166272024-03-06T21:04:46.174Z 14 Acked 1013 Write 8 Done Done Sent false
166282024-03-06T21:04:46.174Z 15 Acked 1014 Read 7 Done Done Sent false
166292024-03-06T21:04:46.174Z 16 Acked 1015 Read 8 Done Done Sent false
166302024-03-06T21:04:46.174Z 17 Acked 1016 Read 1 Done Done Sent false
166312024-03-06T21:04:46.174Z 18 Acked 1017 Write 6 Done Done Sent false
166322024-03-06T21:04:46.174Z 19 Acked 1018 Read 2 Done Done Sent false
166332024-03-06T21:04:46.174Z 20 Acked 1019 Write 7 Done Done Sent false
166342024-03-06T21:04:46.174Z 21 Acked 1020 Read 5 Done Done Sent false
166352024-03-06T21:04:46.174Z 22 Acked 1021 Read 1 Done Done Sent false
166362024-03-06T21:04:46.174Z 23 Acked 1022 Write 6 Done Done Sent false
166372024-03-06T21:04:46.174Z 24 Acked 1023 Read 8 Done Done Sent false
166382024-03-06T21:04:46.174Z 25 Acked 1024 Flush 0 Done Done Sent false
166392024-03-06T21:04:46.174Z 26 Acked 1025 Write 2 Done Done Sent false
166402024-03-06T21:04:46.174Z 27 Acked 1026 Read 5 Done Done Sent false
166412024-03-06T21:04:46.174Z 28 Acked 1027 Write 6 Done Done Sent false
166422024-03-06T21:04:46.174Z 29 Acked 1028 Read 9 Done Done Sent false
166432024-03-06T21:04:46.174Z 30 Acked 1029 Write 3 Sent Sent Sent false
166442024-03-06T21:04:46.174Z STATES DS:0 DS:1 DS:2 TOTAL
166452024-03-06T21:04:46.174Z New 0 0 0 0
166462024-03-06T21:04:46.174Z Sent 1 1 23 25
166472024-03-06T21:04:46.174Z Done 29 29 7 65
166482024-03-06T21:04:46.174Z Skipped 0 0 0 0
166492024-03-06T21:04:46.174Z Error 0 0 0 0
166502024-03-06T21:04:46.175ZLast Flush: 1024 1024 0
166512024-03-06T21:04:46.175ZDownstairs last five completed:
166522024-03-06T21:04:46.175ZUpstairs last five completed: 30 29 28 27 26
166532024-03-06T21:04:46.175Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:04:40.16189232Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1755}
166542024-03-06T21:04:46.175ZWrote out file "/tmp/test_repair_verify.data"
166552024-03-06T21:04:46.175Z
166562024-03-06T21:04:46.175ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
166572024-03-06T21:04:46.175ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
166582024-03-06T21:04:46.175Z 0 000-019 150 150 150 263 263 263 F F F
166592024-03-06T21:04:46.175Z 1 020-039 152 152 152 266 266 266 F F F
166602024-03-06T21:04:46.175Z 2 040-059 151 151 151 265 265 265 F F F
166612024-03-06T21:04:46.175Z 3 060-079 154 154 151 268 268 265 F F F <---
166622024-03-06T21:04:46.175Z 4 080-099 153 153 153 267 267 267 F F F
166632024-03-06T21:04:46.175Z 5 100-119 154 154 153 268 268 267 F F F <---
166642024-03-06T21:04:46.175Z 6 120-139 152 152 152 266 266 266 F F F
166652024-03-06T21:04:46.175Z 7 140-159 154 154 150 268 268 262 F F F <---
166662024-03-06T21:04:46.175Z 8 160-179 153 153 153 267 267 267 F F F
166672024-03-06T21:04:46.175Z 9 180-199 146 146 146 259 259 259 F F F
166682024-03-06T21:04:46.175Z 10 200-219 154 154 153 268 268 267 T T F <---
166692024-03-06T21:04:46.175Z 11 220-239 152 152 152 266 266 266 F F F
166702024-03-06T21:04:46.175Z 12 240-259 147 147 147 260 260 260 F F F
166712024-03-06T21:04:46.175Z 13 260-279 150 150 150 264 264 264 F F F
166722024-03-06T21:04:46.176Z 14 280-299 154 154 152 268 268 266 T T T <---
166732024-03-06T21:04:46.176Z 15 300-319 154 154 152 268 268 266 F F F <---
166742024-03-06T21:04:46.176Z 16 320-339 154 154 152 268 268 266 F F F <---
166752024-03-06T21:04:46.176Z 17 340-359 154 154 148 268 268 261 T T F <---
166762024-03-06T21:04:46.176Z 18 360-379 154 154 150 268 268 264 F F T <---
166772024-03-06T21:04:46.176Z 19 380-399 154 154 148 268 268 261 F F T <---
166782024-03-06T21:04:46.176Z 20 400-419 148 148 148 261 261 261 F F F
166792024-03-06T21:04:46.176Z 21 420-439 153 153 153 267 267 267 F F F
166802024-03-06T21:04:46.176Z 22 440-459 154 154 153 268 268 267 F F T <---
166812024-03-06T21:04:46.291Z 23 460-479 145 145 145 257 257 257 F F F
166822024-03-06T21:04:46.291Z 24 480-499 153 153 153 267 267 267 F F F
166832024-03-06T21:04:46.292Z 25 500-519 154 154 151 268 268 265 F F T <---
166842024-03-06T21:04:46.292Z 26 520-539 154 154 152 268 268 266 F F F <---
166852024-03-06T21:04:46.292Z 27 540-559 151 151 151 265 265 265 F F F
166862024-03-06T21:04:46.292Z 28 560-579 153 153 153 267 267 267 F F F
166872024-03-06T21:04:46.292Z 29 580-599 154 154 148 268 268 261 F F T <---
166882024-03-06T21:04:46.292ZMax gen: 154, Max flush: 268
166892024-03-06T21:04:46.292ZError: Difference in extent metadata found!
166902024-03-06T21:04:46.292ZA Difference in extent metadata is expected here
166912024-03-06T21:04:46.292ZOn loop 77
166922024-03-06T21:04:46.292Z
166932024-03-06T21:04:46.292ZVerifying data now
166942024-03-06T21:04:47.178ZNo change after verify
166952024-03-06T21:04:47.178ZLoop: 77 Downstairs dump after verify (and repair):
166962024-03-06T21:04:47.196ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
166972024-03-06T21:04:47.196Z 0 000-019 150 150 150 263 263 263 F F F
166982024-03-06T21:04:47.196Z 1 020-039 152 152 152 266 266 266 F F F
166992024-03-06T21:04:47.196Z 2 040-059 151 151 151 265 265 265 F F F
167002024-03-06T21:04:47.197Z 3 060-079 154 154 154 268 268 268 F F F
167012024-03-06T21:04:47.197Z 4 080-099 153 153 153 267 267 267 F F F
167022024-03-06T21:04:47.197Z 5 100-119 154 154 154 268 268 268 F F F
167032024-03-06T21:04:47.197Z 6 120-139 152 152 152 266 266 266 F F F
167042024-03-06T21:04:47.197Z 7 140-159 154 154 154 268 268 268 F F F
167052024-03-06T21:04:47.197Z 8 160-179 153 153 153 267 267 267 F F F
167062024-03-06T21:04:47.197Z 9 180-199 146 146 146 259 259 259 F F F
167072024-03-06T21:04:47.197Z 10 200-219 155 155 155 269 269 269 F F F
167082024-03-06T21:04:47.197Z 11 220-239 152 152 152 266 266 266 F F F
167092024-03-06T21:04:47.197Z 12 240-259 147 147 147 260 260 260 F F F
167102024-03-06T21:04:47.197Z 13 260-279 150 150 150 264 264 264 F F F
167112024-03-06T21:04:47.198Z 14 280-299 155 155 155 269 269 269 F F F
167122024-03-06T21:04:47.198Z 15 300-319 154 154 154 268 268 268 F F F
167132024-03-06T21:04:47.198Z 16 320-339 154 154 154 268 268 268 F F F
167142024-03-06T21:04:47.198Z 17 340-359 155 155 155 269 269 269 F F F
167152024-03-06T21:04:47.198Z 18 360-379 154 154 154 268 268 268 F F F
167162024-03-06T21:04:47.198Z 19 380-399 154 154 154 268 268 268 F F F
167172024-03-06T21:04:47.213Z 20 400-419 148 148 148 261 261 261 F F F
167182024-03-06T21:04:47.213Z 21 420-439 153 153 153 267 267 267 F F F
167192024-03-06T21:04:47.213Z 22 440-459 154 154 154 268 268 268 F F F
167202024-03-06T21:04:47.213Z 23 460-479 145 145 145 257 257 257 F F F
167212024-03-06T21:04:47.213Z 24 480-499 153 153 153 267 267 267 F F F
167222024-03-06T21:04:47.214Z 25 500-519 154 154 154 268 268 268 F F F
167232024-03-06T21:04:47.214Z 26 520-539 154 154 154 268 268 268 F F F
167242024-03-06T21:04:47.214Z 27 540-559 151 151 151 265 265 265 F F F
167252024-03-06T21:04:47.214Z 28 560-579 153 153 153 267 267 267 F F F
167262024-03-06T21:04:47.215Z 29 580-599 154 154 154 268 268 268 F F F
167272024-03-06T21:04:47.215ZMax gen: 155, Max flush: 269
167282024-03-06T21:04:47.215Z
167292024-03-06T21:04:47.215ZBegin loop 78
167302024-03-06T21:04:47.215ZDownstairs to restart: 0
167312024-03-06T21:04:47.240Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.965237331Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763}
167322024-03-06T21:04:47.240Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.96554291Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763}
167332024-03-06T21:04:47.240Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.965574797Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763}
167342024-03-06T21:04:47.240Z{"msg":"Crucible 085e3497-9759-4ed3-a469-e67a4909c16a has session id: 7346e0f7-314a-4038-bc3a-50c34d62f69a","v":0,"Crucible runtime is spawned
167352024-03-06T21:04:47.240Zname":"crucible","level":30,"time":"2024-03-06T21:04:45.965624694Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167362024-03-06T21:04:47.241Z{"msg":"Upstairs opts: Upstairs UUID: 085e3497-9759-4ed3-a469-e67a4909c16a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.965673381Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167372024-03-06T21:04:47.241Z{"msg":"Crucible stats registered with UUID: 085e3497-9759-4ed3-a469-e67a4909c16a","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.965713158Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167382024-03-06T21:04:47.241Z{"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.965733067Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763}
167392024-03-06T21:04:47.241Z{"msg":"085e3497-9759-4ed3-a469-e67a4909c16a active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.965765915Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167402024-03-06T21:04:47.241Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.965783453Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"client":"0","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167412024-03-06T21:04:47.241Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.965811291Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"client":"0","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167422024-03-06T21:04:47.241Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.96583054Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"client":"1","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167432024-03-06T21:04:47.242Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.965863858Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"client":"1","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167442024-03-06T21:04:47.242Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.965891766Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"client":"2","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167452024-03-06T21:04:47.242Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.965945112Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"client":"2","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167462024-03-06T21:04:47.242Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.96597218Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"":"io task","client":"0","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167472024-03-06T21:04:47.243Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.966003428Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"":"io task","client":"2","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167482024-03-06T21:04:47.243Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.966049015Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"":"io task","client":"1","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167492024-03-06T21:04:47.243Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.966212384Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"":"io task","client":"1","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167502024-03-06T21:04:47.243Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.966283299Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"":"io task","client":"0","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167512024-03-06T21:04:47.244Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.966327246Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"":"io task","client":"2","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167522024-03-06T21:04:47.672Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.967079484Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"client":"2","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167532024-03-06T21:04:47.672Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.9671367Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"client":"2","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167542024-03-06T21:04:47.672Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.967165388Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"client":"1","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167552024-03-06T21:04:47.672Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.967185737Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"client":"1","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167562024-03-06T21:04:47.672Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.967687502Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"client":"1","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167572024-03-06T21:04:47.672Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.967817173Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"client":"2","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167582024-03-06T21:04:47.672Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.968141981Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"client":"0","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167592024-03-06T21:04:47.672Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.968204216Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"client":"0","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167602024-03-06T21:04:47.673Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.968369995Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"client":"2","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167612024-03-06T21:04:47.673Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.968401803Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"client":"2","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167622024-03-06T21:04:47.673Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.968416922Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167632024-03-06T21:04:47.673Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.968429301Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"client":"1","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167642024-03-06T21:04:47.673Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.968450649Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"client":"1","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167652024-03-06T21:04:47.673Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.968464638Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167662024-03-06T21:04:47.673Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.968755608Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"client":"0","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167672024-03-06T21:04:47.674Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.96917068Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"client":"0","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167682024-03-06T21:04:47.674Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.969200168Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"client":"0","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167692024-03-06T21:04:47.674Z{"msg":"[0]R flush_numbers[0..12]: [263, 266, 265, 268, 267, 268, 266, 268, 267, 259, 269, 266]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.969215357Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"":"downstairs","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167702024-03-06T21:04:47.674Z{"msg":"[0]R generation[0..12]: [150, 152, 151, 154, 153, 154, 152, 154, 153, 146, 155, 152]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.969230116Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"":"downstairs","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167712024-03-06T21:04:47.674Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.969244855Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"":"downstairs","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167722024-03-06T21:04:47.674Z{"msg":"[1]R flush_numbers[0..12]: [263, 266, 265, 268, 267, 268, 266, 268, 267, 259, 269, 266]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.969265323Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"":"downstairs","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167732024-03-06T21:04:47.674Z{"msg":"[1]R generation[0..12]: [150, 152, 151, 154, 153, 154, 152, 154, 153, 146, 155, 152]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.969279782Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"":"downstairs","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167742024-03-06T21:04:47.674Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.969300351Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"":"downstairs","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167752024-03-06T21:04:47.674Z{"msg":"[2]R flush_numbers[0..12]: [263, 266, 265, 268, 267, 268, 266, 268, 267, 259, 269, 266]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.96931427Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"":"downstairs","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167762024-03-06T21:04:47.674Z{"msg":"[2]R generation[0..12]: [150, 152, 151, 154, 153, 154, 152, 154, 153, 146, 155, 152]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.969339258Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"":"downstairs","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167772024-03-06T21:04:47.675Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
167782024-03-06T21:04:47.675Z,"time":"2024-03-06T21:04:45.969373326Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"":"downstairs","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167792024-03-06T21:04:47.675Z{"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.969408124Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"":"downstairs","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167802024-03-06T21:04:47.675Z{"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.969435971Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"":"downstairs","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167812024-03-06T21:04:47.675Z{"msg":"Next flush: 270","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.969470759Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"":"downstairs","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167822024-03-06T21:04:47.675Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.969510896Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"":"downstairs","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167832024-03-06T21:04:47.675Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.969549354Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167842024-03-06T21:04:47.675Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.969585491Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"client":"0","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167852024-03-06T21:04:47.675Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.969621289Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"client":"1","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167862024-03-06T21:04:47.675Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.969657326Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"client":"2","":"downstairs","session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167872024-03-06T21:04:47.675Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.969695673Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167882024-03-06T21:04:47.675Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.969719122Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167892024-03-06T21:04:47.676Z{"msg":"085e3497-9759-4ed3-a469-e67a4909c16a is now active with session: 7346e0f7-314a-4038-bc3a-50c34d62f69a","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.969732361Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167902024-03-06T21:04:47.676Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.96975088Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763,"session_id":"7346e0f7-314a-4038-bc3a-50c34d62f69a"}
167912024-03-06T21:04:47.676Z{"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:45.969763579Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763}
167922024-03-06T21:04:47.676ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
167932024-03-06T21:04:47.676ZLoading write count information from file "/tmp/test_repair_verify.data"
167942024-03-06T21:04:47.676ZRun Repair workload
167952024-03-06T21:04:47.676Z01/30 Write block 362 len 36864 data: 13 16 13 11 9 11 9 13 14
167962024-03-06T21:04:47.676Z02/30 Write block 389 len 8192 data: 12 14
167972024-03-06T21:04:47.676Z03/30 Read block 590 len 20480
167982024-03-06T21:04:47.676Z04/30 Read block 349 len 8192
167992024-03-06T21:04:47.676Z05/30 Read block 353 len 36864
168002024-03-06T21:04:47.676Z06/30 Read block 67 len 40960
168012024-03-06T21:04:47.676Z07/30 Read block 35 len 20480
168022024-03-06T21:04:47.677Z08/30 Flush
168032024-03-06T21:04:47.677Z09/30 Write block 72 len 16384 data: 12 11 11 10
168042024-03-06T21:04:47.677Z10/30 Write block 198 len 40960 data: 15 13 13 11 9 9 10 11 12 9
168052024-03-06T21:04:47.677Z11/30 Flush
168062024-03-06T21:04:47.677Z12/30 Write block 411 len 40960 data: 11 9 11 10 9 12 12 12 10 11
168072024-03-06T21:04:47.677Z13/30 Write block 348 len 20480 data: 13 12 12 12 11
168082024-03-06T21:04:47.677Z14/30 Read block 457 len 8192
168092024-03-06T21:04:47.677Z15/30 Write block 117 len 12288 data: 10 10 11
168102024-03-06T21:04:47.677Z16/30 Write block 3 len 40960 data: 9 12 12 12 13 11 11 11 15 12
168112024-03-06T21:04:47.677Z17/30 Read block 170 len 12288
168122024-03-06T21:04:47.677Z18/30 Write block 133 len 16384 data: 10 10 10 11
168132024-03-06T21:04:47.677Z19/30 Read block 118 len 24576
168142024-03-06T21:04:47.678Z20/30 Write block 300 len 28672 data: 8 8 12 12 14 15 17
168152024-03-06T21:04:47.678Z21/30 Flush
168162024-03-06T21:04:47.717Z22/30 Flush
168172024-03-06T21:04:47.717Z23/30 Write block 207 len 24576 data: 10 14 11 14 11 12
168182024-03-06T21:04:47.717Z24/30 Write block 446 len 24576 data: 10 8 12 11 11 13
168192024-03-06T21:04:47.717Z25/30 Write block 143 len 20480 data: 7 7 7 7 10
168202024-03-06T21:04:47.717Z26/30 Read block 585 len 28672
168212024-03-06T21:04:47.717Z27/30 Write block 579 len 20480 data: 9 8 11 10 10
168222024-03-06T21:04:47.717Z28/30 Read block 2 len 20480
168232024-03-06T21:04:47.717Z29/30 Write block 21 len 40960 data: 14 17 17 14 13 13 13 12 14 12
168242024-03-06T21:04:47.717Z30/30 Read block 38 len 12288
168252024-03-06T21:04:47.717Z----------------------------------------------------------------
168262024-03-06T21:04:47.717Z Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:30
168272024-03-06T21:04:47.717ZStates: Active Active Active
168282024-03-06T21:04:47.717ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
168292024-03-06T21:04:47.717Z 1 Acked 1000 Write 9 Done Done Done false
168302024-03-06T21:04:47.717Z 2 Acked 1001 Write 2 Done Done Done false
168312024-03-06T21:04:47.717Z 3 Acked 1002 Read 5 Done Done Done false
168322024-03-06T21:04:47.718Z 4 Acked 1003 Read 2 Sent Done Done false
168332024-03-06T21:04:47.718Z 5 Acked 1004 Read 9 Sent Done Done false
168342024-03-06T21:04:47.718Z 6 Acked 1005 Read 10 Sent Done Done false
168352024-03-06T21:04:47.718Z 7 Acked 1006 Read 5 Sent Done Done false
168362024-03-06T21:04:47.718Z 8 Acked 1007 Flush 0 Sent Done Done false
168372024-03-06T21:04:47.718Z 9 Acked 1008 Write 4 Sent Done Done false
168382024-03-06T21:04:47.718Z 10 Acked 1009 Write 10 Sent Done Done false
168392024-03-06T21:04:47.718Z 11 Acked 1010 Flush 0 Sent Done Done false
168402024-03-06T21:04:47.718Z 12 Acked 1011 Write 10 Sent Done Done false
168412024-03-06T21:04:47.718Z 13 Acked 1012 Write 5 Sent Done Done false
168422024-03-06T21:04:47.718Z 14 Acked 1013 Read 2 Sent Done Done false
168432024-03-06T21:04:47.718Z 15 Acked 1014 Write 3 Sent Done Done false
168442024-03-06T21:04:47.718Z 16 Acked 1015 Write 10 Sent Done Done false
168452024-03-06T21:04:47.718Z 17 Acked 1016 Read 3 Sent Done Done false
168462024-03-06T21:04:47.718Z 18 Acked 1017 Write 4 Sent Done Done false
168472024-03-06T21:04:47.718Z 19 Acked 1018 Read 6 Sent Done Done false
168482024-03-06T21:04:47.718Z 20 Acked 1019 Write 7 Sent Done Done false
168492024-03-06T21:04:47.719Z 21 Acked 1020 Flush 0 Sent Done Done false
168502024-03-06T21:04:47.719Z 22 Acked 1021 Flush 0 Sent Done Done false
168512024-03-06T21:04:47.719Z 23 Acked 1022 Write 6 Sent Done Done false
168522024-03-06T21:04:47.719Z 24 Acked 1023 Write 6 Sent Done Done false
168532024-03-06T21:04:47.719Z 25 Acked 1024 Write 5 Sent Done Done false
168542024-03-06T21:04:47.719Z 26 Acked 1025 Read 7 Sent Done Done false
168552024-03-06T21:04:47.719Z 27 Acked 1026 Write 5 Sent Done Done false
168562024-03-06T21:04:47.719Z 28 Acked 1027 Read 5 Sent Done Done false
168572024-03-06T21:04:47.719Z 29 Acked 1028 Write 10 Sent Sent Done false
168582024-03-06T21:04:47.719Z 30 Acked 1029 Read 3 Sent Sent Done false
168592024-03-06T21:04:47.720Z STATES DS:0 DS:1 DS:2 TOTAL
168602024-03-06T21:04:47.720Z New 0 0 0 0
168612024-03-06T21:04:47.720Z Sent 27 2 0 29
168622024-03-06T21:04:47.720Z Done 3 28 30 61
168632024-03-06T21:04:47.720Z Skipped 0 0 0 0
168642024-03-06T21:04:47.720Z Error 0 0 0 0
168652024-03-06T21:04:47.720ZLast Flush: 0 1021 1021
168662024-03-06T21:04:47.720ZDownstairs last five completed:
168672024-03-06T21:04:47.720ZUpstairs last five completed: 30 29 28 27 26
168682024-03-06T21:04:47.720Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:04:46.007572893Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1763}
168692024-03-06T21:04:47.720ZWrote out file "/tmp/test_repair_verify.data"
168702024-03-06T21:04:47.720Z
168712024-03-06T21:04:47.720ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
168722024-03-06T21:04:47.720ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
168732024-03-06T21:04:47.720Z 0 000-019 150 156 156 263 272 272 F F F <---
168742024-03-06T21:04:47.720Z 1 020-039 152 152 152 266 266 266 F T T <---
168752024-03-06T21:04:47.720Z 2 040-059 151 151 151 265 265 265 F F F
168762024-03-06T21:04:47.720Z 3 060-079 154 156 156 268 271 271 F F F <---
168772024-03-06T21:04:47.720Z 4 080-099 153 153 153 267 267 267 F F F
168782024-03-06T21:04:47.721Z 5 100-119 154 156 156 268 272 272 F F F <---
168792024-03-06T21:04:47.721Z 6 120-139 152 156 156 266 272 272 F F F <---
168802024-03-06T21:04:47.721Z 7 140-159 154 154 154 268 268 268 F T T <---
168812024-03-06T21:04:47.721Z 8 160-179 153 153 153 267 267 267 F F F
168822024-03-06T21:04:47.721Z 9 180-199 146 156 156 259 271 271 F F F <---
168832024-03-06T21:04:47.721Z 10 200-219 155 156 156 269 271 271 F T T <---
168842024-03-06T21:04:47.721Z 11 220-239 152 152 152 266 266 266 F F F
168852024-03-06T21:04:47.721Z 12 240-259 147 147 147 260 260 260 F F F
168862024-03-06T21:04:47.721Z 13 260-279 150 150 150 264 264 264 F F F
168872024-03-06T21:04:47.721Z 14 280-299 155 155 155 269 269 269 F F F
168882024-03-06T21:04:47.721Z 15 300-319 154 156 156 268 272 272 F F F <---
168892024-03-06T21:04:47.721Z 16 320-339 154 154 154 268 268 268 F F F
168902024-03-06T21:04:47.721Z 17 340-359 155 156 156 269 272 272 F F F <---
168912024-03-06T21:04:47.721Z 18 360-379 154 156 156 268 270 270 T F F <---
168922024-03-06T21:04:47.721Z 19 380-399 154 156 156 268 270 270 T F F <---
168932024-03-06T21:04:47.721Z 20 400-419 148 156 156 261 272 272 F F F <---
168942024-03-06T21:04:47.721Z 21 420-439 153 156 156 267 272 272 F F F <---
168952024-03-06T21:04:47.722Z 22 440-459 154 154 154 268 268 268 F T T <---
168962024-03-06T21:04:47.722Z 23 460-479 145 145 145 257 257 257 F F F
168972024-03-06T21:04:47.722Z 24 480-499 153 153 153 267 267 267 F F F
168982024-03-06T21:04:47.722Z 25 500-519 154 154 154 268 268 268 F F F
168992024-03-06T21:04:47.722Z 26 520-539 154 154 154 268 268 268 F F F
169002024-03-06T21:04:47.722Z 27 540-559 151 151 151 265 265 265 F F F
169012024-03-06T21:04:47.722Z 28 560-579 153 153 153 267 267 267 F T T <---
169022024-03-06T21:04:47.722Z 29 580-599 154 154 154 268 268 268 F T T <---
169032024-03-06T21:04:47.722ZMax gen: 156, Max flush: 272
169042024-03-06T21:04:47.722ZError: Difference in extent metadata found!
169052024-03-06T21:04:47.722ZA Difference in extent metadata is expected here
169062024-03-06T21:04:47.722ZOn loop 78
169072024-03-06T21:04:47.722Z
169082024-03-06T21:04:47.722ZVerifying data now
169092024-03-06T21:04:53.169ZNo change after verify
169102024-03-06T21:04:53.169ZLoop: 78 Downstairs dump after verify (and repair):
169112024-03-06T21:04:53.184ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
169122024-03-06T21:04:53.184Z 0 000-019 156 156 156 272 272 272 F F F
169132024-03-06T21:04:53.184Z 1 020-039 157 157 157 273 273 273 F F F
169142024-03-06T21:04:53.184Z 2 040-059 151 151 151 265 265 265 F F F
169152024-03-06T21:04:53.184Z 3 060-079 156 156 156 271 271 271 F F F
169162024-03-06T21:04:53.184Z 4 080-099 153 153 153 267 267 267 F F F
169172024-03-06T21:04:53.184Z 5 100-119 156 156 156 272 272 272 F F F
169182024-03-06T21:04:53.184Z 6 120-139 156 156 156 272 272 272 F F F
169192024-03-06T21:04:53.184Z 7 140-159 157 157 157 273 273 273 F F F
169202024-03-06T21:04:53.185Z 8 160-179 153 153 153 267 267 267 F F F
169212024-03-06T21:04:53.185Z 9 180-199 156 156 156 271 271 271 F F F
169222024-03-06T21:04:53.185Z 10 200-219 157 157 157 273 273 273 F F F
169232024-03-06T21:04:53.185Z 11 220-239 152 152 152 266 266 266 F F F
169242024-03-06T21:04:53.185Z 12 240-259 147 147 147 260 260 260 F F F
169252024-03-06T21:04:53.185Z 13 260-279 150 150 150 264 264 264 F F F
169262024-03-06T21:04:53.185Z 14 280-299 155 155 155 269 269 269 F F F
169272024-03-06T21:04:53.185Z 15 300-319 156 156 156 272 272 272 F F F
169282024-03-06T21:04:53.185Z 16 320-339 154 154 154 268 268 268 F F F
169292024-03-06T21:04:53.185Z 17 340-359 156 156 156 272 272 272 F F F
169302024-03-06T21:04:53.185Z 18 360-379 156 156 156 270 270 270 F F F
169312024-03-06T21:04:53.185Z 19 380-399 156 156 156 270 270 270 F F F
169322024-03-06T21:04:53.185Z 20 400-419 156 156 156 272 272 272 F F F
169332024-03-06T21:04:53.185Z 21 420-439 156 156 156 272 272 272 F F F
169342024-03-06T21:04:53.185Z 22 440-459 157 157 157 273 273 273 F F F
169352024-03-06T21:04:53.185Z 23 460-479 145 145 145 257 257 257 F F F
169362024-03-06T21:04:53.185Z 24 480-499 153 153 153 267 267 267 F F F
169372024-03-06T21:04:53.185Z 25 500-519 154 154 154 268 268 268 F F F
169382024-03-06T21:04:53.186Z 26 520-539 154 154 154 268 268 268 F F F
169392024-03-06T21:04:53.186Z 27 540-559 151 151 151 265 265 265 F F F
169402024-03-06T21:04:53.186Z 28 560-579 157 157 157 273 273 273 F F F
169412024-03-06T21:04:53.186Z 29 580-599 157 157 157 273 273 273 F F F
169422024-03-06T21:04:53.186ZMax gen: 157, Max flush: 273
169432024-03-06T21:04:53.186Z
169442024-03-06T21:04:53.186ZBegin loop 79
169452024-03-06T21:04:53.186ZDownstairs to restart: 1
169462024-03-06T21:04:53.202Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.943004171Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771}
169472024-03-06T21:04:53.202Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.943365596Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771}
169482024-03-06T21:04:53.202ZCrucible runtime is spawned
169492024-03-06T21:04:53.210Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.943425562Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771}
169502024-03-06T21:04:53.210Z{"msg":"Crucible cf74e551-b24b-483a-8866-27c063bd4337 has session id: b78edcbd-a706-4547-b65c-d93d5de01474","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.943469669Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169512024-03-06T21:04:53.210Z{"msg":"Upstairs opts: Upstairs UUID: cf74e551-b24b-483a-8866-27c063bd4337, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.943522155Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169522024-03-06T21:04:53.210Z{"msg":"Crucible stats registered with UUID: cf74e551-b24b-483a-8866-27c063bd4337","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.943557903Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169532024-03-06T21:04:53.210Z{"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.943587091Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771}
169542024-03-06T21:04:53.210Z{"msg":"cf74e551-b24b-483a-8866-27c063bd4337 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.943621819Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169552024-03-06T21:04:53.210Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.943649907Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"client":"0","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169562024-03-06T21:04:53.210Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.943682864Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"client":"0","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169572024-03-06T21:04:53.210Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.943717282Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"client":"1","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169582024-03-06T21:04:53.210Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.94374768Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"client":"1","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169592024-03-06T21:04:53.210Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.943775408Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"client":"2","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169602024-03-06T21:04:53.211Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.943868072Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"client":"2","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169612024-03-06T21:04:53.211Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.943912279Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"":"io task","client":"2","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169622024-03-06T21:04:53.211Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.943957416Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"":"io task","client":"0","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169632024-03-06T21:04:53.211Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.944017252Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169642024-03-06T21:04:53.211Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.944071918Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"":"io task","client":"0","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169652024-03-06T21:04:53.211Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.944110545Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169662024-03-06T21:04:53.211Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.944142403Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"":"io task","client":"2","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169672024-03-06T21:04:53.211Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.944686185Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"client":"0","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169682024-03-06T21:04:53.211Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.944727663Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"client":"0","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169692024-03-06T21:04:53.211Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.944745731Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"client":"2","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169702024-03-06T21:04:53.211Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.94476235Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"client":"2","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169712024-03-06T21:04:53.211Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.945471781Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"client":"2","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169722024-03-06T21:04:53.212Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.945509958Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"client":"0","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169732024-03-06T21:04:53.212Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.946002815Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"client":"1","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169742024-03-06T21:04:53.212Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.946055621Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"client":"1","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169752024-03-06T21:04:53.212Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.946085689Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"client":"2","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169762024-03-06T21:04:53.212Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.946129166Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"client":"2","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169772024-03-06T21:04:53.212Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.946147865Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169782024-03-06T21:04:53.212Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.946170983Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"client":"0","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169792024-03-06T21:04:53.212Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.946185112Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"client":"0","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169802024-03-06T21:04:53.212Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.946198861Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169812024-03-06T21:04:53.212Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.946566426Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"client":"1","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169822024-03-06T21:04:53.212Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.947038743Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"client":"1","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169832024-03-06T21:04:53.212Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.94708031Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"client":"1","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169842024-03-06T21:04:53.212Z{"msg":"[0]R flush_numbers[0..12]: [272, 273, 265, 271, 267, 272, 272, 273, 267, 271, 273, 266]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.947102659Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"":"downstairs","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169852024-03-06T21:04:53.213Z{"msg":"[0]R generation[0..12]: [156, 157, 151, 156, 153, 156, 156, 157, 153, 156, 157, 152]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.947129187Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"":"downstairs","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169862024-03-06T21:04:53.213Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.947147576Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"":"downstairs","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169872024-03-06T21:04:53.213Z{"msg":"[1]R flush_numbers[0..12]: [272, 273, 265, 271, 267, 272, 272, 273, 267, 271, 273, 266]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.947173624Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"":"downstairs","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169882024-03-06T21:04:53.213Z{"msg":"[1]R generation[0..12]: [156, 157, 151, 156, 153, 156, 156, 157, 153, 156, 157, 152]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.947201472Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"":"downstairs","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169892024-03-06T21:04:53.213Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.947220741Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"":"downstairs","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"Wait for a query_work_queue command to finish before sending IO
169902024-03-06T21:04:53.213Z}
169912024-03-06T21:04:53.213Z{"msg":"[2]R flush_numbers[0..12]: [272, 273, 265, 271, 267, 272, 272, 273, 267, 271, 273, 266]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.947251079Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"":"downstairs","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169922024-03-06T21:04:53.213Z{"msg":"[2]R generation[0..12]: [156, 157, 151, 156, 153, 156, 156, 157, 153, 156, 157, 152]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.947268907Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"":"downstairs","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169932024-03-06T21:04:53.213Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.947295115Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"":"downstairs","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169942024-03-06T21:04:53.213Z{"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.947312814Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"":"downstairs","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169952024-03-06T21:04:53.213Z{"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.947345112Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"":"downstairs","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169962024-03-06T21:04:53.213Z{"msg":"Next flush: 274","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.947361791Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"":"downstairs","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169972024-03-06T21:04:53.213Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.947386459Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"":"downstairs","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169982024-03-06T21:04:53.213Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.947403528Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
169992024-03-06T21:04:53.214Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.947426557Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"client":"0","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
170002024-03-06T21:04:53.214Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.947443455Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"client":"1","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
170012024-03-06T21:04:53.214Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.947467984Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"client":"2","":"downstairs","session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
170022024-03-06T21:04:53.214Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.947484593Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
170032024-03-06T21:04:53.214Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.947500731Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
170042024-03-06T21:04:53.214Z{"msg":"cf74e551-b24b-483a-8866-27c063bd4337 is now active with session: b78edcbd-a706-4547-b65c-d93d5de01474","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.94752469Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
170052024-03-06T21:04:53.214Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.947539989Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771,"session_id":"b78edcbd-a706-4547-b65c-d93d5de01474"}
170062024-03-06T21:04:53.214Z{"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:51.947563237Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771}
170072024-03-06T21:04:53.214ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
170082024-03-06T21:04:53.214ZLoading write count information from file "/tmp/test_repair_verify.data"
170092024-03-06T21:04:53.214ZRun Repair workload
170102024-03-06T21:04:53.214Z01/30 Write block 456 len 20480 data: 11 15 14 16 15
170112024-03-06T21:04:53.220Z02/30 Read block 563 len 20480
170122024-03-06T21:04:53.220Z03/30 Write block 9 len 8192 data: 12 12
170132024-03-06T21:04:53.220Z04/30 Read block 360 len 40960
170142024-03-06T21:04:53.220Z05/30 Read block 298 len 20480
170152024-03-06T21:04:53.220Z06/30 Read block 84 len 12288
170162024-03-06T21:04:53.220Z07/30 Read block 111 len 16384
170172024-03-06T21:04:53.220Z08/30 Write block 469 len 24576 data: 10 7 6 5 6 7
170182024-03-06T21:04:53.221Z09/30 Flush
170192024-03-06T21:04:53.221Z10/30 Write block 440 len 12288 data: 15 13 12
170202024-03-06T21:04:53.221Z11/30 Write block 343 len 16384 data: 14 14 11 9
170212024-03-06T21:04:53.225Z12/30 Read block 30 len 28672
170222024-03-06T21:04:53.225Z13/30 Flush
170232024-03-06T21:04:53.225Z14/30 Write block 388 len 20480 data: 13 13 15 10 9
170242024-03-06T21:04:53.225Z15/30 Read block 210 len 8192
170252024-03-06T21:04:53.225Z16/30 Write block 123 len 12288 data: 6 10 15
170262024-03-06T21:04:53.225Z17/30 Read block 96 len 20480
170272024-03-06T21:04:53.231Z18/30 Read block 368 len 20480
170282024-03-06T21:04:53.231Z19/30 Write block 356 len 24576 data: 17 15 15 13 13 17
170292024-03-06T21:04:53.231Z20/30 Write block 279 len 28672 data: 11 10 11 13 11 15 16
170302024-03-06T21:04:53.231Z21/30 Read block 47 len 32768
170312024-03-06T21:04:53.236Z22/30 Flush
170322024-03-06T21:04:53.236Z23/30 Flush
170332024-03-06T21:04:53.237Z24/30 Write block 439 len 32768 data: 17 16 14 13 11 11 9 11
170342024-03-06T21:04:53.237Z25/30 Write block 528 len 8192 data: 17 16
170352024-03-06T21:04:53.237Z26/30 Read block 388 len 16384
170362024-03-06T21:04:53.242Z27/30 Write block 47 len 12288 data: 12 11 9
170372024-03-06T21:04:53.242Z28/30 Read block 244 len 40960
170382024-03-06T21:04:53.242Z29/30 Read block 6 len 8192
170392024-03-06T21:04:53.243Z30/30 Read block 533 len 8192
170402024-03-06T21:04:53.243Z----------------------------------------------------------------
170412024-03-06T21:04:53.243Z Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30
170422024-03-06T21:04:53.243ZStates: Active Active Active
170432024-03-06T21:04:53.243ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
170442024-03-06T21:04:53.243Z 1 Acked 1000 Write 5 Done Done Done false
170452024-03-06T21:04:53.243Z 2 Acked 1001 Read 5 Done Done Done false
170462024-03-06T21:04:53.243Z 3 Acked 1002 Write 2 Done Sent Done false
170472024-03-06T21:04:53.243Z 4 Acked 1003 Read 10 Done Sent Done false
170482024-03-06T21:04:53.243Z 5 Acked 1004 Read 5 Done Sent Done false
170492024-03-06T21:04:53.243Z 6 Acked 1005 Read 3 Done Sent Done false
170502024-03-06T21:04:53.243Z 7 Acked 1006 Read 4 Done Sent Done false
170512024-03-06T21:04:53.243Z 8 Acked 1007 Write 6 Done Sent Done false
170522024-03-06T21:04:53.244Z 9 Acked 1008 Flush 0 Done Sent Done false
170532024-03-06T21:04:53.244Z 10 Acked 1009 Write 3 Done Sent Done false
170542024-03-06T21:04:53.244Z 11 Acked 1010 Write 4 Done Sent Done false
170552024-03-06T21:04:53.244Z 12 Acked 1011 Read 7 Done Sent Done false
170562024-03-06T21:04:53.244Z 13 Acked 1012 Flush 0 Done Sent Done false
170572024-03-06T21:04:53.244Z 14 Acked 1013 Write 5 Done Sent Done false
170582024-03-06T21:04:53.244Z 15 Acked 1014 Read 2 Done Sent Done false
170592024-03-06T21:04:53.244Z 16 Acked 1015 Write 3 Done Sent Done false
170602024-03-06T21:04:53.244Z 17 Acked 1016 Read 5 Done Sent Done false
170612024-03-06T21:04:53.244Z 18 Acked 1017 Read 5 Done Sent Done false
170622024-03-06T21:04:53.244Z 19 Acked 1018 Write 6 Done Sent Done false
170632024-03-06T21:04:53.244Z 20 Acked 1019 Write 7 Done Sent Done false
170642024-03-06T21:04:53.244Z 21 Acked 1020 Read 8 Done Sent Done false
170652024-03-06T21:04:53.244Z 22 Acked 1021 Flush 0 Done Sent Done false
170662024-03-06T21:04:53.244Z 23 Acked 1022 Flush 0 Done Sent Done false
170672024-03-06T21:04:53.245Z 24 Acked 1023 Write 8 Done Sent Done false
170682024-03-06T21:04:53.245Z 25 Acked 1024 Write 2 Done Sent Done false
170692024-03-06T21:04:53.245Z 26 Acked 1025 Read 4 Done Sent Done false
170702024-03-06T21:04:53.245Z 27 Acked 1026 Write 3 Done Sent Done false
170712024-03-06T21:04:53.245Z 28 Acked 1027 Read 10 Done Sent Done false
170722024-03-06T21:04:53.245Z 29 Acked 1028 Read 2 Done Sent Done false
170732024-03-06T21:04:53.245Z 30 Acked 1029 Read 2 Done Sent Done false
170742024-03-06T21:04:53.245Z STATES DS:0 DS:1 DS:2 TOTAL
170752024-03-06T21:04:53.245Z New 0 0 0 0
170762024-03-06T21:04:53.245Z Sent 0 28 0 28
170772024-03-06T21:04:53.245Z Done 30 2 30 62
170782024-03-06T21:04:53.245Z Skipped 0 0 0 0
170792024-03-06T21:04:53.245Z Error 0 0 0 0
170802024-03-06T21:04:53.245ZLast Flush: 1022 0 1022
170812024-03-06T21:04:53.246ZDownstairs last five completed:
170822024-03-06T21:04:53.246ZUpstairs last five completed: 30 29 28 27 26
170832024-03-06T21:04:53.246Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:04:51.982236527Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1771}
170842024-03-06T21:04:53.246ZWrote out file "/tmp/test_repair_verify.data"
170852024-03-06T21:04:53.252Z
170862024-03-06T21:04:53.252ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
170872024-03-06T21:04:53.279ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
170882024-03-06T21:04:53.280Z 0 000-019 158 156 158 274 272 274 F F F <---
170892024-03-06T21:04:53.280Z 1 020-039 157 157 157 273 273 273 F F F
170902024-03-06T21:04:53.280Z 2 040-059 151 151 151 265 265 265 T F T <---
170912024-03-06T21:04:53.281Z 3 060-079 156 156 156 271 271 271 F F F
170922024-03-06T21:04:53.281Z 4 080-099 153 153 153 267 267 267 F F F
170932024-03-06T21:04:53.281Z 5 100-119 156 156 156 272 272 272 F F F
170942024-03-06T21:04:53.281Z 6 120-139 158 156 158 276 272 276 F F F <---
170952024-03-06T21:04:53.282Z 7 140-159 157 157 157 273 273 273 F F F
170962024-03-06T21:04:53.282Z 8 160-179 153 153 153 267 267 267 F F F
170972024-03-06T21:04:53.282Z 9 180-199 156 156 156 271 271 271 F F F
170982024-03-06T21:04:53.283Z 10 200-219 157 157 157 273 273 273 F F F
170992024-03-06T21:04:53.283Z 11 220-239 152 152 152 266 266 266 F F F
171002024-03-06T21:04:53.283Z 12 240-259 147 147 147 260 260 260 F F F
171012024-03-06T21:04:53.283Z 13 260-279 158 150 158 276 264 276 F F F <---
171022024-03-06T21:04:53.283Z 14 280-299 158 155 158 276 269 276 F F F <---
171032024-03-06T21:04:53.283Z 15 300-319 156 156 156 272 272 272 F F F
171042024-03-06T21:04:53.283Z 16 320-339 154 154 154 268 268 268 F F F
171052024-03-06T21:04:53.284Z 17 340-359 158 156 158 276 272 276 F F F <---
171062024-03-06T21:04:53.290Z 18 360-379 158 156 158 276 270 276 F F F <---
171072024-03-06T21:04:53.291Z 19 380-399 158 156 158 276 270 276 F F F <---
171082024-03-06T21:04:53.291Z 20 400-419 156 156 156 272 272 272 F F F
171092024-03-06T21:04:53.291Z 21 420-439 156 156 156 272 272 272 T F T <---
171102024-03-06T21:04:53.291Z 22 440-459 158 157 158 275 273 275 T T T <---
171112024-03-06T21:04:53.291Z 23 460-479 158 145 158 274 257 274 F T F <---
171122024-03-06T21:04:53.291Z 24 480-499 153 153 153 267 267 267 F F F
171132024-03-06T21:04:53.291Z 25 500-519 154 154 154 268 268 268 F F F
171142024-03-06T21:04:53.291Z 26 520-539 154 154 154 268 268 268 T F T <---
171152024-03-06T21:04:53.291Z 27 540-559 151 151 151 265 265 265 F F F
171162024-03-06T21:04:53.291Z 28 560-579 157 157 157 273 273 273 F F F
171172024-03-06T21:04:53.291Z 29 580-599 157 157 157 273 273 273 F F F
171182024-03-06T21:04:53.292ZMax gen: 158, Max flush: 276
171192024-03-06T21:04:53.292ZError: Difference in extent metadata found!
171202024-03-06T21:04:53.292ZA Difference in extent metadata is expected here
171212024-03-06T21:04:53.292ZOn loop 79
171222024-03-06T21:04:53.292Z
171232024-03-06T21:04:53.292ZVerifying data now
171242024-03-06T21:04:57.915ZNo change after verify
171252024-03-06T21:04:57.915ZLoop: 79 Downstairs dump after verify (and repair):
171262024-03-06T21:04:57.931ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
171272024-03-06T21:04:57.931Z 0 000-019 158 158 158 274 274 274 F F F
171282024-03-06T21:04:57.931Z 1 020-039 157 157 157 273 273 273 F F F
171292024-03-06T21:04:57.931Z 2 040-059 159 159 159 277 277 277 F F F
171302024-03-06T21:04:57.931Z 3 060-079 156 156 156 271 271 271 F F F
171312024-03-06T21:04:57.931Z 4 080-099 153 153 153 267 267 267 F F F
171322024-03-06T21:04:57.931Z 5 100-119 156 156 156 272 272 272 F F F
171332024-03-06T21:04:57.931Z 6 120-139 158 158 158 276 276 276 F F F
171342024-03-06T21:04:57.931Z 7 140-159 157 157 157 273 273 273 F F F
171352024-03-06T21:04:57.931Z 8 160-179 153 153 153 267 267 267 F F F
171362024-03-06T21:04:57.931Z 9 180-199 156 156 156 271 271 271 F F F
171372024-03-06T21:04:57.931Z 10 200-219 157 157 157 273 273 273 F F F
171382024-03-06T21:04:57.932Z 11 220-239 152 152 152 266 266 266 F F F
171392024-03-06T21:04:57.932Z 12 240-259 147 147 147 260 260 260 F F F
171402024-03-06T21:04:57.932Z 13 260-279 158 158 158 276 276 276 F F F
171412024-03-06T21:04:57.932Z 14 280-299 158 158 158 276 276 276 F F F
171422024-03-06T21:04:57.932Z 15 300-319 156 156 156 272 272 272 F F F
171432024-03-06T21:04:57.932Z 16 320-339 154 154 154 268 268 268 F F F
171442024-03-06T21:04:57.932Z 17 340-359 158 158 158 276 276 276 F F F
171452024-03-06T21:04:57.932Z 18 360-379 158 158 158 276 276 276 F F F
171462024-03-06T21:04:57.932Z 19 380-399 158 158 158 276 276 276 F F F
171472024-03-06T21:04:57.932Z 20 400-419 156 156 156 272 272 272 F F F
171482024-03-06T21:04:57.932Z 21 420-439 159 159 159 277 277 277 F F F
171492024-03-06T21:04:57.932Z 22 440-459 159 159 159 277 277 277 F F F
171502024-03-06T21:04:57.932Z 23 460-479 158 158 158 274 274 274 F F F
171512024-03-06T21:04:57.936Z 24 480-499 153 153 153 267 267 267 F F F
171522024-03-06T21:04:57.936Z 25 500-519 154 154 154 268 268 268 F F F
171532024-03-06T21:04:57.936Z 26 520-539 159 159 159 277 277 277 F F F
171542024-03-06T21:04:57.936Z 27 540-559 151 151 151 265 265 265 F F F
171552024-03-06T21:04:57.936Z 28 560-579 157 157 157 273 273 273 F F F
171562024-03-06T21:04:57.936Z 29 580-599 157 157 157 273 273 273 F F F
171572024-03-06T21:04:57.936ZMax gen: 159, Max flush: 277
171582024-03-06T21:04:57.939Z
171592024-03-06T21:04:57.939ZBegin loop 80
171602024-03-06T21:04:57.939ZDownstairs to restart: 1
171612024-03-06T21:04:57.959Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.701086428Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779}
171622024-03-06T21:04:57.959Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.70134482Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779}
171632024-03-06T21:04:57.959Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.701363959Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779}
171642024-03-06T21:04:57.964Z{"msg":"Crucible fa2525c7-44c2-4e3b-96df-0b0a8ca1c5f6 has session id: 1b43be15-43e3-4e8c-8f9b-76f7e75b6807","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.701380048Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171652024-03-06T21:04:57.964ZCrucible runtime is spawned
171662024-03-06T21:04:57.964Z{"msg":"Upstairs opts: Upstairs UUID: fa2525c7-44c2-4e3b-96df-0b0a8ca1c5f6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.701486001Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171672024-03-06T21:04:57.964Z{"msg":"Crucible stats registered with UUID: fa2525c7-44c2-4e3b-96df-0b0a8ca1c5f6","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.701515979Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171682024-03-06T21:04:57.964Z{"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.701552346Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779}
171692024-03-06T21:04:57.964Z{"msg":"fa2525c7-44c2-4e3b-96df-0b0a8ca1c5f6 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.701729984Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171702024-03-06T21:04:57.964Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.701762711Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"client":"0","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171712024-03-06T21:04:57.964Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.70178857Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"client":"0","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171722024-03-06T21:04:57.964Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.701827407Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"client":"1","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171732024-03-06T21:04:57.964Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.701856355Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"client":"1","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171742024-03-06T21:04:57.964Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.701881303Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"client":"2","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171752024-03-06T21:04:57.964Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.701909441Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"client":"2","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171762024-03-06T21:04:57.964Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.701953149Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"":"io task","client":"2","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171772024-03-06T21:04:57.964Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.70207815Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"":"io task","client":"0","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171782024-03-06T21:04:57.965Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.702283216Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"":"io task","client":"1","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171792024-03-06T21:04:57.965Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.702327383Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"":"io task","client":"0","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171802024-03-06T21:04:57.965Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.702354781Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"":"io task","client":"1","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171812024-03-06T21:04:57.965Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.702379459Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"":"io task","client":"2","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171822024-03-06T21:04:57.965Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.70308314Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"client":"2","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171832024-03-06T21:04:57.965Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.703133647Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"client":"2","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171842024-03-06T21:04:57.965Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.703183684Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"client":"0","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171852024-03-06T21:04:57.965Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.703227221Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"client":"0","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171862024-03-06T21:04:57.965Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.703855157Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"client":"0","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171872024-03-06T21:04:57.965Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.703998188Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"client":"2","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171882024-03-06T21:04:57.965Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.704263169Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"client":"1","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171892024-03-06T21:04:57.965Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.704346753Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"client":"1","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171902024-03-06T21:04:57.965Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.704439067Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"client":"0","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171912024-03-06T21:04:57.965Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.704488254Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"client":"0","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171922024-03-06T21:04:57.965Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.704521172Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171932024-03-06T21:04:57.965Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.70454383Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"client":"2","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171942024-03-06T21:04:57.965Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.704612685Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"client":"2","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171952024-03-06T21:04:57.965Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.704658702Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171962024-03-06T21:04:57.965Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.704865908Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"client":"1","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171972024-03-06T21:04:57.970Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.705286949Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"client":"1","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171982024-03-06T21:04:57.970Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.705326816Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"client":"1","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
171992024-03-06T21:04:57.971Z{"msg":"[0]R flush_numbers[0..12]: [274, 273, 277, 271, 267, 272, 276, 273, 267, 271, 273, 266]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.705344255Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"":"downstairs","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
172002024-03-06T21:04:57.971Z{"msg":"[0]R generation[0..12]: [158, 157, 159, 156, 153, 156, 158, 157, 153, 156, 157, 152]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.705358734Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"":"downstairs","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
172012024-03-06T21:04:57.971Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.705380512Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"":"downstairs","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
172022024-03-06T21:04:57.971Z{"msg":"[1]R flush_numbers[0..12]: [274, 273, 277, 271, 267, 272, 276, 273, 267, 271, 273, 266]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.705394721Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"":"downstairs","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
172032024-03-06T21:04:57.971Z{"msg":"[1]R generation[0..12]: [158, 157, 159, 156, 153, 156, 158, 157, 153, 156, 157, 152]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.70541505Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"":"downstairs","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
172042024-03-06T21:04:57.971Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.705429379Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"":"downstairs","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
172052024-03-06T21:04:57.971Z{"msg":"[2]R flush_numbers[0..12]: [274, 273, 277, 271, 267, 272, 276, 273, 267, 271, 273, 266]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.705469336Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"":"downstairs","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
172062024-03-06T21:04:57.971Z{"msg":"[2]R generation[0..12]: [158, 157, 159, 156, 153, 156, 158, 157, 153, 156, 157, 152]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.705484085Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"":"downstairs","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
172072024-03-06T21:04:57.971Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"Wait for a query_work_queue command to finish before sending IO
172082024-03-06T21:04:57.971Zname":"crucible","level":30,"time":"2024-03-06T21:04:56.705501854Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"":"downstairs","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
172092024-03-06T21:04:57.971Z{"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.705522332Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"":"downstairs","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
172102024-03-06T21:04:57.971Z{"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.705535991Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"":"downstairs","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
172112024-03-06T21:04:57.971Z{"msg":"Next flush: 278","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.70555579Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"":"downstairs","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
172122024-03-06T21:04:57.971Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.705569089Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"":"downstairs","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
172132024-03-06T21:04:57.971Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.705589188Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
172142024-03-06T21:04:57.971Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.705601787Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"client":"0","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
172152024-03-06T21:04:57.971Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.705622585Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"client":"1","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
172162024-03-06T21:04:57.971Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.705635925Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"client":"2","":"downstairs","session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
172172024-03-06T21:04:57.971Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.705655913Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
172182024-03-06T21:04:57.971Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.705668362Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
172192024-03-06T21:04:57.971Z{"msg":"fa2525c7-44c2-4e3b-96df-0b0a8ca1c5f6 is now active with session: 1b43be15-43e3-4e8c-8f9b-76f7e75b6807","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.705687911Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
172202024-03-06T21:04:57.971Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.70570011Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779,"session_id":"1b43be15-43e3-4e8c-8f9b-76f7e75b6807"}
172212024-03-06T21:04:57.971Z{"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:04:56.705712829Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779}
172222024-03-06T21:04:57.972ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
172232024-03-06T21:04:57.972ZLoading write count information from file "/tmp/test_repair_verify.data"
172242024-03-06T21:04:57.972ZRun Repair workload
172252024-03-06T21:04:57.972Z01/30 Flush
172262024-03-06T21:04:57.972Z02/30 Write block 14 len 12288 data: 13 15 13
172272024-03-06T21:04:57.972Z03/30 Write block 195 len 16384 data: 13 15 14 16
172282024-03-06T21:04:57.972Z04/30 Write block 14 len 32768 data: 14 16 14 15 14 16 17 15
172292024-03-06T21:04:57.972Z05/30 Write block 435 len 4096 data: 14
172302024-03-06T21:04:57.972Z06/30 Read block 84 len 8192
172312024-03-06T21:04:57.976Z07/30 Write block 266 len 20480 data: 11 11 12 12 11
172322024-03-06T21:04:57.976Z08/30 Write block 381 len 32768 data: 16 17 16 17 17 16 13 14
172332024-03-06T21:04:57.976Z09/30 Write block 349 len 24576 data: 13 13 13 12 12 13
172342024-03-06T21:04:57.976Z10/30 Write block 350 len 16384 data: 14 14 13 13
172352024-03-06T21:04:57.976Z11/30 Write block 493 len 32768 data: 13 11 14 14 14 11 10 10
172362024-03-06T21:04:57.976Z12/30 Read block 440 len 40960
172372024-03-06T21:04:57.979Z13/30 Flush
172382024-03-06T21:04:58.047Z14/30 Write block 255 len 12288 data: 7 7 7
172392024-03-06T21:04:58.047Z15/30 Write block 475 len 20480 data: 7 8 8 10 5
172402024-03-06T21:04:58.048Z16/30 Read block 595 len 4096
172412024-03-06T21:04:58.048Z17/30 Read block 97 len 16384
172422024-03-06T21:04:58.048Z18/30 Read block 28 len 20480
172432024-03-06T21:04:58.048Z19/30 Write block 541 len 32768 data: 10 13 15 16 17 16 14 12
172442024-03-06T21:04:58.048Z20/30 Read block 508 len 36864
172452024-03-06T21:04:58.048Z21/30 Read block 185 len 32768
172462024-03-06T21:04:58.048Z22/30 Write block 508 len 36864 data: 11 12 14 14 14 13 15 17 16
172472024-03-06T21:04:58.048Z23/30 Read block 403 len 12288
172482024-03-06T21:04:58.048Z24/30 Read block 15 len 12288
172492024-03-06T21:04:58.048Z25/30 Read block 220 len 16384
172502024-03-06T21:04:58.048Z26/30 Read block 492 len 40960
172512024-03-06T21:04:58.048Z27/30 Read block 584 len 32768
172522024-03-06T21:04:58.048Z28/30 Write block 440 len 24576 data: 17 15 14 12 12 10
172532024-03-06T21:04:58.048Z29/30 Read block 7 len 36864
172542024-03-06T21:04:58.048Z30/30 Write block 21 len 8192 data: 16 18
172552024-03-06T21:04:58.048Z----------------------------------------------------------------
172562024-03-06T21:04:58.048Z Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30
172572024-03-06T21:04:58.048ZStates: Active Active Active
172582024-03-06T21:04:58.048ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
172592024-03-06T21:04:58.048Z 1 Acked 1000 Flush 0 Done Sent Done false
172602024-03-06T21:04:58.048Z 2 Acked 1001 Write 3 Done Sent Done false
172612024-03-06T21:04:58.048Z 3 Acked 1002 Write 4 Done Sent Done false
172622024-03-06T21:04:58.049Z 4 Acked 1003 Write 8 Done Sent Done false
172632024-03-06T21:04:58.049Z 5 Acked 1004 Write 1 Done Sent Done false
172642024-03-06T21:04:58.049Z 6 Acked 1005 Read 2 Done Sent Done false
172652024-03-06T21:04:58.049Z 7 Acked 1006 Write 5 Done Sent Done false
172662024-03-06T21:04:58.049Z 8 Acked 1007 Write 8 Done Sent Done false
172672024-03-06T21:04:58.049Z 9 Acked 1008 Write 6 Done Sent Done false
172682024-03-06T21:04:58.049Z 10 Acked 1009 Write 4 Done Sent Done false
172692024-03-06T21:04:58.049Z 11 Acked 1010 Write 8 Done Sent Done false
172702024-03-06T21:04:58.049Z 12 Acked 1011 Read 10 Done Sent Done false
172712024-03-06T21:04:58.049Z 13 Acked 1012 Flush 0 Done Sent Done false
172722024-03-06T21:04:58.049Z 14 Acked 1013 Write 3 Done Sent Done false
172732024-03-06T21:04:58.049Z 15 Acked 1014 Write 5 Done Sent Done false
172742024-03-06T21:04:58.049Z 16 Acked 1015 Read 1 Done Sent Done false
172752024-03-06T21:04:58.049Z 17 Acked 1016 Read 4 Done Sent Done false
172762024-03-06T21:04:58.049Z 18 Acked 1017 Read 5 Done Sent Done false
172772024-03-06T21:04:58.049Z 19 Acked 1018 Write 8 Done Sent Done false
172782024-03-06T21:04:58.049Z 20 Acked 1019 Read 9 Done Sent Done false
172792024-03-06T21:04:58.049Z 21 Acked 1020 Read 8 Done Sent Done false
172802024-03-06T21:04:58.049Z 22 Acked 1021 Write 9 Done Sent Done false
172812024-03-06T21:04:58.049Z 23 Acked 1022 Read 3 Done Sent Done false
172822024-03-06T21:04:58.049Z 24 Acked 1023 Read 3 Done Sent Done false
172832024-03-06T21:04:58.049Z 25 Acked 1024 Read 4 Done Sent Done false
172842024-03-06T21:04:58.050Z 26 Acked 1025 Read 10 Done Sent Done false
172852024-03-06T21:04:58.050Z 27 Acked 1026 Read 8 Done Sent Done false
172862024-03-06T21:04:58.050Z 28 Acked 1027 Write 6 Done Sent Done false
172872024-03-06T21:04:58.050Z 29 Acked 1028 Read 9 Done Sent Done false
172882024-03-06T21:04:58.050Z 30 Acked 1029 Write 2 Sent Sent Sent false
172892024-03-06T21:04:58.050Z STATES DS:0 DS:1 DS:2 TOTAL
172902024-03-06T21:04:58.050Z New 0 0 0 0
172912024-03-06T21:04:58.050Z Sent 1 30 1 32
172922024-03-06T21:04:58.050Z Done 29 0 29 58
172932024-03-06T21:04:58.050Z Skipped 0 0 0 0
172942024-03-06T21:04:58.050Z Error 0 0 0 0
172952024-03-06T21:04:58.050ZLast Flush: 1012 0 1012
172962024-03-06T21:04:58.050ZDownstairs last five completed:
172972024-03-06T21:04:58.050ZUpstairs last five completed: 30 29 28 27 26
172982024-03-06T21:04:58.050Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:04:56.740577047Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1779}
172992024-03-06T21:04:58.051ZWrote out file "/tmp/test_repair_verify.data"
173002024-03-06T21:04:58.051Z
173012024-03-06T21:04:58.051ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
173022024-03-06T21:04:58.051ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
173032024-03-06T21:04:58.051Z 0 000-019 160 158 160 279 274 279 F F F <---
173042024-03-06T21:04:58.051Z 1 020-039 160 157 160 279 273 279 T F T <---
173052024-03-06T21:04:58.051Z 2 040-059 159 159 159 277 277 277 F F F
173062024-03-06T21:04:58.051Z 3 060-079 156 156 156 271 271 271 F F F
173072024-03-06T21:04:58.051Z 4 080-099 153 153 153 267 267 267 F F F
173082024-03-06T21:04:58.051Z 5 100-119 156 156 156 272 272 272 F F F
173092024-03-06T21:04:58.051Z 6 120-139 158 158 158 276 276 276 F F F
173102024-03-06T21:04:58.051Z 7 140-159 157 157 157 273 273 273 F F F
173112024-03-06T21:04:58.051Z 8 160-179 153 153 153 267 267 267 F F F
173122024-03-06T21:04:58.051Z 9 180-199 160 156 160 279 271 279 F F F <---
173132024-03-06T21:04:58.051Z 10 200-219 157 157 157 273 273 273 F F F
173142024-03-06T21:04:58.051Z 11 220-239 152 152 152 266 266 266 F F F
173152024-03-06T21:04:58.051Z 12 240-259 147 147 147 260 260 260 T F T <---
173162024-03-06T21:04:58.051Z 13 260-279 160 158 160 279 276 279 F F F <---
173172024-03-06T21:04:58.051Z 14 280-299 158 158 158 276 276 276 F F F
173182024-03-06T21:04:58.051Z 15 300-319 156 156 156 272 272 272 F F F
173192024-03-06T21:04:58.051Z 16 320-339 154 154 154 268 268 268 F F F
173202024-03-06T21:04:58.051Z 17 340-359 160 158 160 279 276 279 F F F <---
173212024-03-06T21:04:58.051Z 18 360-379 158 158 158 276 276 276 F F F
173222024-03-06T21:04:58.051Z 19 380-399 160 158 160 279 276 279 F F F <---
173232024-03-06T21:04:58.052Z 20 400-419 156 156 156 272 272 272 F F F
173242024-03-06T21:04:58.052Z 21 420-439 160 159 160 279 277 279 F F F <---
173252024-03-06T21:04:58.052Z 22 440-459 159 159 159 277 277 277 T F T <---
173262024-03-06T21:04:58.052Z 23 460-479 158 158 158 274 274 274 T F T <---
173272024-03-06T21:04:58.052Z 24 480-499 160 153 160 279 267 279 F F F <---
173282024-03-06T21:04:58.052Z 25 500-519 160 154 160 279 268 279 T F T <---
173292024-03-06T21:04:58.052Z 26 520-539 159 159 159 277 277 277 F F F
173302024-03-06T21:04:58.052Z 27 540-559 151 151 151 265 265 265 T F T <---
173312024-03-06T21:04:58.052Z 28 560-579 157 157 157 273 273 273 F F F
173322024-03-06T21:04:58.052Z 29 580-599 157 157 157 273 273 273 F F F
173332024-03-06T21:04:58.052ZMax gen: 160, Max flush: 279
173342024-03-06T21:04:58.052ZError: Difference in extent metadata found!
173352024-03-06T21:04:58.052ZA Difference in extent metadata is expected here
173362024-03-06T21:04:58.052ZOn loop 80
173372024-03-06T21:04:58.052Z
173382024-03-06T21:04:58.057ZVerifying data now
173392024-03-06T21:05:02.724ZNo change after verify
173402024-03-06T21:05:02.724ZLoop: 80 Downstairs dump after verify (and repair):
173412024-03-06T21:05:02.735ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
173422024-03-06T21:05:02.735Z 0 000-019 160 160 160 279 279 279 F F F
173432024-03-06T21:05:02.735Z 1 020-039 161 161 161 280 280 280 F F F
173442024-03-06T21:05:02.735Z 2 040-059 159 159 159 277 277 277 F F F
173452024-03-06T21:05:02.735Z 3 060-079 156 156 156 271 271 271 F F F
173462024-03-06T21:05:02.735Z 4 080-099 153 153 153 267 267 267 F F F
173472024-03-06T21:05:02.735Z 5 100-119 156 156 156 272 272 272 F F F
173482024-03-06T21:05:02.735Z 6 120-139 158 158 158 276 276 276 F F F
173492024-03-06T21:05:02.735Z 7 140-159 157 157 157 273 273 273 F F F
173502024-03-06T21:05:02.735Z 8 160-179 153 153 153 267 267 267 F F F
173512024-03-06T21:05:02.735Z 9 180-199 160 160 160 279 279 279 F F F
173522024-03-06T21:05:02.735Z 10 200-219 157 157 157 273 273 273 F F F
173532024-03-06T21:05:02.736Z 11 220-239 152 152 152 266 266 266 F F F
173542024-03-06T21:05:02.736Z 12 240-259 161 161 161 280 280 280 F F F
173552024-03-06T21:05:02.736Z 13 260-279 160 160 160 279 279 279 F F F
173562024-03-06T21:05:02.736Z 14 280-299 158 158 158 276 276 276 F F F
173572024-03-06T21:05:02.736Z 15 300-319 156 156 156 272 272 272 F F F
173582024-03-06T21:05:02.736Z 16 320-339 154 154 154 268 268 268 F F F
173592024-03-06T21:05:02.736Z 17 340-359 160 160 160 279 279 279 F F F
173602024-03-06T21:05:02.736Z 18 360-379 158 158 158 276 276 276 F F F
173612024-03-06T21:05:02.736Z 19 380-399 160 160 160 279 279 279 F F F
173622024-03-06T21:05:02.736Z 20 400-419 156 156 156 272 272 272 F F F
173632024-03-06T21:05:02.741Z 21 420-439 160 160 160 279 279 279 F F F
173642024-03-06T21:05:02.741Z 22 440-459 161 161 161 280 280 280 F F F
173652024-03-06T21:05:02.741Z 23 460-479 161 161 161 280 280 280 F F F
173662024-03-06T21:05:02.741Z 24 480-499 160 160 160 279 279 279 F F F
173672024-03-06T21:05:02.741Z 25 500-519 161 161 161 280 280 280 F F F
173682024-03-06T21:05:02.741Z 26 520-539 159 159 159 277 277 277 F F F
173692024-03-06T21:05:02.741Z 27 540-559 161 161 161 280 280 280 F F F
173702024-03-06T21:05:02.741Z 28 560-579 157 157 157 273 273 273 F F F
173712024-03-06T21:05:02.742Z 29 580-599 157 157 157 273 273 273 F F F
173722024-03-06T21:05:02.742ZMax gen: 161, Max flush: 280
173732024-03-06T21:05:02.742Z
173742024-03-06T21:05:02.742ZBegin loop 81
173752024-03-06T21:05:02.742ZDownstairs to restart: 1
173762024-03-06T21:05:02.778Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.501729773Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787}
173772024-03-06T21:05:02.778Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
173782024-03-06T21:05:02.779Z,"time":"2024-03-06T21:05:01.502114386Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787}
173792024-03-06T21:05:02.779Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.502140345Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787}
173802024-03-06T21:05:02.779Z{"msg":"Crucible 78a17a90-7eda-45f6-8fd5-f01a477e829d has session id: c79949fe-1ea3-445d-ba6d-fe137a313f81","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.502164103Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
173812024-03-06T21:05:02.790Z{"msg":"Upstairs opts: Upstairs UUID: 78a17a90-7eda-45f6-8fd5-f01a477e829d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.502228998Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
173822024-03-06T21:05:02.790Z{"msg":"Crucible stats registered with UUID: 78a17a90-7eda-45f6-8fd5-f01a477e829d","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.502269836Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
173832024-03-06T21:05:02.790Z{"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.502288214Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787}
173842024-03-06T21:05:02.790Z{"msg":"78a17a90-7eda-45f6-8fd5-f01a477e829d active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.502326592Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
173852024-03-06T21:05:02.790Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.50235789Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"client":"0","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
173862024-03-06T21:05:02.790Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.502384418Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"client":"0","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
173872024-03-06T21:05:02.790Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.502418065Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"client":"1","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
173882024-03-06T21:05:02.790Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.502451253Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"client":"1","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
173892024-03-06T21:05:02.790Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.502476641Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"client":"2","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
173902024-03-06T21:05:02.790Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.502509129Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"client":"2","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
173912024-03-06T21:05:02.790Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.502533058Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"":"io task","client":"2","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
173922024-03-06T21:05:02.790Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.502564415Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"":"io task","client":"0","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
173932024-03-06T21:05:02.791Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.502600863Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
173942024-03-06T21:05:02.791Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.502750393Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"":"io task","client":"0","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
173952024-03-06T21:05:02.791Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.502803249Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"":"io task","client":"2","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
173962024-03-06T21:05:02.791Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.502829977Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
173972024-03-06T21:05:02.791Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.503607243Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"client":"2","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
173982024-03-06T21:05:02.791Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.503648231Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"client":"2","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
173992024-03-06T21:05:02.791Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.503676799Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"client":"0","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174002024-03-06T21:05:02.791Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.503721536Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"client":"0","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174012024-03-06T21:05:02.791Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.504209592Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"client":"0","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174022024-03-06T21:05:02.792Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.504352502Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"client":"2","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174032024-03-06T21:05:02.792Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.504398009Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"client":"1","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174042024-03-06T21:05:02.792Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.504423157Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"client":"1","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174052024-03-06T21:05:02.792Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.504725057Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"client":"0","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174062024-03-06T21:05:02.792Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.504771263Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"client":"0","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174072024-03-06T21:05:02.792Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.504798121Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174082024-03-06T21:05:02.792Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.504980629Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"client":"2","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174092024-03-06T21:05:02.792Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.505013977Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"client":"2","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174102024-03-06T21:05:02.792Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.505038105Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174112024-03-06T21:05:02.792Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.505067193Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"client":"1","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174122024-03-06T21:05:02.792Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.505474775Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"client":"1","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174132024-03-06T21:05:02.792Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.505510952Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"client":"1","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174142024-03-06T21:05:02.792Z{"msg":"[0]R flush_numbers[0..12]: [279, 280, 277, 271, 267, 272, 276, 273, 267, 279, 273, 266]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.50553823Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"":"downstairs","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174152024-03-06T21:05:02.792Z{"msg":"[0]R generation[0..12]: [160, 161, 159, 156, 153, 156, 158, 157, 153, 160, 157, 152]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.505569578Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"":"downstairs","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174162024-03-06T21:05:02.792Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.505595657Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"":"downstairs","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174172024-03-06T21:05:02.793Z{"msg":"[1]R flush_numbers[0..12]: [279, 280, 277, 271, 267, 272, 276, 273, 267, 279, 273, 266]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.505628464Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"":"downstairs","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174182024-03-06T21:05:02.793Z{"msg":"[1]R generation[0..12]: [160, 161, 159, 156, 153, 156, 158, 157, 153, 160, 157, 152]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.505653202Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"":"downstairs","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174192024-03-06T21:05:02.793ZWait for a query_work_queue command to finish before sending IO
174202024-03-06T21:05:02.793Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.50568541Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"":"downstairs","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174212024-03-06T21:05:02.793Z{"msg":"[2]R flush_numbers[0..12]: [279, 280, 277, 271, 267, 272, 276, 273, 267, 279, 273, 266]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.505712018Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"":"downstairs","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174222024-03-06T21:05:02.793Z{"msg":"[2]R generation[0..12]: [160, 161, 159, 156, 153, 156, 158, 157, 153, 160, 157, 152]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.505743916Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"":"downstairs","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174232024-03-06T21:05:02.793Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.505782884Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"":"downstairs","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174242024-03-06T21:05:02.793Z{"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.505804762Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"":"downstairs","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174252024-03-06T21:05:02.793Z{"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.505851219Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"":"downstairs","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174262024-03-06T21:05:02.793Z{"msg":"Next flush: 281","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.505888896Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"":"downstairs","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174272024-03-06T21:05:02.793Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.505923294Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"":"downstairs","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174282024-03-06T21:05:02.793Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
174292024-03-06T21:05:02.793Z,"time":"2024-03-06T21:05:01.505959201Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174302024-03-06T21:05:02.793Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.50598488Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"client":"0","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174312024-03-06T21:05:02.793Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.506014318Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"client":"1","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174322024-03-06T21:05:02.793Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.506029457Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"client":"2","":"downstairs","session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174332024-03-06T21:05:02.793Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.506050005Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174342024-03-06T21:05:02.793Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.506063894Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174352024-03-06T21:05:02.793Z{"msg":"78a17a90-7eda-45f6-8fd5-f01a477e829d is now active with session: c79949fe-1ea3-445d-ba6d-fe137a313f81","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.506083273Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174362024-03-06T21:05:02.793Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.506095552Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787,"session_id":"c79949fe-1ea3-445d-ba6d-fe137a313f81"}
174372024-03-06T21:05:02.793Z{"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:01.506115321Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787}
174382024-03-06T21:05:02.793ZLoading write count information from file "/tmp/test_repair_verify.data"
174392024-03-06T21:05:02.793ZRun Repair workload
174402024-03-06T21:05:02.793Z01/30 Write block 237 len 12288 data: 18 21 20
174412024-03-06T21:05:02.794Z02/30 Write block 465 len 28672 data: 18 14 14 13 11 8 7
174422024-03-06T21:05:02.794Z03/30 Read block 234 len 24576
174432024-03-06T21:05:02.794Z04/30 Read block 526 len 20480
174442024-03-06T21:05:02.794Z05/30 Flush
174452024-03-06T21:05:02.794Z06/30 Write block 109 len 28672 data: 10 9 10 8 5 8 8
174462024-03-06T21:05:02.794Z07/30 Read block 283 len 28672
174472024-03-06T21:05:02.794Z08/30 Write block 340 len 16384 data: 15 14 13 15
174482024-03-06T21:05:02.794Z09/30 Flush
174492024-03-06T21:05:02.794Z10/30 Write block 358 len 36864 data: 16 14 14 18 14 17 14 12 10
174502024-03-06T21:05:02.794Z11/30 Read block 420 len 20480
174512024-03-06T21:05:02.794Z12/30 Read block 28 len 12288
174522024-03-06T21:05:02.794Z13/30 Read block 510 len 12288
174532024-03-06T21:05:02.794Z14/30 Write block 189 len 16384 data: 15 17 13 15
174542024-03-06T21:05:02.794Z15/30 Read block 532 len 36864
174552024-03-06T21:05:02.811Z16/30 Read block 156 len 4096
174562024-03-06T21:05:02.811Z17/30 Write block 14 len 20480 data: 15 17 15 16 15
174572024-03-06T21:05:02.812Z18/30 Flush
174582024-03-06T21:05:02.812Z19/30 Write block 420 len 8192 data: 12 16
174592024-03-06T21:05:02.812Z20/30 Write block 329 len 36864 data: 12 10 8 8 10 8 7 10 11
174602024-03-06T21:05:02.812Z21/30 Flush
174612024-03-06T21:05:02.812Z22/30 Write block 403 len 4096 data: 9
174622024-03-06T21:05:02.812Z23/30 Read block 429 len 12288
174632024-03-06T21:05:02.812Z24/30 Read block 112 len 12288
174642024-03-06T21:05:02.812Z25/30 Read block 500 len 12288
174652024-03-06T21:05:02.812Z26/30 Write block 413 len 28672 data: 12 11 10 13 13 13 11
174662024-03-06T21:05:02.812Z27/30 Read block 38 len 40960
174672024-03-06T21:05:02.812Z28/30 Read block 507 len 12288
174682024-03-06T21:05:02.812Z29/30 Read block 563 len 36864
174692024-03-06T21:05:02.822Z30/30 Write block 33 len 16384 data: 10 10 10 6
174702024-03-06T21:05:02.822Z----------------------------------------------------------------
174712024-03-06T21:05:02.822Z Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:30
174722024-03-06T21:05:02.823ZStates: Active Active Active
174732024-03-06T21:05:02.823ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
174742024-03-06T21:05:02.823Z 1 Acked 1000 Write 3 Done Done Done false
174752024-03-06T21:05:02.823Z 2 Acked 1001 Write 7 Done Sent Done false
174762024-03-06T21:05:02.823Z 3 Acked 1002 Read 6 Done Sent Done false
174772024-03-06T21:05:02.823Z 4 Acked 1003 Read 5 Done Sent Done false
174782024-03-06T21:05:02.823Z 5 Acked 1004 Flush 0 Done Sent Done false
174792024-03-06T21:05:02.823Z 6 Acked 1005 Write 7 Done Sent Done false
174802024-03-06T21:05:02.823Z 7 Acked 1006 Read 7 Done Sent Done false
174812024-03-06T21:05:02.823Z 8 Acked 1007 Write 4 Done Sent Done false
174822024-03-06T21:05:02.823Z 9 Acked 1008 Flush 0 Done Sent Done false
174832024-03-06T21:05:02.823Z 10 Acked 1009 Write 9 Done Sent Done false
174842024-03-06T21:05:02.823Z 11 Acked 1010 Read 5 Done Sent Done false
174852024-03-06T21:05:02.823Z 12 Acked 1011 Read 3 Done Sent Done false
174862024-03-06T21:05:02.824Z 13 Acked 1012 Read 3 Done Sent Done false
174872024-03-06T21:05:02.824Z 14 Acked 1013 Write 4 Done Sent Done false
174882024-03-06T21:05:02.824Z 15 Acked 1014 Read 9 Done Sent Done false
174892024-03-06T21:05:02.824Z 16 Acked 1015 Read 1 Done Sent Done false
174902024-03-06T21:05:02.824Z 17 Acked 1016 Write 5 Done Sent Done false
174912024-03-06T21:05:02.824Z 18 Acked 1017 Flush 0 Done Sent Done false
174922024-03-06T21:05:02.824Z 19 Acked 1018 Write 2 Done Sent Done false
174932024-03-06T21:05:02.824Z 20 Acked 1019 Write 9 Done Sent Done false
174942024-03-06T21:05:02.824Z 21 Acked 1020 Flush 0 Done Sent Done false
174952024-03-06T21:05:02.825Z 22 Acked 1021 Write 1 Done Sent Done false
174962024-03-06T21:05:02.825Z 23 Acked 1022 Read 3 Done Sent Done false
174972024-03-06T21:05:02.825Z 24 Acked 1023 Read 3 Done Sent Done false
174982024-03-06T21:05:02.825Z 25 Acked 1024 Read 3 Done Sent Done false
174992024-03-06T21:05:02.825Z 26 Acked 1025 Write 7 Done Sent Done false
175002024-03-06T21:05:02.825Z 27 Acked 1026 Read 10 Done Sent Done false
175012024-03-06T21:05:02.825Z 28 Acked 1027 Read 3 Done Sent Done false
175022024-03-06T21:05:02.828Z 29 Acked 1028 Read 9 Done Sent Done false
175032024-03-06T21:05:02.828Z 30 Acked 1029 Write 4 Sent Sent Sent false
175042024-03-06T21:05:02.828Z STATES DS:0 DS:1 DS:2 TOTAL
175052024-03-06T21:05:02.828Z New 0 0 0 0
175062024-03-06T21:05:02.828Z Sent 1 29 1 31
175072024-03-06T21:05:02.828Z Done 29 1 29 59
175082024-03-06T21:05:02.828Z Skipped 0 0 0 0
175092024-03-06T21:05:02.828Z Error 0 0 0 0
175102024-03-06T21:05:02.828ZLast Flush: 1020 0 1020
175112024-03-06T21:05:02.828ZDownstairs last five completed:
175122024-03-06T21:05:02.828ZUpstairs last five completed: 30 29 28 27 26
175132024-03-06T21:05:02.828Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:05:01.542132139Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1787}
175142024-03-06T21:05:02.829ZWrote out file "/tmp/test_repair_verify.data"
175152024-03-06T21:05:02.829Z
175162024-03-06T21:05:02.829ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
175172024-03-06T21:05:02.843ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
175182024-03-06T21:05:02.843Z 0 000-019 162 160 162 283 279 283 F F F <---
175192024-03-06T21:05:02.843Z 1 020-039 161 161 161 280 280 280 T F T <---
175202024-03-06T21:05:02.844Z 2 040-059 159 159 159 277 277 277 F F F
175212024-03-06T21:05:02.844Z 3 060-079 156 156 156 271 271 271 F F F
175222024-03-06T21:05:02.844Z 4 080-099 153 153 153 267 267 267 F F F
175232024-03-06T21:05:02.844Z 5 100-119 162 156 162 282 272 282 F F F <---
175242024-03-06T21:05:02.844Z 6 120-139 158 158 158 276 276 276 F F F
175252024-03-06T21:05:02.844Z 7 140-159 157 157 157 273 273 273 F F F
175262024-03-06T21:05:02.844Z 8 160-179 153 153 153 267 267 267 F F F
175272024-03-06T21:05:02.844Z 9 180-199 162 160 162 283 279 283 F F F <---
175282024-03-06T21:05:02.844Z 10 200-219 157 157 157 273 273 273 F F F
175292024-03-06T21:05:02.844Z 11 220-239 162 152 162 281 266 281 F T F <---
175302024-03-06T21:05:02.845Z 12 240-259 161 161 161 280 280 280 F F F
175312024-03-06T21:05:02.845Z 13 260-279 160 160 160 279 279 279 F F F
175322024-03-06T21:05:02.845Z 14 280-299 158 158 158 276 276 276 F F F
175332024-03-06T21:05:02.845Z 15 300-319 156 156 156 272 272 272 F F F
175342024-03-06T21:05:02.845Z 16 320-339 162 154 162 284 268 284 F F F <---
175352024-03-06T21:05:02.845Z 17 340-359 162 160 162 283 279 283 F F F <---
175362024-03-06T21:05:02.845Z 18 360-379 162 158 162 283 276 283 F F F <---
175372024-03-06T21:05:02.845Z 19 380-399 160 160 160 279 279 279 F F F
175382024-03-06T21:05:02.845Z 20 400-419 156 156 156 272 272 272 T F T <---
175392024-03-06T21:05:02.845Z 21 420-439 162 160 162 284 279 284 F F F <---
175402024-03-06T21:05:02.846Z 22 440-459 161 161 161 280 280 280 F F F
175412024-03-06T21:05:02.846Z 23 460-479 162 161 162 281 280 281 F F F <---
175422024-03-06T21:05:02.846Z 24 480-499 160 160 160 279 279 279 F F F
175432024-03-06T21:05:02.846Z 25 500-519 161 161 161 280 280 280 F F F
175442024-03-06T21:05:02.846Z 26 520-539 159 159 159 277 277 277 F F F
175452024-03-06T21:05:02.846Z 27 540-559 161 161 161 280 280 280 F F F
175462024-03-06T21:05:02.846Z 28 560-579 157 157 157 273 273 273 F F F
175472024-03-06T21:05:02.846Z 29 580-599 157 157 157 273 273 273 F F F
175482024-03-06T21:05:02.846ZMax gen: 162, Max flush: 284
175492024-03-06T21:05:02.846ZError: Difference in extent metadata found!
175502024-03-06T21:05:02.846ZA Difference in extent metadata is expected here
175512024-03-06T21:05:02.846ZOn loop 81
175522024-03-06T21:05:02.847Z
175532024-03-06T21:05:02.847ZVerifying data now
175542024-03-06T21:05:07.426ZNo change after verify
175552024-03-06T21:05:07.426ZLoop: 81 Downstairs dump after verify (and repair):
175562024-03-06T21:05:07.448ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
175572024-03-06T21:05:07.448Z 0 000-019 162 162 162 283 283 283 F F F
175582024-03-06T21:05:07.448Z 1 020-039 163 163 163 285 285 285 F F F
175592024-03-06T21:05:07.448Z 2 040-059 159 159 159 277 277 277 F F F
175602024-03-06T21:05:07.448Z 3 060-079 156 156 156 271 271 271 F F F
175612024-03-06T21:05:07.448Z 4 080-099 153 153 153 267 267 267 F F F
175622024-03-06T21:05:07.448Z 5 100-119 162 162 162 282 282 282 F F F
175632024-03-06T21:05:07.448Z 6 120-139 158 158 158 276 276 276 F F F
175642024-03-06T21:05:07.448Z 7 140-159 157 157 157 273 273 273 F F F
175652024-03-06T21:05:07.448Z 8 160-179 153 153 153 267 267 267 F F F
175662024-03-06T21:05:07.448Z 9 180-199 162 162 162 283 283 283 F F F
175672024-03-06T21:05:07.448Z 10 200-219 157 157 157 273 273 273 F F F
175682024-03-06T21:05:07.448Z 11 220-239 162 162 162 281 281 281 F F F
175692024-03-06T21:05:07.449Z 12 240-259 161 161 161 280 280 280 F F F
175702024-03-06T21:05:07.449Z 13 260-279 160 160 160 279 279 279 F F F
175712024-03-06T21:05:07.449Z 14 280-299 158 158 158 276 276 276 F F F
175722024-03-06T21:05:07.449Z 15 300-319 156 156 156 272 272 272 F F F
175732024-03-06T21:05:07.449Z 16 320-339 162 162 162 284 284 284 F F F
175742024-03-06T21:05:07.449Z 17 340-359 162 162 162 283 283 283 F F F
175752024-03-06T21:05:07.449Z 18 360-379 162 162 162 283 283 283 F F F
175762024-03-06T21:05:07.449Z 19 380-399 160 160 160 279 279 279 F F F
175772024-03-06T21:05:07.449Z 20 400-419 163 163 163 285 285 285 F F F
175782024-03-06T21:05:07.449Z 21 420-439 162 162 162 284 284 284 F F F
175792024-03-06T21:05:07.449Z 22 440-459 161 161 161 280 280 280 F F F
175802024-03-06T21:05:07.449Z 23 460-479 162 162 162 281 281 281 F F F
175812024-03-06T21:05:07.449Z 24 480-499 160 160 160 279 279 279 F F F
175822024-03-06T21:05:07.449Z 25 500-519 161 161 161 280 280 280 F F F
175832024-03-06T21:05:07.466Z 26 520-539 159 159 159 277 277 277 F F F
175842024-03-06T21:05:07.466Z 27 540-559 161 161 161 280 280 280 F F F
175852024-03-06T21:05:07.466Z 28 560-579 157 157 157 273 273 273 F F F
175862024-03-06T21:05:07.466Z 29 580-599 157 157 157 273 273 273 F F F
175872024-03-06T21:05:07.466ZMax gen: 163, Max flush: 285
175882024-03-06T21:05:07.466Z
175892024-03-06T21:05:07.466ZBegin loop 82
175902024-03-06T21:05:07.466ZDownstairs to restart: 1
175912024-03-06T21:05:07.495Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.210547795Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795}
175922024-03-06T21:05:07.495Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.210789248Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795}
175932024-03-06T21:05:07.495Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.210807497Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795}
175942024-03-06T21:05:07.495Z{"msg":"Crucible 73ae34f1-5e22-4361-9b6e-a0078bacdaaf has session id: 5bbbb822-0006-42cf-bff0-c7a678b51602","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.210832245Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
175952024-03-06T21:05:07.495Z{"msg":"Upstairs opts: Upstairs UUID: 73ae34f1-5e22-4361-9b6e-a0078bacdaaf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
175962024-03-06T21:05:07.495Z,"time":"2024-03-06T21:05:06.210881062Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
175972024-03-06T21:05:07.495Z{"msg":"Crucible stats registered with UUID: 73ae34f1-5e22-4361-9b6e-a0078bacdaaf","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.211015323Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
175982024-03-06T21:05:07.495Z{"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.211044781Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795}
175992024-03-06T21:05:07.495Z{"msg":"73ae34f1-5e22-4361-9b6e-a0078bacdaaf active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.211183011Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176002024-03-06T21:05:07.496Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.211226178Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"client":"0","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176012024-03-06T21:05:07.496Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.211253506Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"client":"0","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176022024-03-06T21:05:07.496Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.211297733Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"client":"1","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176032024-03-06T21:05:07.496Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.211329731Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"client":"1","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176042024-03-06T21:05:07.496Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.211365539Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"client":"2","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176052024-03-06T21:05:07.496Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.211420745Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"client":"2","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176062024-03-06T21:05:07.496Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.21149011Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"":"io task","client":"2","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176072024-03-06T21:05:07.496Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.211578724Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"":"io task","client":"1","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176082024-03-06T21:05:07.496Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.21163445Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"":"io task","client":"0","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176092024-03-06T21:05:07.497Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.211713325Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"":"io task","client":"0","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176102024-03-06T21:05:07.497Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.211768801Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"":"io task","client":"1","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176112024-03-06T21:05:07.497Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.211811908Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"":"io task","client":"2","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176122024-03-06T21:05:07.497Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.212379379Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"client":"2","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176132024-03-06T21:05:07.497Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.212447844Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"client":"2","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176142024-03-06T21:05:07.497Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.212483672Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"client":"0","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176152024-03-06T21:05:07.497Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.212530168Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"client":"0","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176162024-03-06T21:05:07.497Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.213048973Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"client":"0","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176172024-03-06T21:05:07.517Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.21323341Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"client":"2","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176182024-03-06T21:05:07.518Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.21366869Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"client":"0","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176192024-03-06T21:05:07.518Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.213751314Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"client":"0","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176202024-03-06T21:05:07.518Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.21381385Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176212024-03-06T21:05:07.518Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.213855887Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"client":"1","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176222024-03-06T21:05:07.518Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.213894224Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"client":"1","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176232024-03-06T21:05:07.518Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.213928862Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"client":"2","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176242024-03-06T21:05:07.518Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.21395568Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"client":"2","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176252024-03-06T21:05:07.518Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.213979838Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176262024-03-06T21:05:07.518Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.214408979Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"client":"1","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176272024-03-06T21:05:07.518Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.214818611Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"client":"1","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176282024-03-06T21:05:07.518Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.214845469Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"client":"1","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176292024-03-06T21:05:07.518Z{"msg":"[0]R flush_numbers[0..12]: [283, 285, 277, 271, 267, 282, 276, 273, 267, 283, 273, 281]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.214860158Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"":"downstairs","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176302024-03-06T21:05:07.518Z{"msg":"[0]R generation[0..12]: [162, 163, 159, 156, 153, 162, 158, 157, 153, 162, 157, 162]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.214875147Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"":"downstairs","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176312024-03-06T21:05:07.519Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.214896515Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"":"downstairs","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176322024-03-06T21:05:07.519Z{"msg":"[1]R flush_numbers[0..12]: [283, 285, 277, 271, 267, 282, 276, 273, 267, 283, 273, 281]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.214911134Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"":"downstairs","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176332024-03-06T21:05:07.519Z{"msg":"[1]R generation[0..12]: [162, 163, 159, 156, 153, 162, 158, 157, 153, 162, 157, 162]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.214931423Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"":"downstairs","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176342024-03-06T21:05:07.519Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.214945872Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"":"downstairs","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176352024-03-06T21:05:07.519Z{"msg":"[2]R flush_numbers[0..12]: [283, 285, 277, 271, 267, 282, 276, 273, 267, 283, 273, 281]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.21496643Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"":"downstairs","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176362024-03-06T21:05:07.519Z{"msg":"[2]R generation[0..12]: [162, 163, 159, 156, 153, 162, 158, 157, 153, 162, 157, 162]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.214980829Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"":"downstairs","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176372024-03-06T21:05:07.519Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.215007898Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"":"downstairs","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176382024-03-06T21:05:07.519ZWait for a query_work_queue command to finish before sending IO
176392024-03-06T21:05:07.519Z{"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.215036936Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"":"downstairs","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176402024-03-06T21:05:07.519Z{"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.215053524Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"":"downstairs","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176412024-03-06T21:05:07.519Z{"msg":"Next flush: 286","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.215073563Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"":"downstairs","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176422024-03-06T21:05:07.519Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.215086892Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"":"downstairs","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176432024-03-06T21:05:07.519Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.215106741Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176442024-03-06T21:05:07.519Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.21511904Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"client":"0","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176452024-03-06T21:05:07.519Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.215138578Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"client":"1","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176462024-03-06T21:05:07.519Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.215151908Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"client":"2","":"downstairs","session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176472024-03-06T21:05:07.520Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.215171476Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176482024-03-06T21:05:07.520Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.215184066Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176492024-03-06T21:05:07.520Z{"msg":"73ae34f1-5e22-4361-9b6e-a0078bacdaaf is now active with session: 5bbbb822-0006-42cf-bff0-c7a678b51602","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.215196994Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176502024-03-06T21:05:07.520Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.215215383Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795,"session_id":"5bbbb822-0006-42cf-bff0-c7a678b51602"}
176512024-03-06T21:05:07.520Z{"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:06.215227652Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795}
176522024-03-06T21:05:07.520ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
176532024-03-06T21:05:07.520ZLoading write count information from file "/tmp/test_repair_verify.data"
176542024-03-06T21:05:07.520ZRun Repair workload
176552024-03-06T21:05:07.520Z01/30 Write block 286 len 28672 data: 18 18 19 17 15 13 10
176562024-03-06T21:05:07.520Z02/30 Flush
176572024-03-06T21:05:07.520Z03/30 Write block 77 len 28672 data: 6 9 11 10 10 10 10
176582024-03-06T21:05:07.520Z04/30 Write block 578 len 8192 data: 8 10
176592024-03-06T21:05:07.520Z05/30 Write block 17 len 12288 data: 17 16 17
176602024-03-06T21:05:07.520Z06/30 Read block 262 len 8192
176612024-03-06T21:05:07.520Z07/30 Read block 87 len 32768
176622024-03-06T21:05:07.520Z08/30 Read block 316 len 40960
176632024-03-06T21:05:07.520Z09/30 Flush
176642024-03-06T21:05:07.520Z10/30 Write block 244 len 32768 data: 14 13 12 10 8 7 6 7
176652024-03-06T21:05:07.520Z11/30 Write block 450 len 8192 data: 12 14
176662024-03-06T21:05:07.520Z12/30 Read block 311 len 4096
176672024-03-06T21:05:07.520Z13/30 Write block 517 len 24576 data: 14 13 12 7 7 7
176682024-03-06T21:05:07.520Z14/30 Read block 209 len 24576
176692024-03-06T21:05:07.521Z15/30 Write block 103 len 20480 data: 12 10 8 13 14
176702024-03-06T21:05:07.521Z16/30 Read block 311 len 16384
176712024-03-06T21:05:07.521Z17/30 Read block 147 len 8192
176722024-03-06T21:05:07.521Z18/30 Read block 584 len 12288
176732024-03-06T21:05:07.521Z19/30 Write block 31 len 32768 data: 10 11 11 11 11 7 4 8
176742024-03-06T21:05:07.521Z20/30 Write block 301 len 24576 data: 9 13 13 15 16 18
176752024-03-06T21:05:07.521Z21/30 Flush
176762024-03-06T21:05:07.521Z22/30 Write block 163 len 40960 data: 11 11 11 9 10 12 11 11 11 13
176772024-03-06T21:05:07.521Z23/30 Write block 419 len 36864 data: 12 13 17 14 14 12 12 12 13
176782024-03-06T21:05:07.521Z24/30 Write block 110 len 40960 data: 10 11 9 6 9 9 8 11 11 12
176792024-03-06T21:05:07.521Z25/30 Flush
176802024-03-06T21:05:07.531Z26/30 Write block 8 len 36864 data: 12 13 13 16 13 14 16 18 16
176812024-03-06T21:05:07.531Z27/30 Read block 102 len 16384
176822024-03-06T21:05:07.531Z28/30 Write block 293 len 16384 data: 7 6 9 9
176832024-03-06T21:05:07.531Z29/30 Write block 108 len 4096 data: 10
176842024-03-06T21:05:07.531Z30/30 Read block 168 len 24576
176852024-03-06T21:05:07.531Z----------------------------------------------------------------
176862024-03-06T21:05:07.531Z Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:28
176872024-03-06T21:05:07.531ZStates: Active Active Active
176882024-03-06T21:05:07.531ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
176892024-03-06T21:05:07.531Z 3 Acked 1002 Write 7 Done Done Done false
176902024-03-06T21:05:07.531Z 4 Acked 1003 Write 2 Done Done Done false
176912024-03-06T21:05:07.531Z 5 Acked 1004 Write 3 Done Sent Done false
176922024-03-06T21:05:07.532Z 6 Acked 1005 Read 2 Done Sent Done false
176932024-03-06T21:05:07.532Z 7 Acked 1006 Read 8 Done Sent Done false
176942024-03-06T21:05:07.532Z 8 Acked 1007 Read 10 Done Sent Done false
176952024-03-06T21:05:07.532Z 9 Acked 1008 Flush 0 Done Sent Done false
176962024-03-06T21:05:07.532Z 10 Acked 1009 Write 8 Done Sent Done false
176972024-03-06T21:05:07.532Z 11 Acked 1010 Write 2 Done Sent Done false
176982024-03-06T21:05:07.532Z 12 Acked 1011 Read 1 Done Sent Done false
176992024-03-06T21:05:07.532Z 13 Acked 1012 Write 6 Done Sent Done false
177002024-03-06T21:05:07.532Z 14 Acked 1013 Read 6 Done Sent Done false
177012024-03-06T21:05:07.532Z 15 Acked 1014 Write 5 Done Sent Done false
177022024-03-06T21:05:07.532Z 16 Acked 1015 Read 4 Done Sent Done false
177032024-03-06T21:05:07.532Z 17 Acked 1016 Read 2 Done Sent Done false
177042024-03-06T21:05:07.532Z 18 Acked 1017 Read 3 Done Sent Done false
177052024-03-06T21:05:07.532Z 19 Acked 1018 Write 8 Done Sent Done false
177062024-03-06T21:05:07.533Z 20 Acked 1019 Write 6 Done Sent Done false
177072024-03-06T21:05:07.533Z 21 Acked 1020 Flush 0 Done Sent Done false
177082024-03-06T21:05:07.533Z 22 Acked 1021 Write 10 Done Sent Done false
177092024-03-06T21:05:07.533Z 23 Acked 1022 Write 9 Done Sent Done false
177102024-03-06T21:05:07.533Z 24 Acked 1023 Write 10 Done Sent Done false
177112024-03-06T21:05:07.533Z 25 Acked 1024 Flush 0 Done Sent Done false
177122024-03-06T21:05:07.533Z 26 Acked 1025 Write 9 Done Sent Done false
177132024-03-06T21:05:07.533Z 27 Acked 1026 Read 4 Done Sent Done false
177142024-03-06T21:05:07.534Z 28 Acked 1027 Write 4 Done Sent Done false
177152024-03-06T21:05:07.534Z 29 Acked 1028 Write 1 Done Sent Done false
177162024-03-06T21:05:07.534Z 30 Acked 1029 Read 6 Done Sent Done false
177172024-03-06T21:05:07.534Z STATES DS:0 DS:1 DS:2 TOTAL
177182024-03-06T21:05:07.534Z New 0 0 0 0
177192024-03-06T21:05:07.534Z Sent 0 26 0 26
177202024-03-06T21:05:07.534Z Done 28 2 28 58
177212024-03-06T21:05:07.534Z Skipped 0 0 0 0
177222024-03-06T21:05:07.534Z Error 0 0 0 0
177232024-03-06T21:05:07.534ZLast Flush: 1024 1001 1024
177242024-03-06T21:05:07.534ZDownstairs last five completed: 1001 1000
177252024-03-06T21:05:07.534ZUpstairs last five completed: 30 29 28 27 26
177262024-03-06T21:05:07.534Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:05:06.253100792Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1795}
177272024-03-06T21:05:07.534ZWrote out file "/tmp/test_repair_verify.data"
177282024-03-06T21:05:07.534Z
177292024-03-06T21:05:07.534ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
177302024-03-06T21:05:07.546ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
177312024-03-06T21:05:07.546Z 0 000-019 164 162 164 287 283 287 T F T <---
177322024-03-06T21:05:07.546Z 1 020-039 164 163 164 288 285 288 F F F <---
177332024-03-06T21:05:07.547Z 2 040-059 159 159 159 277 277 277 F F F
177342024-03-06T21:05:07.547Z 3 060-079 164 156 164 287 271 287 F T F <---
177352024-03-06T21:05:07.547Z 4 080-099 164 153 164 287 267 287 F T F <---
177362024-03-06T21:05:07.547Z 5 100-119 164 162 164 289 282 289 T F T <---
177372024-03-06T21:05:07.547Z 6 120-139 158 158 158 276 276 276 F F F
177382024-03-06T21:05:07.557Z 7 140-159 157 157 157 273 273 273 F F F
177392024-03-06T21:05:07.557Z 8 160-179 164 153 164 289 267 289 F F F <---
177402024-03-06T21:05:07.557Z 9 180-199 162 162 162 283 283 283 F F F
177412024-03-06T21:05:07.557Z 10 200-219 157 157 157 273 273 273 F F F
177422024-03-06T21:05:07.558Z 11 220-239 162 162 162 281 281 281 F F F
177432024-03-06T21:05:07.558Z 12 240-259 164 161 164 288 280 288 F F F <---
177442024-03-06T21:05:07.558Z 13 260-279 160 160 160 279 279 279 F F F
177452024-03-06T21:05:07.558Z 14 280-299 164 164 164 286 286 286 T F T <---
177462024-03-06T21:05:07.558Z 15 300-319 164 156 164 288 272 288 F F F <---
177472024-03-06T21:05:07.559Z 16 320-339 162 162 162 284 284 284 F F F
177482024-03-06T21:05:07.559Z 17 340-359 162 162 162 283 283 283 F F F
177492024-03-06T21:05:07.559Z 18 360-379 162 162 162 283 283 283 F F F
177502024-03-06T21:05:07.559Z 19 380-399 160 160 160 279 279 279 F F F
177512024-03-06T21:05:07.559Z 20 400-419 164 163 164 289 285 289 F F F <---
177522024-03-06T21:05:07.559Z 21 420-439 164 162 164 289 284 289 F F F <---
177532024-03-06T21:05:07.559Z 22 440-459 164 161 164 288 280 288 F F F <---
177542024-03-06T21:05:07.559Z 23 460-479 162 162 162 281 281 281 F F F
177552024-03-06T21:05:07.559Z 24 480-499 160 160 160 279 279 279 F F F
177562024-03-06T21:05:07.559Z 25 500-519 164 161 164 288 280 288 F F F <---
177572024-03-06T21:05:07.560Z 26 520-539 164 159 164 288 277 288 F F F <---
177582024-03-06T21:05:07.560Z 27 540-559 161 161 161 280 280 280 F F F
177592024-03-06T21:05:07.560Z 28 560-579 164 157 164 287 273 287 F T F <---
177602024-03-06T21:05:07.560Z 29 580-599 157 157 157 273 273 273 F F F
177612024-03-06T21:05:07.561ZMax gen: 164, Max flush: 289
177622024-03-06T21:05:07.561ZError: Difference in extent metadata found!
177632024-03-06T21:05:07.561ZA Difference in extent metadata is expected here
177642024-03-06T21:05:07.561ZOn loop 82
177652024-03-06T21:05:07.561Z
177662024-03-06T21:05:07.562ZVerifying data now
177672024-03-06T21:05:13.325ZNo change after verify
177682024-03-06T21:05:13.325ZLoop: 82 Downstairs dump after verify (and repair):
177692024-03-06T21:05:13.339ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
177702024-03-06T21:05:13.339Z 0 000-019 165 165 165 290 290 290 F F F
177712024-03-06T21:05:13.339Z 1 020-039 164 164 164 288 288 288 F F F
177722024-03-06T21:05:13.339Z 2 040-059 159 159 159 277 277 277 F F F
177732024-03-06T21:05:13.339Z 3 060-079 164 164 164 287 287 287 F F F
177742024-03-06T21:05:13.339Z 4 080-099 164 164 164 287 287 287 F F F
177752024-03-06T21:05:13.339Z 5 100-119 165 165 165 290 290 290 F F F
177762024-03-06T21:05:13.340Z 6 120-139 158 158 158 276 276 276 F F F
177772024-03-06T21:05:13.340Z 7 140-159 157 157 157 273 273 273 F F F
177782024-03-06T21:05:13.340Z 8 160-179 164 164 164 289 289 289 F F F
177792024-03-06T21:05:13.340Z 9 180-199 162 162 162 283 283 283 F F F
177802024-03-06T21:05:13.340Z 10 200-219 157 157 157 273 273 273 F F F
177812024-03-06T21:05:13.340Z 11 220-239 162 162 162 281 281 281 F F F
177822024-03-06T21:05:13.340Z 12 240-259 164 164 164 288 288 288 F F F
177832024-03-06T21:05:13.340Z 13 260-279 160 160 160 279 279 279 F F F
177842024-03-06T21:05:13.340Z 14 280-299 165 165 165 290 290 290 F F F
177852024-03-06T21:05:13.340Z 15 300-319 164 164 164 288 288 288 F F F
177862024-03-06T21:05:13.341Z 16 320-339 162 162 162 284 284 284 F F F
177872024-03-06T21:05:13.341Z 17 340-359 162 162 162 283 283 283 F F F
177882024-03-06T21:05:13.341Z 18 360-379 162 162 162 283 283 283 F F F
177892024-03-06T21:05:13.341Z 19 380-399 160 160 160 279 279 279 F F F
177902024-03-06T21:05:13.341Z 20 400-419 164 164 164 289 289 289 F F F
177912024-03-06T21:05:13.341Z 21 420-439 164 164 164 289 289 289 F F F
177922024-03-06T21:05:13.341Z 22 440-459 164 164 164 288 288 288 F F F
177932024-03-06T21:05:13.341Z 23 460-479 162 162 162 281 281 281 F F F
177942024-03-06T21:05:13.341Z 24 480-499 160 160 160 279 279 279 F F F
177952024-03-06T21:05:13.341Z 25 500-519 164 164 164 288 288 288 F F F
177962024-03-06T21:05:13.342Z 26 520-539 164 164 164 288 288 288 F F F
177972024-03-06T21:05:13.342Z 27 540-559 161 161 161 280 280 280 F F F
177982024-03-06T21:05:13.342Z 28 560-579 164 164 164 287 287 287 F F F
177992024-03-06T21:05:13.342Z 29 580-599 157 157 157 273 273 273 F F F
178002024-03-06T21:05:13.342ZMax gen: 165, Max flush: 290
178012024-03-06T21:05:13.352Z
178022024-03-06T21:05:13.352ZBegin loop 83
178032024-03-06T21:05:13.352ZDownstairs to restart: 2
178042024-03-06T21:05:13.369Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.0978554Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803}
178052024-03-06T21:05:13.370Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.098169299Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803}
178062024-03-06T21:05:13.378Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.098220835Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803}
178072024-03-06T21:05:13.379ZCrucible runtime is spawned
178082024-03-06T21:05:13.379Z{"msg":"Crucible 7bcf3d9d-ae43-4966-8b44-d5b2a47bd637 has session id: 3af37575-7367-4958-8735-f1df8e4004bb","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.098281381Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178092024-03-06T21:05:13.379Z{"msg":"Upstairs opts: Upstairs UUID: 7bcf3d9d-ae43-4966-8b44-d5b2a47bd637, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.098346117Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178102024-03-06T21:05:13.379Z{"msg":"Crucible stats registered with UUID: 7bcf3d9d-ae43-4966-8b44-d5b2a47bd637","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.098377224Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178112024-03-06T21:05:13.379Z{"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.098403073Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803}
178122024-03-06T21:05:13.379Z{"msg":"7bcf3d9d-ae43-4966-8b44-d5b2a47bd637 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.098497436Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178132024-03-06T21:05:13.379Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.098540903Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"client":"0","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178142024-03-06T21:05:13.379Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.098563372Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"client":"0","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178152024-03-06T21:05:13.379Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.098599509Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"client":"1","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178162024-03-06T21:05:13.379Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.098631387Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"client":"1","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178172024-03-06T21:05:13.380Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.098674054Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"client":"2","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178182024-03-06T21:05:13.380Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.098696153Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"client":"2","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178192024-03-06T21:05:13.380Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.09873192Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"":"io task","client":"0","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178202024-03-06T21:05:13.380Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.098769848Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"":"io task","client":"1","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178212024-03-06T21:05:13.380Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.098801865Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"":"io task","client":"2","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178222024-03-06T21:05:13.380Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.098841712Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"":"io task","client":"0","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178232024-03-06T21:05:13.380Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.098870581Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"":"io task","client":"2","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178242024-03-06T21:05:13.380Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.098904788Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"":"io task","client":"1","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178252024-03-06T21:05:13.381Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.099639917Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"client":"0","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178262024-03-06T21:05:13.381Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.099670455Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"client":"0","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178272024-03-06T21:05:13.381Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.099686534Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"client":"1","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178282024-03-06T21:05:13.381Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.099719462Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"client":"1","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178292024-03-06T21:05:13.381Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.100320491Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"client":"1","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178302024-03-06T21:05:13.381Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.100461431Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"client":"0","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178312024-03-06T21:05:13.381Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.100594632Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"client":"2","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178322024-03-06T21:05:13.381Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.10062682Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"client":"2","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178332024-03-06T21:05:13.381Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.100875322Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"client":"1","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178342024-03-06T21:05:13.381Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.100927429Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"client":"1","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178352024-03-06T21:05:13.381Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.100981825Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178362024-03-06T21:05:13.382Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.101018653Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"client":"0","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178372024-03-06T21:05:13.382Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.101041131Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"client":"0","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178382024-03-06T21:05:13.382Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.101069789Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178392024-03-06T21:05:13.382Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.101285224Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"client":"2","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178402024-03-06T21:05:13.382Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.101728573Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"client":"2","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178412024-03-06T21:05:13.382Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.10177984Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"client":"2","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178422024-03-06T21:05:13.382Z{"msg":"[0]R flush_numbers[0..12]: [290, 288, 277, 287, 287, 290, 276, 273, 289, 283, 273, 281]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.101799889Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"":"downstairs","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178432024-03-06T21:05:13.382Z{"msg":"[0]R generation[0..12]: [165, 164, 159, 164, 164, 165, 158, 157, 164, 162, 157, 162]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.101870444Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"":"downstairs","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178442024-03-06T21:05:13.382Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.101889962Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"":"downstairs","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178452024-03-06T21:05:13.382Z{"msg":"[1]R flush_numbers[0..12]: [290, 288, 277, 287, 287, 290, 276, 273, 289, 283, 273, 281]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.101912791Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"":"downstairs","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178462024-03-06T21:05:13.382Z{"msg":"[1]R generation[0..12]: [165, 164, 159, 164, 164, 165, 158, 157, 164, 162, 157, 162]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.10192761Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"":"downstairs","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178472024-03-06T21:05:13.382Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.101941949Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"":"downstairs","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178482024-03-06T21:05:13.382Z{"msg":"[2]R flush_numbers[0..12]: [290, 288, 277, 287, 287, 290, 276, 273, 289, 283, 273, 281]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.101964687Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"":"downstairs","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178492024-03-06T21:05:13.382Z{"msg":"[2]R generation[0..12]: [165, 164, 159, 164, 164, 165, 158, 157, 164, 162, 157, 162]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
178502024-03-06T21:05:13.382Z,"time":"2024-03-06T21:05:12.101979486Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"":"downstairs","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178512024-03-06T21:05:13.382Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.101995415Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"":"downstairs","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178522024-03-06T21:05:13.383Z{"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.102016364Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"":"downstairs","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178532024-03-06T21:05:13.383Z{"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.102031073Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"":"downstairs","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178542024-03-06T21:05:13.383Z{"msg":"Next flush: 291","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.102052981Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"":"downstairs","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178552024-03-06T21:05:13.383Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.10206721Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"":"downstairs","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178562024-03-06T21:05:13.383Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.102088389Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178572024-03-06T21:05:13.383Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.102100888Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"client":"0","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178582024-03-06T21:05:13.383Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.102122057Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"client":"1","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178592024-03-06T21:05:13.383Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.102135765Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"client":"2","":"downstairs","session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178602024-03-06T21:05:13.383Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.102155664Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178612024-03-06T21:05:13.383Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.102168283Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178622024-03-06T21:05:13.383Z{"msg":"7bcf3d9d-ae43-4966-8b44-d5b2a47bd637 is now active with session: 3af37575-7367-4958-8735-f1df8e4004bb","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.102187802Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178632024-03-06T21:05:13.383Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.102206631Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803,"session_id":"3af37575-7367-4958-8735-f1df8e4004bb"}
178642024-03-06T21:05:13.384Z{"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:12.1022208Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803}
178652024-03-06T21:05:13.384ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
178662024-03-06T21:05:13.384ZLoading write count information from file "/tmp/test_repair_verify.data"
178672024-03-06T21:05:13.384ZRun Repair workload
178682024-03-06T21:05:13.384Z01/30 Flush
178692024-03-06T21:05:13.384Z02/30 Write block 426 len 32768 data: 13 14 13 12 12 10 10 11
178702024-03-06T21:05:13.384Z03/30 Read block 475 len 32768
178712024-03-06T21:05:13.384Z04/30 Read block 197 len 4096
178722024-03-06T21:05:13.384Z05/30 Read block 140 len 4096
178732024-03-06T21:05:13.384Z06/30 Write block 182 len 36864 data: 16 14 14 13 11 10 10 16 18
178742024-03-06T21:05:13.384Z07/30 Write block 460 len 36864 data: 16 18 15 20 20 19 15 15 14
178752024-03-06T21:05:13.384Z08/30 Read block 147 len 24576
178762024-03-06T21:05:13.384Z09/30 Write block 544 len 24576 data: 17 18 17 15 13 10
178772024-03-06T21:05:13.384Z10/30 Write block 69 len 8192 data: 14 10
178782024-03-06T21:05:13.393Z11/30 Write block 279 len 36864 data: 12 11 12 14 12 16 17 19 19
178792024-03-06T21:05:13.394Z12/30 Read block 257 len 24576
178802024-03-06T21:05:13.394Z13/30 Write block 252 len 20480 data: 8 8 7 8 8
178812024-03-06T21:05:13.394Z14/30 Write block 387 len 20480 data: 14 15 14 16 11
178822024-03-06T21:05:13.394Z15/30 Write block 162 len 32768 data: 13 12 12 12 10 11 13 12
178832024-03-06T21:05:13.394Z16/30 Read block 215 len 8192
178842024-03-06T21:05:13.394Z17/30 Read block 352 len 28672
178852024-03-06T21:05:13.394Z18/30 Flush
178862024-03-06T21:05:13.394Z19/30 Write block 382 len 28672 data: 18 17 18 18 17 15 16
178872024-03-06T21:05:13.394Z20/30 Flush
178882024-03-06T21:05:13.394Z21/30 Write block 202 len 8192 data: 10 10
178892024-03-06T21:05:13.394Z22/30 Flush
178902024-03-06T21:05:13.402Z23/30 Write block 190 len 4096 data: 19
178912024-03-06T21:05:13.402Z24/30 Read block 529 len 40960
178922024-03-06T21:05:13.402Z25/30 Read block 300 len 32768
178932024-03-06T21:05:13.402Z26/30 Write block 36 len 28672 data: 8 5 9 11 13 16 17
178942024-03-06T21:05:13.402Z27/30 Read block 535 len 28672
178952024-03-06T21:05:13.402Z28/30 Read block 578 len 24576
178962024-03-06T21:05:13.402Z29/30 Read block 98 len 32768
178972024-03-06T21:05:13.402Z30/30 Read block 542 len 24576
178982024-03-06T21:05:13.402Z----------------------------------------------------------------
178992024-03-06T21:05:13.402Z Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:29
179002024-03-06T21:05:13.402ZStates: Active Active Active
179012024-03-06T21:05:13.402ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
179022024-03-06T21:05:13.402Z 2 Acked 1001 Write 8 Done Done Sent false
179032024-03-06T21:05:13.403Z 3 Acked 1002 Read 8 Done Done Sent false
179042024-03-06T21:05:13.403Z 4 Acked 1003 Read 1 Done Done Sent false
179052024-03-06T21:05:13.403Z 5 Acked 1004 Read 1 Done Done Sent false
179062024-03-06T21:05:13.403Z 6 Acked 1005 Write 9 Done Done Sent false
179072024-03-06T21:05:13.403Z 7 Acked 1006 Write 9 Done Done Sent false
179082024-03-06T21:05:13.403Z 8 Acked 1007 Read 6 Done Done Sent false
179092024-03-06T21:05:13.403Z 9 Acked 1008 Write 6 Done Done Sent false
179102024-03-06T21:05:13.403Z 10 Acked 1009 Write 2 Done Done Sent false
179112024-03-06T21:05:13.403Z 11 Acked 1010 Write 9 Done Done Sent false
179122024-03-06T21:05:13.403Z 12 Acked 1011 Read 6 Done Done Sent false
179132024-03-06T21:05:13.403Z 13 Acked 1012 Write 5 Done Done Sent false
179142024-03-06T21:05:13.403Z 14 Acked 1013 Write 5 Done Done Sent false
179152024-03-06T21:05:13.403Z 15 Acked 1014 Write 8 Done Done Sent false
179162024-03-06T21:05:13.403Z 16 Acked 1015 Read 2 Done Done Sent false
179172024-03-06T21:05:13.403Z 17 Acked 1016 Read 7 Done Done Sent false
179182024-03-06T21:05:13.403Z 18 Acked 1017 Flush 0 Done Done Sent false
179192024-03-06T21:05:13.403Z 19 Acked 1018 Write 7 Done Done Sent false
179202024-03-06T21:05:13.403Z 20 Acked 1019 Flush 0 Done Done Sent false
179212024-03-06T21:05:13.403Z 21 Acked 1020 Write 2 Done Done Sent false
179222024-03-06T21:05:13.403Z 22 Acked 1021 Flush 0 Done Done Sent false
179232024-03-06T21:05:13.403Z 23 Acked 1022 Write 1 Done Done Sent false
179242024-03-06T21:05:13.404Z 24 Acked 1023 Read 10 Done Done Sent false
179252024-03-06T21:05:13.404Z 25 Acked 1024 Read 8 Done Done Sent false
179262024-03-06T21:05:13.404Z 26 Acked 1025 Write 7 Done Done Sent false
179272024-03-06T21:05:13.404Z 27 Acked 1026 Read 7 Done Done Sent false
179282024-03-06T21:05:13.404Z 28 Acked 1027 Read 6 Done Done Sent false
179292024-03-06T21:05:13.404Z 29 Acked 1028 Read 8 Done Done Sent false
179302024-03-06T21:05:13.404Z 30 Acked 1029 Read 6 Done Done Sent false
179312024-03-06T21:05:13.404Z STATES DS:0 DS:1 DS:2 TOTAL
179322024-03-06T21:05:13.404Z New 0 0 0 0
179332024-03-06T21:05:13.404Z Sent 0 0 29 29
179342024-03-06T21:05:13.404Z Done 29 29 0 58
179352024-03-06T21:05:13.404Z Skipped 0 0 0 0
179362024-03-06T21:05:13.404Z Error 0 0 0 0
179372024-03-06T21:05:13.404ZLast Flush: 1021 1021 1000
179382024-03-06T21:05:13.404ZDownstairs last five completed: 1000
179392024-03-06T21:05:13.404ZUpstairs last five completed: 30 29 28 27 26
179402024-03-06T21:05:13.404Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:05:12.139120817Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1803}
179412024-03-06T21:05:13.404ZWrote out file "/tmp/test_repair_verify.data"
179422024-03-06T21:05:13.414Z
179432024-03-06T21:05:13.414ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
179442024-03-06T21:05:13.430ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
179452024-03-06T21:05:13.430Z 0 000-019 165 165 165 290 290 290 F F F
179462024-03-06T21:05:13.431Z 1 020-039 164 164 164 288 288 288 T T F <---
179472024-03-06T21:05:13.431Z 2 040-059 159 159 159 277 277 277 T T F <---
179482024-03-06T21:05:13.431Z 3 060-079 166 166 164 292 292 287 F F F <---
179492024-03-06T21:05:13.431Z 4 080-099 164 164 164 287 287 287 F F F
179502024-03-06T21:05:13.431Z 5 100-119 165 165 165 290 290 290 F F F
179512024-03-06T21:05:13.431Z 6 120-139 158 158 158 276 276 276 F F F
179522024-03-06T21:05:13.431Z 7 140-159 157 157 157 273 273 273 F F F
179532024-03-06T21:05:13.431Z 8 160-179 166 166 164 292 292 289 F F F <---
179542024-03-06T21:05:13.431Z 9 180-199 166 166 162 292 292 283 T T F <---
179552024-03-06T21:05:13.431Z 10 200-219 166 166 157 294 294 273 F F F <---
179562024-03-06T21:05:13.431Z 11 220-239 162 162 162 281 281 281 F F F
179572024-03-06T21:05:13.431Z 12 240-259 166 166 164 292 292 288 F F F <---
179582024-03-06T21:05:13.431Z 13 260-279 166 166 160 292 292 279 F F F <---
179592024-03-06T21:05:13.432Z 14 280-299 166 166 165 292 292 290 F F F <---
179602024-03-06T21:05:13.432Z 15 300-319 164 164 164 288 288 288 F F F
179612024-03-06T21:05:13.432Z 16 320-339 162 162 162 284 284 284 F F F
179622024-03-06T21:05:13.432Z 17 340-359 162 162 162 283 283 283 F F F
179632024-03-06T21:05:13.432Z 18 360-379 162 162 162 283 283 283 F F F
179642024-03-06T21:05:13.432Z 19 380-399 166 166 160 293 293 279 F F F <---
179652024-03-06T21:05:13.432Z 20 400-419 164 164 164 289 289 289 F F F
179662024-03-06T21:05:13.445Z 21 420-439 166 166 164 292 292 289 F F F <---
179672024-03-06T21:05:13.445Z 22 440-459 164 164 164 288 288 288 F F F
179682024-03-06T21:05:13.445Z 23 460-479 166 166 162 292 292 281 F F F <---
179692024-03-06T21:05:13.445Z 24 480-499 160 160 160 279 279 279 F F F
179702024-03-06T21:05:13.445Z 25 500-519 164 164 164 288 288 288 F F F
179712024-03-06T21:05:13.445Z 26 520-539 164 164 164 288 288 288 F F F
179722024-03-06T21:05:13.445Z 27 540-559 166 166 161 292 292 280 F F F <---
179732024-03-06T21:05:13.446Z 28 560-579 164 164 164 287 287 287 F F F
179742024-03-06T21:05:13.446Z 29 580-599 157 157 157 273 273 273 F F F
179752024-03-06T21:05:13.446ZMax gen: 166, Max flush: 294
179762024-03-06T21:05:13.446ZError: Difference in extent metadata found!
179772024-03-06T21:05:13.446ZA Difference in extent metadata is expected here
179782024-03-06T21:05:13.446ZOn loop 83
179792024-03-06T21:05:13.446Z
179802024-03-06T21:05:13.446ZVerifying data now
179812024-03-06T21:05:20.009ZNo change after verify
179822024-03-06T21:05:20.010ZLoop: 83 Downstairs dump after verify (and repair):
179832024-03-06T21:05:20.457ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
179842024-03-06T21:05:20.461Z 0 000-019 165 165 165 290 290 290 F F F
179852024-03-06T21:05:20.461Z 1 020-039 167 167 167 295 295 295 F F F
179862024-03-06T21:05:20.461Z 2 040-059 167 167 167 295 295 295 F F F
179872024-03-06T21:05:20.461Z 3 060-079 166 166 166 292 292 292 F F F
179882024-03-06T21:05:20.461Z 4 080-099 164 164 164 287 287 287 F F F
179892024-03-06T21:05:20.461Z 5 100-119 165 165 165 290 290 290 F F F
179902024-03-06T21:05:20.462Z 6 120-139 158 158 158 276 276 276 F F F
179912024-03-06T21:05:20.462Z 7 140-159 157 157 157 273 273 273 F F F
179922024-03-06T21:05:20.462Z 8 160-179 166 166 166 292 292 292 F F F
179932024-03-06T21:05:20.462Z 9 180-199 167 167 167 295 295 295 F F F
179942024-03-06T21:05:20.462Z 10 200-219 166 166 166 294 294 294 F F F
179952024-03-06T21:05:20.462Z 11 220-239 162 162 162 281 281 281 F F F
179962024-03-06T21:05:20.462Z 12 240-259 166 166 166 292 292 292 F F F
179972024-03-06T21:05:20.462Z 13 260-279 166 166 166 292 292 292 F F F
179982024-03-06T21:05:20.462Z 14 280-299 166 166 166 292 292 292 F F F
179992024-03-06T21:05:20.462Z 15 300-319 164 164 164 288 288 288 F F F
180002024-03-06T21:05:20.462Z 16 320-339 162 162 162 284 284 284 F F F
180012024-03-06T21:05:20.462Z 17 340-359 162 162 162 283 283 283 F F F
180022024-03-06T21:05:20.462Z 18 360-379 162 162 162 283 283 283 F F F
180032024-03-06T21:05:20.463Z 19 380-399 166 166 166 293 293 293 F F F
180042024-03-06T21:05:20.463Z 20 400-419 164 164 164 289 289 289 F F F
180052024-03-06T21:05:20.463Z 21 420-439 166 166 166 292 292 292 F F F
180062024-03-06T21:05:20.463Z 22 440-459 164 164 164 288 288 288 F F F
180072024-03-06T21:05:20.463Z 23 460-479 166 166 166 292 292 292 F F F
180082024-03-06T21:05:20.463Z 24 480-499 160 160 160 279 279 279 F F F
180092024-03-06T21:05:20.463Z 25 500-519 164 164 164 288 288 288 F F F
180102024-03-06T21:05:20.463Z 26 520-539 164 164 164 288 288 288 F F F
180112024-03-06T21:05:20.463Z 27 540-559 166 166 166 292 292 292 F F F
180122024-03-06T21:05:20.463Z 28 560-579 164 164 164 287 287 287 F F F
180132024-03-06T21:05:20.463Z 29 580-599 157 157 157 273 273 273 F F F
180142024-03-06T21:05:20.463ZMax gen: 167, Max flush: 295
180152024-03-06T21:05:20.464Z
180162024-03-06T21:05:20.464ZBegin loop 84
180172024-03-06T21:05:20.464ZDownstairs to restart: 0
180182024-03-06T21:05:20.464Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.897959761Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811}
180192024-03-06T21:05:20.464Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
180202024-03-06T21:05:20.464Z,"time":"2024-03-06T21:05:16.89826705Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811}
180212024-03-06T21:05:20.464Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.898376273Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811}
180222024-03-06T21:05:20.464Z{"msg":"Crucible bdb17862-778e-4f0e-9012-422a729ee99d has session id: 9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.89841575Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180232024-03-06T21:05:20.464Z{"msg":"Upstairs opts: Upstairs UUID: bdb17862-778e-4f0e-9012-422a729ee99d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.898445248Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180242024-03-06T21:05:20.464Z{"msg":"Crucible stats registered with UUID: bdb17862-778e-4f0e-9012-422a729ee99d","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.898469876Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180252024-03-06T21:05:20.464Z{"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.898484205Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811}
180262024-03-06T21:05:20.464Z{"msg":"bdb17862-778e-4f0e-9012-422a729ee99d active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.898617296Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180272024-03-06T21:05:20.465Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.898642454Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"client":"0","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180282024-03-06T21:05:20.465Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.898658163Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"client":"0","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180292024-03-06T21:05:20.465Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.898672312Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"client":"1","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180302024-03-06T21:05:20.465Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.89870241Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"client":"1","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180312024-03-06T21:05:20.465Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.898719519Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"client":"2","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180322024-03-06T21:05:20.465Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.898748887Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"client":"2","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180332024-03-06T21:05:20.465Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.898769715Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"":"io task","client":"1","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180342024-03-06T21:05:20.465Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.898794964Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"":"io task","client":"0","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180352024-03-06T21:05:20.465Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.898829711Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"":"io task","client":"2","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180362024-03-06T21:05:20.465Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.898936644Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"":"io task","client":"0","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180372024-03-06T21:05:20.465Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.898965652Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"":"io task","client":"1","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180382024-03-06T21:05:20.465Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.89899047Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"":"io task","client":"2","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180392024-03-06T21:05:20.465Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.899813494Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"client":"2","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180402024-03-06T21:05:20.466Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.899849291Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"client":"2","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180412024-03-06T21:05:20.466Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.89986592Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"client":"1","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180422024-03-06T21:05:20.466Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.899898908Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"client":"1","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180432024-03-06T21:05:20.466Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.900383734Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"client":"1","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180442024-03-06T21:05:20.466Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.900620528Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"client":"2","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180452024-03-06T21:05:20.466Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.900802906Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"client":"0","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180462024-03-06T21:05:20.466Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.900901129Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"client":"0","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180472024-03-06T21:05:20.466Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.900952015Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"client":"1","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180482024-03-06T21:05:20.466Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.900995512Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"client":"1","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180492024-03-06T21:05:20.466Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.9010302Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180502024-03-06T21:05:20.466Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.901067467Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"client":"2","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180512024-03-06T21:05:20.466Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.901084196Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"client":"2","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180522024-03-06T21:05:20.466Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.901105805Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180532024-03-06T21:05:20.467Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.901342148Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"client":"0","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180542024-03-06T21:05:20.467Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.90175506Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"client":"0","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180552024-03-06T21:05:20.467Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.901782558Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"client":"0","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180562024-03-06T21:05:20.467Z{"msg":"[0]R flush_numbers[0..12]: [290, 295, 295, 292, 287, 290, 276, 273, 292, 295, 294, 281]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.901798497Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"":"downstairs","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180572024-03-06T21:05:20.467Z{"msg":"[0]R generation[0..12]: [165, 167, 167, 166, 164, 165, 158, 157, 166, 167, 166, 162]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.901825045Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"":"downstairs","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180582024-03-06T21:05:20.467Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.901846203Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"":"downstairs","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180592024-03-06T21:05:20.467Z{"msg":"[1]R flush_numbers[0..12]: [290, 295, 295, 292, 287, 290, 276, 273, 292, 295, 294, 281]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.901869162Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"":"downstairs","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180602024-03-06T21:05:20.467Z{"msg":"[1]R generation[0..12]: [165, 167, 167, 166, 164, 165, 158, 157, 166, 167, 166, 162]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.901884161Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"":"downstairs","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180612024-03-06T21:05:20.467Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.90190515Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"":"downstairs","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180622024-03-06T21:05:20.467Z{"msg":"[2]R flush_numbers[0..12]: [290, 295, 295, 292, 287, 290, 276, 273, 292, 295, 294, 281]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
180632024-03-06T21:05:20.467Z,"time":"2024-03-06T21:05:16.901919289Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"":"downstairs","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180642024-03-06T21:05:20.467Z{"msg":"[2]R generation[0..12]: [165, 167, 167, 166, 164, 165, 158, 157, 166, 167, 166, 162]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.901947767Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"":"downstairs","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180652024-03-06T21:05:20.467Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.901968755Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"":"downstairs","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180662024-03-06T21:05:20.468Z{"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.901982014Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"":"downstairs","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180672024-03-06T21:05:20.468Z{"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.902002333Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"":"downstairs","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180682024-03-06T21:05:20.468Z{"msg":"Next flush: 296","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.902015612Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"":"downstairs","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180692024-03-06T21:05:20.468Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.902029211Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"":"downstairs","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180702024-03-06T21:05:20.468Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.90204302Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180712024-03-06T21:05:20.468Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.902061889Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"client":"0","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180722024-03-06T21:05:20.468Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.902075738Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"client":"1","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180732024-03-06T21:05:20.468Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.902089697Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"client":"2","":"downstairs","session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180742024-03-06T21:05:20.468Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.902109465Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180752024-03-06T21:05:20.468Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.902122904Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180762024-03-06T21:05:20.468Z{"msg":"bdb17862-778e-4f0e-9012-422a729ee99d is now active with session: 9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.902136334Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180772024-03-06T21:05:20.468Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.902155282Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811,"session_id":"9f0f8a0c-1225-4dcd-a9d3-3ea43a2cef9d"}
180782024-03-06T21:05:20.469Z{"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:16.902167652Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811}
180792024-03-06T21:05:20.469ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
180802024-03-06T21:05:20.469ZLoading write count information from file "/tmp/test_repair_verify.data"
180812024-03-06T21:05:20.469ZRun Repair workload
180822024-03-06T21:05:20.469Z01/30 Write block 494 len 24576 data: 12 15 15 15 12 11
180832024-03-06T21:05:20.483Z02/30 Read block 482 len 16384
180842024-03-06T21:05:20.483Z03/30 Write block 291 len 16384 data: 14 11 8 7
180852024-03-06T21:05:20.483Z04/30 Read block 188 len 8192
180862024-03-06T21:05:20.483Z05/30 Write block 537 len 16384 data: 13 9 7 11
180872024-03-06T21:05:20.483Z06/30 Write block 118 len 24576 data: 12 13 10 9 9 7
180882024-03-06T21:05:20.484Z07/30 Read block 558 len 4096
180892024-03-06T21:05:20.484Z08/30 Read block 166 len 20480
180902024-03-06T21:05:20.484Z09/30 Read block 239 len 16384
180912024-03-06T21:05:20.484Z10/30 Write block 488 len 36864 data: 12 14 10 11 11 14 13 16 16
180922024-03-06T21:05:20.484Z11/30 Read block 530 len 4096
180932024-03-06T21:05:20.484Z12/30 Read block 10 len 8192
180942024-03-06T21:05:20.484Z13/30 Write block 405 len 28672 data: 11 11 14 15 13 14 12
180952024-03-06T21:05:20.484Z14/30 Write block 43 len 8192 data: 16 14
180962024-03-06T21:05:20.484Z15/30 Read block 79 len 32768
180972024-03-06T21:05:20.484Z16/30 Read block 300 len 4096
180982024-03-06T21:05:20.484Z17/30 Read block 178 len 40960
180992024-03-06T21:05:20.484Z18/30 Flush
181002024-03-06T21:05:20.484Z19/30 Flush
181012024-03-06T21:05:20.485Z20/30 Write block 188 len 4096 data: 11
181022024-03-06T21:05:20.485Z21/30 Write block 189 len 24576 data: 17 20 14 16 12 11
181032024-03-06T21:05:20.485Z22/30 Write block 44 len 12288 data: 15 9 10
181042024-03-06T21:05:20.485Z23/30 Read block 325 len 8192
181052024-03-06T21:05:20.485Z24/30 Write block 340 len 40960 data: 16 15 14 16 15 12 10 10 14 14
181062024-03-06T21:05:20.485Z25/30 Write block 334 len 8192 data: 9 8
181072024-03-06T21:05:20.485Z26/30 Read block 349 len 8192
181082024-03-06T21:05:20.485Z27/30 Write block 302 len 40960 data: 14 14 16 17 19 18 17 15 14 15
181092024-03-06T21:05:20.485Z28/30 Read block 141 len 12288
181102024-03-06T21:05:20.486Z29/30 Write block 309 len 8192 data: 16 15
181112024-03-06T21:05:20.486Z30/30 Read block 209 len 24576
181122024-03-06T21:05:20.486Z----------------------------------------------------------------
181132024-03-06T21:05:20.486Z Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:30
181142024-03-06T21:05:20.486ZStates: Active Active Active
181152024-03-06T21:05:20.486ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
181162024-03-06T21:05:20.486Z 1 Acked 1000 Write 6 Sent Done Done false
181172024-03-06T21:05:20.486Z 2 Acked 1001 Read 4 Sent Done Done false
181182024-03-06T21:05:20.486Z 3 Acked 1002 Write 4 Sent Done Done false
181192024-03-06T21:05:20.486Z 4 Acked 1003 Read 2 Sent Done Done false
181202024-03-06T21:05:20.486Z 5 Acked 1004 Write 4 Sent Done Done false
181212024-03-06T21:05:20.486Z 6 Acked 1005 Write 6 Sent Done Done false
181222024-03-06T21:05:20.486Z 7 Acked 1006 Read 1 Sent Done Done false
181232024-03-06T21:05:20.487Z 8 Acked 1007 Read 5 Sent Done Done false
181242024-03-06T21:05:20.487Z 9 Acked 1008 Read 4 Sent Done Done false
181252024-03-06T21:05:20.487Z 10 Acked 1009 Write 9 Sent Done Done false
181262024-03-06T21:05:20.487Z 11 Acked 1010 Read 1 Sent Done Done false
181272024-03-06T21:05:20.487Z 12 Acked 1011 Read 2 Sent Done Done false
181282024-03-06T21:05:20.487Z 13 Acked 1012 Write 7 Sent Done Done false
181292024-03-06T21:05:20.487Z 14 Acked 1013 Write 2 Sent Done Done false
181302024-03-06T21:05:20.487Z 15 Acked 1014 Read 8 Sent Done Done false
181312024-03-06T21:05:20.487Z 16 Acked 1015 Read 1 Sent Done Done false
181322024-03-06T21:05:20.487Z 17 Acked 1016 Read 10 Sent Done Done false
181332024-03-06T21:05:20.487Z 18 Acked 1017 Flush 0 Sent Done Done false
181342024-03-06T21:05:20.487Z 19 Acked 1018 Flush 0 Sent Done Done false
181352024-03-06T21:05:20.487Z 20 Acked 1019 Write 1 Sent Done Done false
181362024-03-06T21:05:20.487Z 21 Acked 1020 Write 6 Sent Done Done false
181372024-03-06T21:05:20.488Z 22 Acked 1021 Write 3 Sent Done Done false
181382024-03-06T21:05:20.488Z 23 Acked 1022 Read 2 Sent Done Done false
181392024-03-06T21:05:20.488Z 24 Acked 1023 Write 10 Sent Done Done false
181402024-03-06T21:05:20.488Z 25 Acked 1024 Write 2 Sent Done Done false
181412024-03-06T21:05:20.488Z 26 Acked 1025 Read 2 Sent Done Done false
181422024-03-06T21:05:20.488Z 27 Acked 1026 Write 10 Sent Done Done false
181432024-03-06T21:05:20.488Z 28 Acked 1027 Read 3 Sent Done Done false
181442024-03-06T21:05:20.488Z 29 Acked 1028 Write 2 Sent Done Done false
181452024-03-06T21:05:20.488Z 30 Acked 1029 Read 6 Sent Done Done false
181462024-03-06T21:05:20.488Z STATES DS:0 DS:1 DS:2 TOTAL
181472024-03-06T21:05:20.488Z New 0 0 0 0
181482024-03-06T21:05:20.488Z Sent 30 0 0 30
181492024-03-06T21:05:20.488Z Done 0 30 30 60
181502024-03-06T21:05:20.489Z Skipped 0 0 0 0
181512024-03-06T21:05:20.489Z Error 0 0 0 0
181522024-03-06T21:05:20.489ZLast Flush: 0 1018 1018
181532024-03-06T21:05:20.489ZDownstairs last five completed:
181542024-03-06T21:05:20.489ZUpstairs last five completed: 30 29 28 27 26
181552024-03-06T21:05:20.489Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:05:16.933698659Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1811}
181562024-03-06T21:05:20.489ZWrote out file "/tmp/test_repair_verify.data"
181572024-03-06T21:05:20.489Z
181582024-03-06T21:05:20.489ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
181592024-03-06T21:05:20.489ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
181602024-03-06T21:05:20.489Z 0 000-019 165 165 165 290 290 290 F F F
181612024-03-06T21:05:20.489Z 1 020-039 167 167 167 295 295 295 F F F
181622024-03-06T21:05:20.489Z 2 040-059 167 168 168 295 296 296 F T T <---
181632024-03-06T21:05:20.490Z 3 060-079 166 166 166 292 292 292 F F F
181642024-03-06T21:05:20.490Z 4 080-099 164 164 164 287 287 287 F F F
181652024-03-06T21:05:20.490Z 5 100-119 165 168 168 290 296 296 F F F <---
181662024-03-06T21:05:20.490Z 6 120-139 158 168 168 276 296 296 F F F <---
181672024-03-06T21:05:20.490Z 7 140-159 157 157 157 273 273 273 F F F
181682024-03-06T21:05:20.490Z 8 160-179 166 166 166 292 292 292 F F F
181692024-03-06T21:05:20.490Z 9 180-199 167 167 167 295 295 295 F T T <---
181702024-03-06T21:05:20.490Z 10 200-219 166 166 166 294 294 294 F F F
181712024-03-06T21:05:20.490Z 11 220-239 162 162 162 281 281 281 F F F
181722024-03-06T21:05:20.490Z 12 240-259 166 166 166 292 292 292 F F F
181732024-03-06T21:05:20.490Z 13 260-279 166 166 166 292 292 292 F F F
181742024-03-06T21:05:20.490Z 14 280-299 166 168 168 292 296 296 F F F <---
181752024-03-06T21:05:20.490Z 15 300-319 164 164 164 288 288 288 F T T <---
181762024-03-06T21:05:20.490Z 16 320-339 162 162 162 284 284 284 F T T <---
181772024-03-06T21:05:20.491Z 17 340-359 162 162 162 283 283 283 F T T <---
181782024-03-06T21:05:20.491Z 18 360-379 162 162 162 283 283 283 F F F
181792024-03-06T21:05:20.491Z 19 380-399 166 166 166 293 293 293 F F F
181802024-03-06T21:05:20.491Z 20 400-419 164 168 168 289 296 296 F F F <---
181812024-03-06T21:05:20.491Z 21 420-439 166 166 166 292 292 292 F F F
181822024-03-06T21:05:20.491Z 22 440-459 164 164 164 288 288 288 F F F
181832024-03-06T21:05:20.510Z 23 460-479 166 166 166 292 292 292 F F F
181842024-03-06T21:05:20.510Z 24 480-499 160 168 168 279 296 296 F F F <---
181852024-03-06T21:05:20.510Z 25 500-519 164 164 164 288 288 288 F F F
181862024-03-06T21:05:20.511Z 26 520-539 164 168 168 288 296 296 F F F <---
181872024-03-06T21:05:20.511Z 27 540-559 166 168 168 292 296 296 F F F <---
181882024-03-06T21:05:20.511Z 28 560-579 164 164 164 287 287 287 F F F
181892024-03-06T21:05:20.511Z 29 580-599 157 157 157 273 273 273 F F F
181902024-03-06T21:05:20.511ZMax gen: 168, Max flush: 296
181912024-03-06T21:05:20.511ZError: Difference in extent metadata found!
181922024-03-06T21:05:20.511ZA Difference in extent metadata is expected here
181932024-03-06T21:05:20.511ZOn loop 84
181942024-03-06T21:05:20.511Z
181952024-03-06T21:05:20.511ZVerifying data now
181962024-03-06T21:05:22.928ZNo change after verify
181972024-03-06T21:05:22.928ZLoop: 84 Downstairs dump after verify (and repair):
181982024-03-06T21:05:22.970ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
181992024-03-06T21:05:22.970Z 0 000-019 165 165 165 290 290 290 F F F
182002024-03-06T21:05:22.970Z 1 020-039 167 167 167 295 295 295 F F F
182012024-03-06T21:05:22.970Z 2 040-059 169 169 169 297 297 297 F F F
182022024-03-06T21:05:22.970Z 3 060-079 166 166 166 292 292 292 F F F
182032024-03-06T21:05:22.971Z 4 080-099 164 164 164 287 287 287 F F F
182042024-03-06T21:05:22.971Z 5 100-119 168 168 168 296 296 296 F F F
182052024-03-06T21:05:22.971Z 6 120-139 168 168 168 296 296 296 F F F
182062024-03-06T21:05:22.971Z 7 140-159 157 157 157 273 273 273 F F F
182072024-03-06T21:05:22.971Z 8 160-179 166 166 166 292 292 292 F F F
182082024-03-06T21:05:22.971Z 9 180-199 169 169 169 297 297 297 F F F
182092024-03-06T21:05:22.971Z 10 200-219 166 166 166 294 294 294 F F F
182102024-03-06T21:05:22.971Z 11 220-239 162 162 162 281 281 281 F F F
182112024-03-06T21:05:22.971Z 12 240-259 166 166 166 292 292 292 F F F
182122024-03-06T21:05:22.971Z 13 260-279 166 166 166 292 292 292 F F F
182132024-03-06T21:05:22.971Z 14 280-299 168 168 168 296 296 296 F F F
182142024-03-06T21:05:22.972Z 15 300-319 169 169 169 297 297 297 F F F
182152024-03-06T21:05:22.972Z 16 320-339 169 169 169 297 297 297 F F F
182162024-03-06T21:05:22.972Z 17 340-359 169 169 169 297 297 297 F F F
182172024-03-06T21:05:22.972Z 18 360-379 162 162 162 283 283 283 F F F
182182024-03-06T21:05:22.972Z 19 380-399 166 166 166 293 293 293 F F F
182192024-03-06T21:05:22.972Z 20 400-419 168 168 168 296 296 296 F F F
182202024-03-06T21:05:22.972Z 21 420-439 166 166 166 292 292 292 F F F
182212024-03-06T21:05:22.972Z 22 440-459 164 164 164 288 288 288 F F F
182222024-03-06T21:05:22.972Z 23 460-479 166 166 166 292 292 292 F F F
182232024-03-06T21:05:22.972Z 24 480-499 168 168 168 296 296 296 F F F
182242024-03-06T21:05:22.972Z 25 500-519 164 164 164 288 288 288 F F F
182252024-03-06T21:05:22.972Z 26 520-539 168 168 168 296 296 296 F F F
182262024-03-06T21:05:22.972Z 27 540-559 168 168 168 296 296 296 F F F
182272024-03-06T21:05:22.973Z 28 560-579 164 164 164 287 287 287 F F F
182282024-03-06T21:05:22.973Z 29 580-599 157 157 157 273 273 273 F F F
182292024-03-06T21:05:22.973ZMax gen: 169, Max flush: 297
182302024-03-06T21:05:22.973Z
182312024-03-06T21:05:22.973ZBegin loop 85
182322024-03-06T21:05:22.973ZDownstairs to restart: 0
182332024-03-06T21:05:22.973Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.640476092Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819}
182342024-03-06T21:05:22.973Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.640716496Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819}
182352024-03-06T21:05:22.973Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.640737294Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819}
182362024-03-06T21:05:22.973Z{"msg":"Crucible 910b1c0b-e9a2-46b9-8cf1-01ee9a6550ee has session id: dd49c7f2-50ea-4606-8529-51816310ef4f","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
182372024-03-06T21:05:22.973Z2024-03-06T21:05:21.640767262Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182382024-03-06T21:05:22.973Z{"msg":"Upstairs opts: Upstairs UUID: 910b1c0b-e9a2-46b9-8cf1-01ee9a6550ee, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.640825688Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182392024-03-06T21:05:22.973Z{"msg":"Crucible stats registered with UUID: 910b1c0b-e9a2-46b9-8cf1-01ee9a6550ee","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.640864405Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182402024-03-06T21:05:22.974Z{"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.640883734Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819}
182412024-03-06T21:05:22.974Z{"msg":"910b1c0b-e9a2-46b9-8cf1-01ee9a6550ee active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.641050703Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182422024-03-06T21:05:22.974Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.641098649Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"client":"0","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182432024-03-06T21:05:22.974Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.641115878Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"client":"0","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182442024-03-06T21:05:22.974Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.641149996Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"client":"1","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182452024-03-06T21:05:22.974Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.641195563Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"client":"1","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182462024-03-06T21:05:22.974Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.64123535Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"client":"2","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182472024-03-06T21:05:22.974Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.641270337Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"client":"2","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182482024-03-06T21:05:22.974Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.641299195Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"":"io task","client":"2","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182492024-03-06T21:05:22.974Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.641329613Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"":"io task","client":"0","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182502024-03-06T21:05:22.974Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.64137079Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"":"io task","client":"1","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182512024-03-06T21:05:22.974Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.641505121Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"":"io task","client":"1","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182522024-03-06T21:05:22.974Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.641538789Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"":"io task","client":"0","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182532024-03-06T21:05:22.974Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.641562487Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"":"io task","client":"2","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182542024-03-06T21:05:22.974Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.642220402Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"client":"2","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182552024-03-06T21:05:22.974Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.642260209Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"client":"2","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182562024-03-06T21:05:22.974Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.642276778Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"client":"1","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182572024-03-06T21:05:22.974Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.642302536Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"client":"1","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182582024-03-06T21:05:22.974Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.642908595Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"client":"1","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182592024-03-06T21:05:22.975Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.642963561Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"client":"2","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182602024-03-06T21:05:22.975Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.643593627Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"client":"1","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182612024-03-06T21:05:22.975Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.643641364Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"client":"1","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182622024-03-06T21:05:22.975Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.643668372Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182632024-03-06T21:05:22.975Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.64369932Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"client":"0","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182642024-03-06T21:05:22.975Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.643720369Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"client":"0","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182652024-03-06T21:05:22.975Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.643755896Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"client":"2","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182662024-03-06T21:05:22.975Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.643792814Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"client":"2","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182672024-03-06T21:05:22.975Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.643828771Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182682024-03-06T21:05:22.975Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.644298919Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"client":"0","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182692024-03-06T21:05:22.975Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.644672043Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"client":"0","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182702024-03-06T21:05:22.975Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.644705291Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"client":"0","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182712024-03-06T21:05:22.975Z{"msg":"[0]R flush_numbers[0..12]: [290, 295, 297, 292, 287, 296, 296, 273, 292, 297, 294, 281]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.64472194Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"":"downstairs","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182722024-03-06T21:05:22.975Z{"msg":"[0]R generation[0..12]: [165, 167, 169, 166, 164, 168, 168, 157, 166, 169, 166, 162]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.644743538Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"":"downstairs","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182732024-03-06T21:05:22.975Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.644763857Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"":"downstairs","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182742024-03-06T21:05:22.975Z{"msg":"[1]R flush_numbers[0..12]: [290, 295, 297, 292, 287, 296, 296, 273, 292, 297, 294, 281]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.644786125Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"":"downstairs","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182752024-03-06T21:05:22.975Z{"msg":"[1]R generation[0..12]: [165, 167, 169, 166, 164, 168, 168, 157, 166, 169, 166, 162]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.644813323Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"":"downstairs","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182762024-03-06T21:05:22.975Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.644828572Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"":"downstairs","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182772024-03-06T21:05:22.975Z{"msg":"[2]R flush_numbers[0..12]: [290, 295, 297, 292, 287, 296, 296, 273, 292, 297, 294, 281]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.644849221Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"":"downstairs","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182782024-03-06T21:05:22.975Z{"msg":"[2]R generation[0..12]: [165, 167, 169, 166, 164, 168, 168, 157, 166, 169, 166, 162]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.64486428Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"":"downstairs","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182792024-03-06T21:05:22.975Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]",Wait for a query_work_queue command to finish before sending IO
182802024-03-06T21:05:22.975Z"v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.644887878Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"":"downstairs","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182812024-03-06T21:05:22.975Z{"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.644901997Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"":"downstairs","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182822024-03-06T21:05:22.976Z{"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.644921866Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"":"downstairs","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182832024-03-06T21:05:22.976Z{"msg":"Next flush: 298","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.644935155Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"":"downstairs","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182842024-03-06T21:05:22.976Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.644954554Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"":"downstairs","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182852024-03-06T21:05:22.976Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.644967943Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182862024-03-06T21:05:22.976Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.644992371Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"client":"0","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182872024-03-06T21:05:22.976Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.645017009Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"client":"1","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182882024-03-06T21:05:22.976Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.645054907Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"client":"2","":"downstairs","session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182892024-03-06T21:05:22.976Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.645098604Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182902024-03-06T21:05:22.976Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.645123272Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182912024-03-06T21:05:22.976Z{"msg":"910b1c0b-e9a2-46b9-8cf1-01ee9a6550ee is now active with session: dd49c7f2-50ea-4606-8529-51816310ef4f","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.6451478Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182922024-03-06T21:05:22.976Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.645190007Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819,"session_id":"dd49c7f2-50ea-4606-8529-51816310ef4f"}
182932024-03-06T21:05:22.976Z{"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:21.645215196Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819}
182942024-03-06T21:05:22.976ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
182952024-03-06T21:05:22.976ZLoading write count information from file "/tmp/test_repair_verify.data"
182962024-03-06T21:05:22.976ZRun Repair workload
182972024-03-06T21:05:22.976Z01/30 Write block 580 len 20480 data: 9 12 11 11 12
182982024-03-06T21:05:22.993Z02/30 Write block 541 len 36864 data: 11 14 16 18 19 18 16 14 11
182992024-03-06T21:05:22.993Z03/30 Flush
183002024-03-06T21:05:22.993Z04/30 Write block 151 len 8192 data: 13 10
183012024-03-06T21:05:22.993Z05/30 Write block 355 len 8192 data: 13 18
183022024-03-06T21:05:22.993Z06/30 Read block 350 len 16384
183032024-03-06T21:05:22.993Z07/30 Read block 424 len 32768
183042024-03-06T21:05:22.994Z08/30 Flush
183052024-03-06T21:05:22.994Z09/30 Write block 123 len 28672 data: 8 11 16 14 15 14 14
183062024-03-06T21:05:22.994Z10/30 Read block 552 len 20480
183072024-03-06T21:05:22.994Z11/30 Write block 358 len 16384 data: 17 15 15 19
183082024-03-06T21:05:22.994Z12/30 Write block 253 len 24576 data: 9 8 9 9 8 8
183092024-03-06T21:05:22.994Z13/30 Read block 595 len 8192
183102024-03-06T21:05:22.994Z14/30 Flush
183112024-03-06T21:05:22.994Z15/30 Write block 198 len 12288 data: 17 14 14
183122024-03-06T21:05:22.994Z16/30 Write block 482 len 40960 data: 10 12 12 14 11 11 13 15 11 12
183132024-03-06T21:05:22.994Z17/30 Read block 28 len 12288
183142024-03-06T21:05:22.994Z18/30 Write block 277 len 4096 data: 11
183152024-03-06T21:05:22.994Z19/30 Write block 519 len 4096 data: 13
183162024-03-06T21:05:22.994Z20/30 Read block 393 len 32768
183172024-03-06T21:05:22.994Z21/30 Read block 142 len 40960
183182024-03-06T21:05:22.994Z22/30 Write block 183 len 20480 data: 15 15 14 12 11
183192024-03-06T21:05:22.995Z23/30 Read block 333 len 4096
183202024-03-06T21:05:22.995Z24/30 Read block 6 len 32768
183212024-03-06T21:05:22.995Z25/30 Write block 362 len 40960 data: 15 18 15 13 11 12 10 14 15 16
183222024-03-06T21:05:22.995Z26/30 Write block 277 len 28672 data: 12 11 13 12 13 15 13
183232024-03-06T21:05:22.995Z27/30 Write block 294 len 36864 data: 8 10 10 8 7 6 9 10 15
183242024-03-06T21:05:22.995Z28/30 Write block 380 len 20480 data: 17 17 19 18 19
183252024-03-06T21:05:22.995Z29/30 Read block 193 len 12288
183262024-03-06T21:05:22.995Z30/30 Read block 577 len 28672
183272024-03-06T21:05:22.995Z----------------------------------------------------------------
183282024-03-06T21:05:22.995Z Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30
183292024-03-06T21:05:22.995ZStates: Active Active Active
183302024-03-06T21:05:22.995ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
183312024-03-06T21:05:22.995Z 1 Acked 1000 Write 5 Sent Done Done false
183322024-03-06T21:05:22.995Z 2 Acked 1001 Write 9 Sent Done Done false
183332024-03-06T21:05:22.995Z 3 Acked 1002 Flush 0 Sent Done Done false
183342024-03-06T21:05:22.995Z 4 Acked 1003 Write 2 Sent Done Done false
183352024-03-06T21:05:22.995Z 5 Acked 1004 Write 2 Sent Done Done false
183362024-03-06T21:05:22.995Z 6 Acked 1005 Read 4 Sent Done Done false
183372024-03-06T21:05:22.995Z 7 Acked 1006 Read 8 Sent Done Done false
183382024-03-06T21:05:22.995Z 8 Acked 1007 Flush 0 Sent Done Done false
183392024-03-06T21:05:22.995Z 9 Acked 1008 Write 7 Sent Done Done false
183402024-03-06T21:05:22.996Z 10 Acked 1009 Read 5 Sent Done Done false
183412024-03-06T21:05:22.996Z 11 Acked 1010 Write 4 Sent Done Done false
183422024-03-06T21:05:22.996Z 12 Acked 1011 Write 6 Sent Done Done false
183432024-03-06T21:05:22.996Z 13 Acked 1012 Read 2 Sent Done Done false
183442024-03-06T21:05:22.996Z 14 Acked 1013 Flush 0 Sent Done Done false
183452024-03-06T21:05:22.996Z 15 Acked 1014 Write 3 Sent Done Done false
183462024-03-06T21:05:22.996Z 16 Acked 1015 Write 10 Sent Done Done false
183472024-03-06T21:05:22.996Z 17 Acked 1016 Read 3 Sent Done Done false
183482024-03-06T21:05:22.996Z 18 Acked 1017 Write 1 Sent Done Done false
183492024-03-06T21:05:22.996Z 19 Acked 1018 Write 1 Sent Done Done false
183502024-03-06T21:05:22.996Z 20 Acked 1019 Read 8 Sent Done Done false
183512024-03-06T21:05:22.996Z 21 Acked 1020 Read 10 Sent Done Done false
183522024-03-06T21:05:22.996Z 22 Acked 1021 Write 5 Sent Done Done false
183532024-03-06T21:05:22.996Z 23 Acked 1022 Read 1 Sent Done Done false
183542024-03-06T21:05:22.996Z 24 Acked 1023 Read 8 Sent Done Done false
183552024-03-06T21:05:22.996Z 25 Acked 1024 Write 10 Sent Done Done false
183562024-03-06T21:05:22.996Z 26 Acked 1025 Write 7 Sent Done Done false
183572024-03-06T21:05:22.996Z 27 Acked 1026 Write 9 Sent Done Done false
183582024-03-06T21:05:22.996Z 28 Acked 1027 Write 5 Sent Done Done false
183592024-03-06T21:05:22.996Z 29 Acked 1028 Read 3 Sent Done Done false
183602024-03-06T21:05:22.996Z 30 Acked 1029 Read 7 Sent Done Done false
183612024-03-06T21:05:22.996Z STATES DS:0 DS:1 DS:2 TOTAL
183622024-03-06T21:05:22.996Z New 0 0 0 0
183632024-03-06T21:05:22.996Z Sent 30 0 0 30
183642024-03-06T21:05:22.997Z Done 0 30 30 60
183652024-03-06T21:05:22.997Z Skipped 0 0 0 0
183662024-03-06T21:05:22.997Z Error 0 0 0 0
183672024-03-06T21:05:22.997ZLast Flush: 0 1013 1013
183682024-03-06T21:05:22.997ZDownstairs last five completed:
183692024-03-06T21:05:22.997ZUpstairs last five completed: 30 29 28 27 26
183702024-03-06T21:05:23.001Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:05:21.682715211Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1819}
183712024-03-06T21:05:23.002ZWrote out file "/tmp/test_repair_verify.data"
183722024-03-06T21:05:23.002Z
183732024-03-06T21:05:23.002ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
183742024-03-06T21:05:23.002ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
183752024-03-06T21:05:23.002Z 0 000-019 165 165 165 290 290 290 F F F
183762024-03-06T21:05:23.002Z 1 020-039 167 167 167 295 295 295 F F F
183772024-03-06T21:05:23.002Z 2 040-059 169 169 169 297 297 297 F F F
183782024-03-06T21:05:23.002Z 3 060-079 166 166 166 292 292 292 F F F
183792024-03-06T21:05:23.002Z 4 080-099 164 164 164 287 287 287 F F F
183802024-03-06T21:05:23.002Z 5 100-119 168 168 168 296 296 296 F F F
183812024-03-06T21:05:23.002Z 6 120-139 168 170 170 296 300 300 F F F <---
183822024-03-06T21:05:23.003Z 7 140-159 157 170 170 273 299 299 F F F <---
183832024-03-06T21:05:23.003Z 8 160-179 166 166 166 292 292 292 F F F
183842024-03-06T21:05:23.003Z 9 180-199 169 169 169 297 297 297 F T T <---
183852024-03-06T21:05:23.003Z 10 200-219 166 166 166 294 294 294 F T T <---
183862024-03-06T21:05:23.003Z 11 220-239 162 162 162 281 281 281 F F F
183872024-03-06T21:05:23.003Z 12 240-259 166 170 170 292 300 300 F F F <---
183882024-03-06T21:05:23.003Z 13 260-279 166 166 166 292 292 292 F T T <---
183892024-03-06T21:05:23.003Z 14 280-299 168 168 168 296 296 296 F T T <---
183902024-03-06T21:05:23.003Z 15 300-319 169 169 169 297 297 297 F T T <---
183912024-03-06T21:05:23.003Z 16 320-339 169 169 169 297 297 297 F F F
183922024-03-06T21:05:23.004Z 17 340-359 169 170 170 297 300 300 F F F <---
183932024-03-06T21:05:23.004Z 18 360-379 162 170 170 283 300 300 F T T <---
183942024-03-06T21:05:23.004Z 19 380-399 166 166 166 293 293 293 F T T <---
183952024-03-06T21:05:23.004Z 20 400-419 168 168 168 296 296 296 F F F
183962024-03-06T21:05:23.004Z 21 420-439 166 166 166 292 292 292 F F F
183972024-03-06T21:05:23.004Z 22 440-459 164 164 164 288 288 288 F F F
183982024-03-06T21:05:23.024Z 23 460-479 166 166 166 292 292 292 F F F
183992024-03-06T21:05:23.024Z 24 480-499 168 168 168 296 296 296 F T T <---
184002024-03-06T21:05:23.025Z 25 500-519 164 164 164 288 288 288 F T T <---
184012024-03-06T21:05:23.025Z 26 520-539 168 168 168 296 296 296 F F F
184022024-03-06T21:05:23.025Z 27 540-559 168 170 170 296 298 298 F F F <---
184032024-03-06T21:05:23.025Z 28 560-579 164 164 164 287 287 287 F F F
184042024-03-06T21:05:23.025Z 29 580-599 157 170 170 273 298 298 F F F <---
184052024-03-06T21:05:23.025ZMax gen: 170, Max flush: 300
184062024-03-06T21:05:23.025ZError: Difference in extent metadata found!
184072024-03-06T21:05:23.025ZA Difference in extent metadata is expected here
184082024-03-06T21:05:23.025ZOn loop 85
184092024-03-06T21:05:23.025Z
184102024-03-06T21:05:23.025ZVerifying data now
184112024-03-06T21:05:27.747ZNo change after verify
184122024-03-06T21:05:27.747ZLoop: 85 Downstairs dump after verify (and repair):
184132024-03-06T21:05:27.762ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
184142024-03-06T21:05:27.762Z 0 000-019 165 165 165 290 290 290 F F F
184152024-03-06T21:05:27.762Z 1 020-039 167 167 167 295 295 295 F F F
184162024-03-06T21:05:27.762Z 2 040-059 169 169 169 297 297 297 F F F
184172024-03-06T21:05:27.762Z 3 060-079 166 166 166 292 292 292 F F F
184182024-03-06T21:05:27.762Z 4 080-099 164 164 164 287 287 287 F F F
184192024-03-06T21:05:27.762Z 5 100-119 168 168 168 296 296 296 F F F
184202024-03-06T21:05:27.762Z 6 120-139 170 170 170 300 300 300 F F F
184212024-03-06T21:05:27.763Z 7 140-159 170 170 170 299 299 299 F F F
184222024-03-06T21:05:27.763Z 8 160-179 166 166 166 292 292 292 F F F
184232024-03-06T21:05:27.763Z 9 180-199 171 171 171 301 301 301 F F F
184242024-03-06T21:05:27.763Z 10 200-219 171 171 171 301 301 301 F F F
184252024-03-06T21:05:27.763Z 11 220-239 162 162 162 281 281 281 F F F
184262024-03-06T21:05:27.763Z 12 240-259 170 170 170 300 300 300 F F F
184272024-03-06T21:05:27.763Z 13 260-279 171 171 171 301 301 301 F F F
184282024-03-06T21:05:27.763Z 14 280-299 171 171 171 301 301 301 F F F
184292024-03-06T21:05:27.763Z 15 300-319 171 171 171 301 301 301 F F F
184302024-03-06T21:05:27.763Z 16 320-339 169 169 169 297 297 297 F F F
184312024-03-06T21:05:27.763Z 17 340-359 170 170 170 300 300 300 F F F
184322024-03-06T21:05:27.763Z 18 360-379 171 171 171 301 301 301 F F F
184332024-03-06T21:05:27.763Z 19 380-399 171 171 171 301 301 301 F F F
184342024-03-06T21:05:27.763Z 20 400-419 168 168 168 296 296 296 F F F
184352024-03-06T21:05:27.763Z 21 420-439 166 166 166 292 292 292 F F F
184362024-03-06T21:05:27.763Z 22 440-459 164 164 164 288 288 288 F F F
184372024-03-06T21:05:27.767Z 23 460-479 166 166 166 292 292 292 F F F
184382024-03-06T21:05:27.767Z 24 480-499 171 171 171 301 301 301 F F F
184392024-03-06T21:05:27.767Z 25 500-519 171 171 171 301 301 301 F F F
184402024-03-06T21:05:27.767Z 26 520-539 168 168 168 296 296 296 F F F
184412024-03-06T21:05:27.767Z 27 540-559 170 170 170 298 298 298 F F F
184422024-03-06T21:05:27.767Z 28 560-579 164 164 164 287 287 287 F F F
184432024-03-06T21:05:27.767Z 29 580-599 170 170 170 298 298 298 F F F
184442024-03-06T21:05:27.767ZMax gen: 171, Max flush: 301
184452024-03-06T21:05:27.767Z
184462024-03-06T21:05:27.768ZBegin loop 86
184472024-03-06T21:05:27.768ZDownstairs to restart: 2
184482024-03-06T21:05:27.791Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.530330629Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827}
184492024-03-06T21:05:27.791Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
184502024-03-06T21:05:27.791Z,"time":"2024-03-06T21:05:26.530677075Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827}
184512024-03-06T21:05:27.791Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.530710223Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827}
184522024-03-06T21:05:27.798Z{"msg":"Crucible 17eb5fd6-4a89-4256-8b2b-b0a4f9c4278d has session id: fed9fb3b-beda-4428-9e4d-dc8f695bfa7b","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.53075164Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184532024-03-06T21:05:27.798Z{"msg":"Upstairs opts: Upstairs UUID: 17eb5fd6-4a89-4256-8b2b-b0a4f9c4278d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.530808376Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184542024-03-06T21:05:27.798Z{"msg":"Crucible stats registered with UUID: 17eb5fd6-4a89-4256-8b2b-b0a4f9c4278d","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.530853203Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184552024-03-06T21:05:27.798Z{"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.53088548Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827}
184562024-03-06T21:05:27.798Z{"msg":"17eb5fd6-4a89-4256-8b2b-b0a4f9c4278d active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.530903789Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184572024-03-06T21:05:27.798Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.530924538Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"client":"0","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184582024-03-06T21:05:27.798Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.530944636Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"client":"0","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184592024-03-06T21:05:27.798Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.530977854Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"client":"1","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184602024-03-06T21:05:27.799Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.531050049Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"client":"1","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184612024-03-06T21:05:27.799Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.531093686Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"client":"2","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184622024-03-06T21:05:27.799Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.531129114Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"client":"2","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184632024-03-06T21:05:27.799Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.531154572Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"":"io task","client":"2","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184642024-03-06T21:05:27.799Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.531184Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"":"io task","client":"1","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184652024-03-06T21:05:27.799Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.531216448Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"":"io task","client":"0","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184662024-03-06T21:05:27.799Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.531334869Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"":"io task","client":"0","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184672024-03-06T21:05:27.799Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.531391696Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"":"io task","client":"1","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184682024-03-06T21:05:27.799Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.531448462Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"":"io task","client":"2","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184692024-03-06T21:05:27.799Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.532264965Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"client":"0","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184702024-03-06T21:05:27.799Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.532313792Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"client":"0","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184712024-03-06T21:05:27.799Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.53233972Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"client":"1","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184722024-03-06T21:05:27.799Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.532373558Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"client":"1","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184732024-03-06T21:05:27.799Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.532896362Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"client":"2","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184742024-03-06T21:05:27.799Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.532943949Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"client":"2","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184752024-03-06T21:05:27.799Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.533054721Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"client":"1","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184762024-03-06T21:05:27.799Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.533098578Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"client":"0","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184772024-03-06T21:05:27.799Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.53365341Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"client":"2","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184782024-03-06T21:05:27.799Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.533834537Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"client":"1","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184792024-03-06T21:05:27.799Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.533864725Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"client":"1","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184802024-03-06T21:05:27.799Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.533882414Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184812024-03-06T21:05:27.799Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.533905163Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"client":"0","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184822024-03-06T21:05:27.800Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.533919091Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"client":"0","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184832024-03-06T21:05:27.800Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.53393894Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184842024-03-06T21:05:27.800Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.534191613Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"client":"2","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184852024-03-06T21:05:27.800Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.53422857Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"client":"2","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184862024-03-06T21:05:27.800Z{"msg":"[0]R flush_numbers[0..12]: [290, 295, 297, 292, 287, 296, 300, 299, 292, 301, 301, 281]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.534245329Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"":"downstairs","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184872024-03-06T21:05:27.800Z{"msg":"[0]R generation[0..12]: [165, 167, 169, 166, 164, 168, 170, 170, 166, 171, 171, 162]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.534268207Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"":"downstairs","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184882024-03-06T21:05:27.800Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.534283646Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"":"downstairs","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184892024-03-06T21:05:27.800Z{"msg":"[1]R flush_numbers[0..12]: [290, 295, 297, 292, 287, 296, 300, 299, 292, 301, 301, 281]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.534304145Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"":"downstairs","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184902024-03-06T21:05:27.800Z{"msg":"[1]R generation[0..12]: [165, 167, 169, 166, 164, 168, 170, 170, 166, 171, 171, 162]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.534318564Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"":"downstairs","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184912024-03-06T21:05:27.800Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.534349382Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"":"downstairs","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184922024-03-06T21:05:27.800Z{"msg":"[2]R flush_numbers[0..12]: [290, 295, 297, 292, 287, 296, 300, 299, 292, 301, 301, 281]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
184932024-03-06T21:05:27.800Z,"time":"2024-03-06T21:05:26.5343747Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"":"downstairs","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184942024-03-06T21:05:27.800Z{"msg":"[2]R generation[0..12]: [165, 167, 169, 166, 164, 168, 170, 170, 166, 171, 171, 162]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.534408278Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"":"downstairs","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184952024-03-06T21:05:27.800Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.534441226Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"":"downstairs","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184962024-03-06T21:05:27.800Z{"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.534458784Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"":"downstairs","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184972024-03-06T21:05:27.800Z{"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.534479603Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"":"downstairs","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184982024-03-06T21:05:27.800Z{"msg":"Next flush: 302","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.534496362Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"":"downstairs","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
184992024-03-06T21:05:27.800Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.534529529Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"":"downstairs","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
185002024-03-06T21:05:27.800Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.534565157Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
185012024-03-06T21:05:27.800Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.534589115Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"client":"0","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
185022024-03-06T21:05:27.800Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.534603114Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"client":"1","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
185032024-03-06T21:05:27.800Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.534633492Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"client":"2","":"downstairs","session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
185042024-03-06T21:05:27.800Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.53466838Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
185052024-03-06T21:05:27.800Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.534694148Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
185062024-03-06T21:05:27.801Z{"msg":"17eb5fd6-4a89-4256-8b2b-b0a4f9c4278d is now active with session: fed9fb3b-beda-4428-9e4d-dc8f695bfa7b","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.534707467Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
185072024-03-06T21:05:27.801Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.534720176Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827,"session_id":"fed9fb3b-beda-4428-9e4d-dc8f695bfa7b"}
185082024-03-06T21:05:27.801Z{"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:26.534732905Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827}
185092024-03-06T21:05:27.804ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
185102024-03-06T21:05:27.805ZLoading write count information from file "/tmp/test_repair_verify.data"
185112024-03-06T21:05:27.805ZRun Repair workload
185122024-03-06T21:05:27.805Z01/30 Write block 193 len 20480 data: 13 12 14 16 15
185132024-03-06T21:05:27.805Z02/30 Read block 338 len 20480
185142024-03-06T21:05:27.805Z03/30 Flush
185152024-03-06T21:05:27.805Z04/30 Write block 21 len 24576 data: 17 19 18 15 14 14
185162024-03-06T21:05:27.805Z05/30 Write block 156 len 36864 data: 17 17 14 12 11 13 14 13 13
185172024-03-06T21:05:27.805Z06/30 Read block 439 len 8192
185182024-03-06T21:05:27.805Z07/30 Write block 200 len 8192 data: 15 12
185192024-03-06T21:05:27.808Z08/30 Read block 263 len 8192
185202024-03-06T21:05:27.808Z09/30 Read block 298 len 32768
185212024-03-06T21:05:27.808Z10/30 Read block 321 len 4096
185222024-03-06T21:05:27.808Z11/30 Flush
185232024-03-06T21:05:27.812Z12/30 Write block 591 len 16384 data: 16 16 11 12
185242024-03-06T21:05:27.812Z13/30 Write block 565 len 32768 data: 14 14 11 10 9 11 13 12
185252024-03-06T21:05:27.812Z14/30 Read block 271 len 40960
185262024-03-06T21:05:27.815Z15/30 Read block 135 len 12288
185272024-03-06T21:05:27.815Z16/30 Write block 454 len 32768 data: 12 12 12 16 15 17 17 19
185282024-03-06T21:05:27.818Z17/30 Write block 491 len 32768 data: 13 12 15 14 17 17 16 13
185292024-03-06T21:05:27.819Z18/30 Flush
185302024-03-06T21:05:27.822Z19/30 Write block 122 len 20480 data: 10 9 12 17 15
185312024-03-06T21:05:27.822Z20/30 Read block 90 len 16384
185322024-03-06T21:05:27.822Z21/30 Write block 17 len 24576 data: 18 17 18 18 18 20
185332024-03-06T21:05:27.849Z22/30 Read block 134 len 32768
185342024-03-06T21:05:27.849Z23/30 Write block 36 len 8192 data: 9 6
185352024-03-06T21:05:27.849Z24/30 Read block 310 len 16384
185362024-03-06T21:05:27.849Z25/30 Flush
185372024-03-06T21:05:27.849Z26/30 Write block 442 len 36864 data: 15 13 13 11 12 9 13 12 13
185382024-03-06T21:05:27.849Z27/30 Read block 1 len 20480
185392024-03-06T21:05:27.849Z28/30 Write block 571 len 40960 data: 14 13 12 11 12 14 10 9 11 10
185402024-03-06T21:05:27.849Z29/30 Read block 382 len 12288
185412024-03-06T21:05:27.849Z30/30 Write block 336 len 40960 data: 11 12 15 13 17 16 15 17 16 13
185422024-03-06T21:05:27.849Z----------------------------------------------------------------
185432024-03-06T21:05:27.849Z Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30
185442024-03-06T21:05:27.849ZStates: Active Active Active
185452024-03-06T21:05:27.849ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
185462024-03-06T21:05:27.849Z 1 Acked 1000 Write 5 Done Done Sent false
185472024-03-06T21:05:27.849Z 2 Acked 1001 Read 5 Done Done Sent false
185482024-03-06T21:05:27.850Z 3 Acked 1002 Flush 0 Done Done Sent false
185492024-03-06T21:05:27.850Z 4 Acked 1003 Write 6 Done Done Sent false
185502024-03-06T21:05:27.850Z 5 Acked 1004 Write 9 Done Done Sent false
185512024-03-06T21:05:27.850Z 6 Acked 1005 Read 2 Done Done Sent false
185522024-03-06T21:05:27.850Z 7 Acked 1006 Write 2 Done Done Sent false
185532024-03-06T21:05:27.850Z 8 Acked 1007 Read 2 Done Done Sent false
185542024-03-06T21:05:27.850Z 9 Acked 1008 Read 8 Done Done Sent false
185552024-03-06T21:05:27.850Z 10 Acked 1009 Read 1 Done Done Sent false
185562024-03-06T21:05:27.850Z 11 Acked 1010 Flush 0 Done Done Sent false
185572024-03-06T21:05:27.850Z 12 Acked 1011 Write 4 Done Done Sent false
185582024-03-06T21:05:27.850Z 13 Acked 1012 Write 8 Done Done Sent false
185592024-03-06T21:05:27.850Z 14 Acked 1013 Read 10 Done Done Sent false
185602024-03-06T21:05:27.850Z 15 Acked 1014 Read 3 Done Done Sent false
185612024-03-06T21:05:27.850Z 16 Acked 1015 Write 8 Done Done Sent false
185622024-03-06T21:05:27.850Z 17 Acked 1016 Write 8 Done Done Sent false
185632024-03-06T21:05:27.850Z 18 Acked 1017 Flush 0 Done Done Sent false
185642024-03-06T21:05:27.850Z 19 Acked 1018 Write 5 Done Done Sent false
185652024-03-06T21:05:27.850Z 20 Acked 1019 Read 4 Done Done Sent false
185662024-03-06T21:05:27.850Z 21 Acked 1020 Write 6 Done Done Sent false
185672024-03-06T21:05:27.850Z 22 Acked 1021 Read 8 Done Done Sent false
185682024-03-06T21:05:27.850Z 23 Acked 1022 Write 2 Done Done Sent false
185692024-03-06T21:05:27.850Z 24 Acked 1023 Read 4 Done Done Sent false
185702024-03-06T21:05:27.850Z 25 Acked 1024 Flush 0 Done Done Sent false
185712024-03-06T21:05:27.851Z 26 Acked 1025 Write 9 Done Done Sent false
185722024-03-06T21:05:27.851Z 27 Acked 1026 Read 5 Done Done Sent false
185732024-03-06T21:05:27.851Z 28 Acked 1027 Write 10 Done Done Sent false
185742024-03-06T21:05:27.851Z 29 Acked 1028 Read 3 Done Done Sent false
185752024-03-06T21:05:27.851Z 30 Acked 1029 Write 10 Sent Sent Sent false
185762024-03-06T21:05:27.851Z STATES DS:0 DS:1 DS:2 TOTAL
185772024-03-06T21:05:27.851Z New 0 0 0 0
185782024-03-06T21:05:27.851Z Sent 1 1 30 32
185792024-03-06T21:05:27.851Z Done 29 29 0 58
185802024-03-06T21:05:27.851Z Skipped 0 0 0 0
185812024-03-06T21:05:27.851Z Error 0 0 0 0
185822024-03-06T21:05:27.851ZLast Flush: 1024 1024 0
185832024-03-06T21:05:27.851ZDownstairs last five completed:
185842024-03-06T21:05:27.851ZUpstairs last five completed: 30 29 28 27 26
185852024-03-06T21:05:27.851Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:05:26.573496234Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1827}
185862024-03-06T21:05:27.851ZWrote out file "/tmp/test_repair_verify.data"
185872024-03-06T21:05:27.851Z
185882024-03-06T21:05:27.851ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
185892024-03-06T21:05:27.861ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
185902024-03-06T21:05:27.861Z 0 000-019 172 172 165 305 305 290 F F F <---
185912024-03-06T21:05:27.861Z 1 020-039 172 172 167 305 305 295 F F F <---
185922024-03-06T21:05:27.861Z 2 040-059 169 169 169 297 297 297 F F F
185932024-03-06T21:05:27.861Z 3 060-079 166 166 166 292 292 292 F F F
185942024-03-06T21:05:27.861Z 4 080-099 164 164 164 287 287 287 F F F
185952024-03-06T21:05:27.861Z 5 100-119 168 168 168 296 296 296 F F F
185962024-03-06T21:05:27.861Z 6 120-139 172 172 170 305 305 300 F F F <---
185972024-03-06T21:05:27.861Z 7 140-159 172 172 170 303 303 299 F F F <---
185982024-03-06T21:05:27.861Z 8 160-179 172 172 166 303 303 292 F F F <---
185992024-03-06T21:05:27.861Z 9 180-199 172 172 171 302 302 301 F F F <---
186002024-03-06T21:05:27.861Z 10 200-219 172 172 171 303 303 301 F F F <---
186012024-03-06T21:05:27.869Z 11 220-239 162 162 162 281 281 281 F F F
186022024-03-06T21:05:27.869Z 12 240-259 170 170 170 300 300 300 F F F
186032024-03-06T21:05:27.869Z 13 260-279 171 171 171 301 301 301 F F F
186042024-03-06T21:05:27.869Z 14 280-299 171 171 171 301 301 301 F F F
186052024-03-06T21:05:27.870Z 15 300-319 171 171 171 301 301 301 F F F
186062024-03-06T21:05:27.870Z 16 320-339 169 169 169 297 297 297 T T F <---
186072024-03-06T21:05:27.870Z 17 340-359 170 170 170 300 300 300 T T F <---
186082024-03-06T21:05:27.870Z 18 360-379 171 171 171 301 301 301 F F F
186092024-03-06T21:05:27.870Z 19 380-399 171 171 171 301 301 301 F F F
186102024-03-06T21:05:27.870Z 20 400-419 168 168 168 296 296 296 F F F
186112024-03-06T21:05:27.870Z 21 420-439 166 166 166 292 292 292 F F F
186122024-03-06T21:05:27.870Z 22 440-459 172 172 164 304 304 288 T T F <---
186132024-03-06T21:05:27.870Z 23 460-479 172 172 166 304 304 292 F F F <---
186142024-03-06T21:05:27.871Z 24 480-499 172 172 171 304 304 301 F F F <---
186152024-03-06T21:05:27.871Z 25 500-519 171 171 171 301 301 301 F F F
186162024-03-06T21:05:27.871Z 26 520-539 168 168 168 296 296 296 F F F
186172024-03-06T21:05:27.871Z 27 540-559 170 170 170 298 298 298 F F F
186182024-03-06T21:05:27.871Z 28 560-579 172 172 164 304 304 287 T T F <---
186192024-03-06T21:05:27.871Z 29 580-599 172 172 170 304 304 298 T T F <---
186202024-03-06T21:05:27.872ZMax gen: 172, Max flush: 305
186212024-03-06T21:05:27.872ZError: Difference in extent metadata found!
186222024-03-06T21:05:27.883ZA Difference in extent metadata is expected here
186232024-03-06T21:05:27.883ZOn loop 86
186242024-03-06T21:05:27.883Z
186252024-03-06T21:05:27.883ZVerifying data now
186262024-03-06T21:05:32.600ZNo change after verify
186272024-03-06T21:05:32.600ZLoop: 86 Downstairs dump after verify (and repair):
186282024-03-06T21:05:32.616ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
186292024-03-06T21:05:32.616Z 0 000-019 172 172 172 305 305 305 F F F
186302024-03-06T21:05:32.616Z 1 020-039 172 172 172 305 305 305 F F F
186312024-03-06T21:05:32.616Z 2 040-059 169 169 169 297 297 297 F F F
186322024-03-06T21:05:32.616Z 3 060-079 166 166 166 292 292 292 F F F
186332024-03-06T21:05:32.616Z 4 080-099 164 164 164 287 287 287 F F F
186342024-03-06T21:05:32.616Z 5 100-119 168 168 168 296 296 296 F F F
186352024-03-06T21:05:32.616Z 6 120-139 172 172 172 305 305 305 F F F
186362024-03-06T21:05:32.616Z 7 140-159 172 172 172 303 303 303 F F F
186372024-03-06T21:05:32.617Z 8 160-179 172 172 172 303 303 303 F F F
186382024-03-06T21:05:32.617Z 9 180-199 172 172 172 302 302 302 F F F
186392024-03-06T21:05:32.617Z 10 200-219 172 172 172 303 303 303 F F F
186402024-03-06T21:05:32.617Z 11 220-239 162 162 162 281 281 281 F F F
186412024-03-06T21:05:32.617Z 12 240-259 170 170 170 300 300 300 F F F
186422024-03-06T21:05:32.617Z 13 260-279 171 171 171 301 301 301 F F F
186432024-03-06T21:05:32.617Z 14 280-299 171 171 171 301 301 301 F F F
186442024-03-06T21:05:32.617Z 15 300-319 171 171 171 301 301 301 F F F
186452024-03-06T21:05:32.617Z 16 320-339 173 173 173 306 306 306 F F F
186462024-03-06T21:05:32.617Z 17 340-359 173 173 173 306 306 306 F F F
186472024-03-06T21:05:32.617Z 18 360-379 171 171 171 301 301 301 F F F
186482024-03-06T21:05:32.617Z 19 380-399 171 171 171 301 301 301 F F F
186492024-03-06T21:05:32.617Z 20 400-419 168 168 168 296 296 296 F F F
186502024-03-06T21:05:32.622Z 21 420-439 166 166 166 292 292 292 F F F
186512024-03-06T21:05:32.622Z 22 440-459 173 173 173 306 306 306 F F F
186522024-03-06T21:05:32.622Z 23 460-479 172 172 172 304 304 304 F F F
186532024-03-06T21:05:32.622Z 24 480-499 172 172 172 304 304 304 F F F
186542024-03-06T21:05:32.622Z 25 500-519 171 171 171 301 301 301 F F F
186552024-03-06T21:05:32.622Z 26 520-539 168 168 168 296 296 296 F F F
186562024-03-06T21:05:32.622Z 27 540-559 170 170 170 298 298 298 F F F
186572024-03-06T21:05:32.622Z 28 560-579 173 173 173 306 306 306 F F F
186582024-03-06T21:05:32.622Z 29 580-599 173 173 173 306 306 306 F F F
186592024-03-06T21:05:32.622ZMax gen: 173, Max flush: 306
186602024-03-06T21:05:32.622Z
186612024-03-06T21:05:32.623ZBegin loop 87
186622024-03-06T21:05:32.623ZDownstairs to restart: 2
186632024-03-06T21:05:32.643Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.38209473Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835}
186642024-03-06T21:05:32.643Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.382433717Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835}
186652024-03-06T21:05:32.653Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.382480403Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835}
186662024-03-06T21:05:32.653Z{"msg":"Crucible d8d3742f-8e5e-4abc-9ad3-0e152581694f has session id: a53f39f7-31b7-4337-ba1c-63cc8a26c022","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.382505552Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186672024-03-06T21:05:32.653Z{"msg":"Upstairs opts: Upstairs UUID: d8d3742f-8e5e-4abc-9ad3-0e152581694f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.382542239Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186682024-03-06T21:05:32.653Z{"msg":"Crucible stats registered with UUID: d8d3742f-8e5e-4abc-9ad3-0e152581694f","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.382573687Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186692024-03-06T21:05:32.653ZCrucible runtime is spawned
186702024-03-06T21:05:32.654Z{"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.382768114Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835}
186712024-03-06T21:05:32.654Z{"msg":"d8d3742f-8e5e-4abc-9ad3-0e152581694f active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.382860637Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186722024-03-06T21:05:32.654Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.382882455Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"client":"0","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186732024-03-06T21:05:32.654Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.382899264Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"client":"0","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186742024-03-06T21:05:32.654Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.382929972Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"client":"1","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186752024-03-06T21:05:32.654Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.382952521Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"client":"1","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186762024-03-06T21:05:32.654Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.382973589Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"client":"2","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186772024-03-06T21:05:32.654Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.383006377Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"client":"2","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186782024-03-06T21:05:32.654Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.383026476Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"":"io task","client":"2","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186792024-03-06T21:05:32.654Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.383056963Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"":"io task","client":"0","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186802024-03-06T21:05:32.654Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.383072693Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"":"io task","client":"1","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186812024-03-06T21:05:32.654Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.38325526Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"":"io task","client":"1","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186822024-03-06T21:05:32.654Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.383336984Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"":"io task","client":"2","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186832024-03-06T21:05:32.654Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.383422398Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"":"io task","client":"0","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186842024-03-06T21:05:32.655Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.384156078Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"client":"0","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186852024-03-06T21:05:32.655Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.384194185Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"client":"0","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186862024-03-06T21:05:32.655Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.384210514Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"client":"1","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186872024-03-06T21:05:32.655Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.384229893Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"client":"1","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186882024-03-06T21:05:32.655Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.384819642Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"client":"1","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186892024-03-06T21:05:32.655Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.384952943Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"client":"2","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186902024-03-06T21:05:32.655Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.38498985Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"client":"2","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186912024-03-06T21:05:32.655Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.385019248Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"client":"0","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186922024-03-06T21:05:32.655Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.385415411Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"client":"1","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186932024-03-06T21:05:32.655Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.385483426Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"client":"1","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186942024-03-06T21:05:32.655Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.385524713Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186952024-03-06T21:05:32.655Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.385549652Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"client":"0","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186962024-03-06T21:05:32.655Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.385585069Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"client":"0","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186972024-03-06T21:05:32.655Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.385607088Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186982024-03-06T21:05:32.655Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.385738709Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"client":"2","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
186992024-03-06T21:05:32.655Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.38615713Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"client":"2","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
187002024-03-06T21:05:32.655Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.386209076Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"client":"2","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
187012024-03-06T21:05:32.656Z{"msg":"[0]R flush_numbers[0..12]: [305, 305, 297, 292, 287, 296, 305, 303, 303, 302, 303, 281]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.386227195Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"":"downstairs","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
187022024-03-06T21:05:32.656Z{"msg":"[0]R generation[0..12]: [172, 172, 169, 166, 164, 168, 172, 172, 172, 172, 172, 162]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.386250863Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"":"downstairs","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
187032024-03-06T21:05:32.656Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.386266442Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"":"downstairs","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
187042024-03-06T21:05:32.656Z{"msg":"[1]R flush_numbers[0..12]: [305, 305, 297, 292, 287, 296, 305, 303, 303, 302, 303, 281]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.386288051Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"":"downstairs","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
187052024-03-06T21:05:32.656Z{"msg":"[1]R generation[0..12]: [172, 172, 169, 166, 164, 168, 172, 172, 172, 172, 172, 162]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.38630359Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"":"downstairs","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
187062024-03-06T21:05:32.656Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.386318409Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"":"downstairs","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
187072024-03-06T21:05:32.656Z{"msg":"[2]R flush_numbers[0..12]: [305, 305, 297, 292, 287, 296, 305, 303, 303, 302, 303, 281]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.386333588Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"":"downstairs","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
187082024-03-06T21:05:32.656Z{"msg":"[2]R generation[0..12]: [172, 172, 169, 166, 164, 168, 172, 172, 172, 172, 172, 162]","v":0,"Wait for a query_work_queue command to finish before sending IO
187092024-03-06T21:05:32.656Zname":"crucible","level":30,"time":"2024-03-06T21:05:31.386358036Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"":"downstairs","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
187102024-03-06T21:05:32.656Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.386373025Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"":"downstairs","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
187112024-03-06T21:05:32.656Z{"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.386387174Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"":"downstairs","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
187122024-03-06T21:05:32.656Z{"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.386408143Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"":"downstairs","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
187132024-03-06T21:05:32.656Z{"msg":"Next flush: 307","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.386421942Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"":"downstairs","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
187142024-03-06T21:05:32.656Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.386435391Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"":"downstairs","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
187152024-03-06T21:05:32.656Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.386464259Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
187162024-03-06T21:05:32.657Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.386487697Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"client":"0","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
187172024-03-06T21:05:32.657Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.386520915Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"client":"1","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
187182024-03-06T21:05:32.657Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.386543773Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"client":"2","":"downstairs","session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
187192024-03-06T21:05:32.657Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.386578511Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
187202024-03-06T21:05:32.657Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.38659468Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
187212024-03-06T21:05:32.657Z{"msg":"d8d3742f-8e5e-4abc-9ad3-0e152581694f is now active with session: a53f39f7-31b7-4337-ba1c-63cc8a26c022","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.386627348Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
187222024-03-06T21:05:32.657Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.386648126Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835,"session_id":"a53f39f7-31b7-4337-ba1c-63cc8a26c022"}
187232024-03-06T21:05:32.657ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
187242024-03-06T21:05:32.657Z{"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:31.386677024Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835}
187252024-03-06T21:05:32.663ZLoading write count information from file "/tmp/test_repair_verify.data"
187262024-03-06T21:05:32.663ZRun Repair workload
187272024-03-06T21:05:32.663Z01/30 Write block 0 len 36864 data: 3 4 8 10 13 13 13 14 13
187282024-03-06T21:05:32.664Z02/30 Read block 569 len 40960
187292024-03-06T21:05:32.664Z03/30 Write block 285 len 32768 data: 18 20 20 20 18 16 15 12
187302024-03-06T21:05:32.664Z04/30 Write block 65 len 40960 data: 20 19 18 17 15 11 10 13 12 12
187312024-03-06T21:05:32.664Z05/30 Read block 407 len 40960
187322024-03-06T21:05:32.664Z06/30 Read block 3 len 28672
187332024-03-06T21:05:32.664Z07/30 Write block 35 len 8192 data: 12 10
187342024-03-06T21:05:32.664Z08/30 Write block 232 len 8192 data: 13 12
187352024-03-06T21:05:32.665Z09/30 Write block 94 len 40960 data: 10 10 13 11 13 13 10 12 12 13
187362024-03-06T21:05:32.665Z10/30 Read block 562 len 24576
187372024-03-06T21:05:32.665Z11/30 Flush
187382024-03-06T21:05:32.695Z12/30 Write block 429 len 24576 data: 13 13 11 11 12 15
187392024-03-06T21:05:32.695Z13/30 Write block 514 len 12288 data: 16 18 17
187402024-03-06T21:05:32.695Z14/30 Read block 191 len 20480
187412024-03-06T21:05:32.695Z15/30 Write block 169 len 16384 data: 13 12 12 14
187422024-03-06T21:05:32.695Z16/30 Write block 553 len 36864 data: 6 7 7 6 8 8 8 12 12
187432024-03-06T21:05:32.695Z17/30 Read block 90 len 24576
187442024-03-06T21:05:32.695Z18/30 Read block 472 len 20480
187452024-03-06T21:05:32.705Z19/30 Write block 77 len 36864 data: 7 10 12 11 11 11 11 8 8
187462024-03-06T21:05:32.705Z20/30 Read block 3 len 8192
187472024-03-06T21:05:32.705Z21/30 Write block 377 len 28672 data: 12 12 17 18 18 20 19
187482024-03-06T21:05:32.705Z22/30 Write block 289 len 16384 data: 19 17 16 13
187492024-03-06T21:05:32.705Z23/30 Read block 339 len 28672
187502024-03-06T21:05:32.706Z24/30 Write block 57 len 12288 data: 10 13 16
187512024-03-06T21:05:32.706Z25/30 Read block 560 len 40960
187522024-03-06T21:05:32.706Z26/30 Read block 459 len 28672
187532024-03-06T21:05:32.706Z27/30 Write block 135 len 4096 data: 11
187542024-03-06T21:05:32.706Z28/30 Write block 47 len 20480 data: 13 12 10 6 8
187552024-03-06T21:05:32.706Z29/30 Read block 542 len 16384
187562024-03-06T21:05:32.706Z30/30 Read block 506 len 4096
187572024-03-06T21:05:32.706Z----------------------------------------------------------------
187582024-03-06T21:05:32.707Z Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:30
187592024-03-06T21:05:32.707ZStates: Active Active Active
187602024-03-06T21:05:32.707ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
187612024-03-06T21:05:32.707Z 1 Acked 1000 Write 9 Done Done Done false
187622024-03-06T21:05:32.707Z 2 Acked 1001 Read 10 Done Done Done false
187632024-03-06T21:05:32.707Z 3 Acked 1002 Write 8 Done Done Sent false
187642024-03-06T21:05:32.707Z 4 Acked 1003 Write 10 Done Done Sent false
187652024-03-06T21:05:32.707Z 5 Acked 1004 Read 10 Done Done Sent false
187662024-03-06T21:05:32.708Z 6 Acked 1005 Read 7 Done Done Sent false
187672024-03-06T21:05:32.708Z 7 Acked 1006 Write 2 Done Done Sent false
187682024-03-06T21:05:32.708Z 8 Acked 1007 Write 2 Done Done Sent false
187692024-03-06T21:05:32.708Z 9 Acked 1008 Write 10 Done Done Sent false
187702024-03-06T21:05:32.708Z 10 Acked 1009 Read 6 Done Done Sent false
187712024-03-06T21:05:32.708Z 11 Acked 1010 Flush 0 Done Done Sent false
187722024-03-06T21:05:32.708Z 12 Acked 1011 Write 6 Done Done Sent false
187732024-03-06T21:05:32.709Z 13 Acked 1012 Write 3 Done Done Sent false
187742024-03-06T21:05:32.709Z 14 Acked 1013 Read 5 Done Done Sent false
187752024-03-06T21:05:32.709Z 15 Acked 1014 Write 4 Done Done Sent false
187762024-03-06T21:05:32.709Z 16 Acked 1015 Write 9 Done Done Sent false
187772024-03-06T21:05:32.709Z 17 Acked 1016 Read 6 Done Done Sent false
187782024-03-06T21:05:32.709Z 18 Acked 1017 Read 5 Done Done Sent false
187792024-03-06T21:05:32.709Z 19 Acked 1018 Write 9 Done Done Sent false
187802024-03-06T21:05:32.709Z 20 Acked 1019 Read 2 Done Done Sent false
187812024-03-06T21:05:32.710Z 21 Acked 1020 Write 7 Done Done Sent false
187822024-03-06T21:05:32.710Z 22 Acked 1021 Write 4 Done Done Sent false
187832024-03-06T21:05:32.710Z 23 Acked 1022 Read 7 Done Done Sent false
187842024-03-06T21:05:32.710Z 24 Acked 1023 Write 3 Done Done Sent false
187852024-03-06T21:05:32.710Z 25 Acked 1024 Read 10 Done Done Sent false
187862024-03-06T21:05:32.710Z 26 Acked 1025 Read 7 Done Done Sent false
187872024-03-06T21:05:32.710Z 27 Acked 1026 Write 1 Done Done Sent false
187882024-03-06T21:05:32.710Z 28 Acked 1027 Write 5 Done Done Sent false
187892024-03-06T21:05:32.710Z 29 Acked 1028 Read 4 Done Done Sent false
187902024-03-06T21:05:32.710Z 30 Acked 1029 Read 1 Done Done Sent false
187912024-03-06T21:05:32.710Z STATES DS:0 DS:1 DS:2 TOTAL
187922024-03-06T21:05:32.710Z New 0 0 0 0
187932024-03-06T21:05:32.710Z Sent 0 0 28 28
187942024-03-06T21:05:32.711Z Done 30 30 2 62
187952024-03-06T21:05:32.711Z Skipped 0 0 0 0
187962024-03-06T21:05:32.711Z Error 0 0 0 0
187972024-03-06T21:05:32.711ZLast Flush: 1010 1010 0
187982024-03-06T21:05:32.711ZDownstairs last five completed:
187992024-03-06T21:05:32.711ZUpstairs last five completed: 30 29 28 27 26
188002024-03-06T21:05:32.711Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:05:31.421443938Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1835}
188012024-03-06T21:05:32.711ZWrote out file "/tmp/test_repair_verify.data"
188022024-03-06T21:05:32.711Z
188032024-03-06T21:05:32.711ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
188042024-03-06T21:05:32.717ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
188052024-03-06T21:05:32.717Z 0 000-019 174 174 172 307 307 305 F F T <---
188062024-03-06T21:05:32.717Z 1 020-039 174 174 172 307 307 305 F F F <---
188072024-03-06T21:05:32.717Z 2 040-059 169 169 169 297 297 297 T T F <---
188082024-03-06T21:05:32.717Z 3 060-079 174 174 166 307 307 292 T T F <---
188092024-03-06T21:05:32.717Z 4 080-099 174 174 164 307 307 287 T T F <---
188102024-03-06T21:05:32.717Z 5 100-119 174 174 168 307 307 296 F F F <---
188112024-03-06T21:05:32.717Z 6 120-139 172 172 172 305 305 305 T T F <---
188122024-03-06T21:05:32.717Z 7 140-159 172 172 172 303 303 303 F F F
188132024-03-06T21:05:32.717Z 8 160-179 172 172 172 303 303 303 T T F <---
188142024-03-06T21:05:32.717Z 9 180-199 172 172 172 302 302 302 F F F
188152024-03-06T21:05:32.718Z 10 200-219 172 172 172 303 303 303 F F F
188162024-03-06T21:05:32.718Z 11 220-239 174 174 162 307 307 281 F F F <---
188172024-03-06T21:05:32.718Z 12 240-259 170 170 170 300 300 300 F F F
188182024-03-06T21:05:32.718Z 13 260-279 171 171 171 301 301 301 F F F
188192024-03-06T21:05:32.718Z 14 280-299 174 174 171 307 307 301 T T F <---
188202024-03-06T21:05:32.718Z 15 300-319 171 171 171 301 301 301 F F F
188212024-03-06T21:05:32.718Z 16 320-339 173 173 173 306 306 306 F F F
188222024-03-06T21:05:32.718Z 17 340-359 173 173 173 306 306 306 F F F
188232024-03-06T21:05:32.718Z 18 360-379 171 171 171 301 301 301 T T F <---
188242024-03-06T21:05:32.718Z 19 380-399 171 171 171 301 301 301 T T F <---
188252024-03-06T21:05:32.718Z 20 400-419 168 168 168 296 296 296 F F F
188262024-03-06T21:05:32.718Z 21 420-439 166 166 166 292 292 292 T T F <---
188272024-03-06T21:05:32.718Z 22 440-459 173 173 173 306 306 306 F F F
188282024-03-06T21:05:32.719Z 23 460-479 172 172 172 304 304 304 F F F
188292024-03-06T21:05:32.719Z 24 480-499 172 172 172 304 304 304 F F F
188302024-03-06T21:05:32.719Z 25 500-519 171 171 171 301 301 301 T T F <---
188312024-03-06T21:05:32.719Z 26 520-539 168 168 168 296 296 296 F F F
188322024-03-06T21:05:32.719Z 27 540-559 170 170 170 298 298 298 T T F <---
188332024-03-06T21:05:32.719Z 28 560-579 173 173 173 306 306 306 T T F <---
188342024-03-06T21:05:32.719Z 29 580-599 173 173 173 306 306 306 F F F
188352024-03-06T21:05:32.719ZMax gen: 174, Max flush: 307
188362024-03-06T21:05:32.719ZError: Difference in extent metadata found!
188372024-03-06T21:05:32.723ZA Difference in extent metadata is expected here
188382024-03-06T21:05:32.723ZOn loop 87
188392024-03-06T21:05:32.723Z
188402024-03-06T21:05:32.724ZVerifying data now
188412024-03-06T21:05:37.561ZNo change after verify
188422024-03-06T21:05:37.561ZLoop: 87 Downstairs dump after verify (and repair):
188432024-03-06T21:05:37.583ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
188442024-03-06T21:05:37.583Z 0 000-019 174 174 174 307 307 307 F F F
188452024-03-06T21:05:37.584Z 1 020-039 174 174 174 307 307 307 F F F
188462024-03-06T21:05:37.584Z 2 040-059 175 175 175 308 308 308 F F F
188472024-03-06T21:05:37.584Z 3 060-079 175 175 175 308 308 308 F F F
188482024-03-06T21:05:37.584Z 4 080-099 175 175 175 308 308 308 F F F
188492024-03-06T21:05:37.584Z 5 100-119 174 174 174 307 307 307 F F F
188502024-03-06T21:05:37.584Z 6 120-139 175 175 175 308 308 308 F F F
188512024-03-06T21:05:37.584Z 7 140-159 172 172 172 303 303 303 F F F
188522024-03-06T21:05:37.584Z 8 160-179 175 175 175 308 308 308 F F F
188532024-03-06T21:05:38.569Z 9 180-199 172 172 172 302 302 302 F F F
188542024-03-06T21:05:38.570Z 10 200-219 172 172 172 303 303 303 F F F
188552024-03-06T21:05:38.570Z 11 220-239 174 174 174 307 307 307 F F F
188562024-03-06T21:05:38.570Z 12 240-259 170 170 170 300 300 300 F F F
188572024-03-06T21:05:38.570Z 13 260-279 171 171 171 301 301 301 F F F
188582024-03-06T21:05:38.570Z 14 280-299 175 175 175 308 308 308 F F F
188592024-03-06T21:05:38.570Z 15 300-319 171 171 171 301 301 301 F F F
188602024-03-06T21:05:38.570Z 16 320-339 173 173 173 306 306 306 F F F
188612024-03-06T21:05:38.570Z 17 340-359 173 173 173 306 306 306 F F F
188622024-03-06T21:05:38.570Z 18 360-379 175 175 175 308 308 308 F F F
188632024-03-06T21:05:38.570Z 19 380-399 175 175 175 308 308 308 F F F
188642024-03-06T21:05:38.570Z 20 400-419 168 168 168 296 296 296 F F F
188652024-03-06T21:05:38.570Z 21 420-439 175 175 175 308 308 308 F F F
188662024-03-06T21:05:38.570Z 22 440-459 173 173 173 306 306 306 F F F
188672024-03-06T21:05:38.571Z 23 460-479 172 172 172 304 304 304 F F F
188682024-03-06T21:05:38.571Z 24 480-499 172 172 172 304 304 304 F F F
188692024-03-06T21:05:38.571Z 25 500-519 175 175 175 308 308 308 F F F
188702024-03-06T21:05:38.571Z 26 520-539 168 168 168 296 296 296 F F F
188712024-03-06T21:05:38.571Z 27 540-559 175 175 175 308 308 308 F F F
188722024-03-06T21:05:38.571Z 28 560-579 175 175 175 308 308 308 F F F
188732024-03-06T21:05:38.571Z 29 580-599 173 173 173 306 306 306 F F F
188742024-03-06T21:05:38.571ZMax gen: 175, Max flush: 308
188752024-03-06T21:05:38.571Z
188762024-03-06T21:05:38.571ZBegin loop 88
188772024-03-06T21:05:38.571ZDownstairs to restart: 2
188782024-03-06T21:05:38.571Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.341364122Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843}
188792024-03-06T21:05:38.571Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.341653412Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843}
188802024-03-06T21:05:38.572Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.34167736Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843}
188812024-03-06T21:05:38.572Z{"msg":"Crucible 07efb96b-1334-4687-bf50-5525423a7dcf has session id: 8df5a236-fa63-4c7c-881e-8d6283a89b78","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.341729317Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
188822024-03-06T21:05:38.572Z{"msg":"Upstairs opts: Upstairs UUID: 07efb96b-1334-4687-bf50-5525423a7dcf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.341789833Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
188832024-03-06T21:05:38.572Z{"msg":"Crucible stats registered with UUID: 07efb96b-1334-4687-bf50-5525423a7dcf","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.34181676Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
188842024-03-06T21:05:38.572ZCrucible runtime is spawned
188852024-03-06T21:05:38.572Z{"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.341955011Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843}
188862024-03-06T21:05:38.572Z{"msg":"07efb96b-1334-4687-bf50-5525423a7dcf active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.342088262Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
188872024-03-06T21:05:38.572Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.342128599Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"client":"0","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
188882024-03-06T21:05:38.572Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.342175996Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"client":"0","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
188892024-03-06T21:05:38.572Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.342236342Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"client":"1","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
188902024-03-06T21:05:38.572Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.342347254Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"client":"1","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
188912024-03-06T21:05:38.572Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.342385601Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"client":"2","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
188922024-03-06T21:05:38.572Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.342412039Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"client":"2","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
188932024-03-06T21:05:38.572Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.342448247Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"":"io task","client":"2","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
188942024-03-06T21:05:38.572Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.342590897Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"":"io task","client":"1","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
188952024-03-06T21:05:38.572Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.342655203Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"":"io task","client":"0","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
188962024-03-06T21:05:38.572Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.34269373Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"":"io task","client":"2","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
188972024-03-06T21:05:38.573Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.342746556Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"":"io task","client":"0","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
188982024-03-06T21:05:38.573Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.342787714Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"":"io task","client":"1","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
188992024-03-06T21:05:38.573Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.343318257Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"client":"0","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189002024-03-06T21:05:38.573Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.343354685Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"client":"0","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189012024-03-06T21:05:38.573Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.343371923Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"client":"1","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189022024-03-06T21:05:38.573Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.343393752Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"client":"1","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189032024-03-06T21:05:38.573Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.343968622Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"client":"1","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189042024-03-06T21:05:38.573Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.3441462Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"client":"0","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189052024-03-06T21:05:38.573Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.344531904Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"client":"1","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189062024-03-06T21:05:38.573Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.344590669Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"client":"1","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189072024-03-06T21:05:38.573Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.344616758Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189082024-03-06T21:05:38.573Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.344638386Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"client":"0","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189092024-03-06T21:05:38.573Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.344675814Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"client":"0","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189102024-03-06T21:05:38.573Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.344702602Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189112024-03-06T21:05:38.573Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.34472798Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"client":"2","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189122024-03-06T21:05:38.573Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.344742469Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"client":"2","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189132024-03-06T21:05:38.573Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.345222296Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"client":"2","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189142024-03-06T21:05:38.573Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.345671215Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"client":"2","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189152024-03-06T21:05:38.573Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.345727441Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"client":"2","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189162024-03-06T21:05:38.573Z{"msg":"[0]R flush_numbers[0..12]: [307, 307, 308, 308, 308, 307, 308, 303, 308, 302, 303, 307]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.345759609Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"":"downstairs","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189172024-03-06T21:05:38.573Z{"msg":"[0]R generation[0..12]: [174, 174, 175, 175, 175, 174, 175, 172, 175, 172, 172, 174]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.345786067Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"":"downstairs","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189182024-03-06T21:05:38.573Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.345804586Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"":"downstairs","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189192024-03-06T21:05:38.573Z{"msg":"[1]R flush_numbers[0..12]: [307, 307, 308, 308, 308, 307, 308, 303, 308, 302, 303, 307]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.345830174Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"":"downstairs","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189202024-03-06T21:05:38.573Z{"msg":"[1]R generation[0..12]: [174, 174, 175, 175, 175, 174, 175, 172, 175, 172, 172, 174]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.345849533Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"":"downstairs","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189212024-03-06T21:05:38.574Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.345867471Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"":"downstairs","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189222024-03-06T21:05:38.574Z{"msg":"[2]R flush_numbers[0..12]: [307, 307, 308, 308, 308, 307, 308, 303, 308, 302, 303, 307]","v":0,"Wait for a query_work_queue command to finish before sending IO
189232024-03-06T21:05:38.574Zname":"crucible","level":30,"time":"2024-03-06T21:05:36.34589567Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"":"downstairs","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189242024-03-06T21:05:38.574Z{"msg":"[2]R generation[0..12]: [174, 174, 175, 175, 175, 174, 175, 172, 175, 172, 172, 174]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.345926957Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"":"downstairs","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189252024-03-06T21:05:38.574Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.345958245Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"":"downstairs","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189262024-03-06T21:05:38.574Z{"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.345986873Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"":"downstairs","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189272024-03-06T21:05:38.574Z{"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.346015821Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"":"downstairs","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189282024-03-06T21:05:38.574Z{"msg":"Next flush: 309","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.34603312Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"":"downstairs","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189292024-03-06T21:05:38.574Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.346064268Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"":"downstairs","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189302024-03-06T21:05:38.574Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.346081057Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189312024-03-06T21:05:38.574Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.346106665Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"client":"0","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189322024-03-06T21:05:38.574Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.346124414Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"client":"1","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189332024-03-06T21:05:38.574Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.346149042Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"client":"2","":"downstairs","session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189342024-03-06T21:05:38.574Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.346167271Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189352024-03-06T21:05:38.574Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.346191059Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189362024-03-06T21:05:38.574Z{"msg":"07efb96b-1334-4687-bf50-5525423a7dcf is now active with session: 8df5a236-fa63-4c7c-881e-8d6283a89b78","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.346207598Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189372024-03-06T21:05:38.574Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
189382024-03-06T21:05:38.574Z,"time":"2024-03-06T21:05:36.346230566Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843,"session_id":"8df5a236-fa63-4c7c-881e-8d6283a89b78"}
189392024-03-06T21:05:38.574Z{"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:36.346250135Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843}
189402024-03-06T21:05:38.575ZLoading write count information from file "/tmp/test_repair_verify.data"
189412024-03-06T21:05:38.575ZRun Repair workload
189422024-03-06T21:05:38.575Z01/30 Write block 451 len 16384 data: 15 13 11 13
189432024-03-06T21:05:38.575Z02/30 Read block 516 len 24576
189442024-03-06T21:05:38.575Z03/30 Write block 149 len 28672 data: 15 15 14 11 11 12 14
189452024-03-06T21:05:38.575Z04/30 Read block 268 len 12288
189462024-03-06T21:05:38.575Z05/30 Read block 34 len 16384
189472024-03-06T21:05:38.575Z06/30 Write block 403 len 40960 data: 10 8 12 12 15 16 14 15 13 10
189482024-03-06T21:05:38.575Z07/30 Write block 413 len 28672 data: 13 12 11 14 14 14 13
189492024-03-06T21:05:38.575Z08/30 Read block 449 len 12288
189502024-03-06T21:05:38.575Z09/30 Read block 589 len 8192
189512024-03-06T21:05:38.575Z10/30 Read block 577 len 40960
189522024-03-06T21:05:38.575Z11/30 Read block 47 len 8192
189532024-03-06T21:05:38.632Z12/30 Read block 464 len 12288
189542024-03-06T21:05:38.632Z13/30 Read block 448 len 12288
189552024-03-06T21:05:38.632Z14/30 Read block 20 len 4096
189562024-03-06T21:05:38.632Z15/30 Write block 372 len 36864 data: 14 15 14 18 14 13 13 18 19
189572024-03-06T21:05:38.632Z16/30 Write block 541 len 28672 data: 12 15 17 19 20 19 17
189582024-03-06T21:05:38.632Z17/30 Read block 86 len 32768
189592024-03-06T21:05:38.632Z18/30 Read block 503 len 12288
189602024-03-06T21:05:38.633Z19/30 Write block 129 len 20480 data: 15 11 9 9 11
189612024-03-06T21:05:38.633Z20/30 Write block 379 len 12288 data: 19 20 19
189622024-03-06T21:05:38.633Z21/30 Read block 555 len 24576
189632024-03-06T21:05:38.633Z22/30 Write block 401 len 16384 data: 8 8 11 9
189642024-03-06T21:05:38.633Z23/30 Write block 360 len 28672 data: 16 20 16 19 16 14 12
189652024-03-06T21:05:38.633Z24/30 Read block 418 len 40960
189662024-03-06T21:05:38.633Z25/30 Read block 147 len 28672
189672024-03-06T21:05:38.633Z26/30 Write block 434 len 12288 data: 16 15 15
189682024-03-06T21:05:38.633Z27/30 Write block 316 len 28672 data: 11 9 13 13 12 15 14
189692024-03-06T21:05:38.633Z28/30 Read block 68 len 28672
189702024-03-06T21:05:38.633Z29/30 Write block 488 len 28672 data: 14 16 12 14 13 16 15
189712024-03-06T21:05:38.634Z30/30 Write block 466 len 16384 data: 16 16 15 12
189722024-03-06T21:05:38.634Z----------------------------------------------------------------
189732024-03-06T21:05:38.634Z Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:30
189742024-03-06T21:05:38.634ZStates: Active Active Active
189752024-03-06T21:05:38.634ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
189762024-03-06T21:05:38.634Z 1 Acked 1000 Write 4 Done Done Done false
189772024-03-06T21:05:38.634Z 2 Acked 1001 Read 6 Done Done Done false
189782024-03-06T21:05:38.634Z 3 Acked 1002 Write 7 Done Done Done false
189792024-03-06T21:05:38.634Z 4 Acked 1003 Read 3 Done Done Done false
189802024-03-06T21:05:38.634Z 5 Acked 1004 Read 4 Done Done Sent false
189812024-03-06T21:05:38.634Z 6 Acked 1005 Write 10 Done Done Sent false
189822024-03-06T21:05:38.634Z 7 Acked 1006 Write 7 Done Done Sent false
189832024-03-06T21:05:38.635Z 8 Acked 1007 Read 3 Done Done Sent false
189842024-03-06T21:05:38.635Z 9 Acked 1008 Read 2 Done Done Sent false
189852024-03-06T21:05:38.635Z 10 Acked 1009 Read 10 Done Done Sent false
189862024-03-06T21:05:38.635Z 11 Acked 1010 Read 2 Done Done Sent false
189872024-03-06T21:05:38.635Z 12 Acked 1011 Read 3 Done Done Sent false
189882024-03-06T21:05:38.635Z 13 Acked 1012 Read 3 Done Done Sent false
189892024-03-06T21:05:38.635Z 14 Acked 1013 Read 1 Done Done Sent false
189902024-03-06T21:05:38.635Z 15 Acked 1014 Write 9 Done Done Sent false
189912024-03-06T21:05:38.635Z 16 Acked 1015 Write 7 Done Done Sent false
189922024-03-06T21:05:38.635Z 17 Acked 1016 Read 8 Done Done Sent false
189932024-03-06T21:05:38.636Z 18 Acked 1017 Read 3 Done Done Sent false
189942024-03-06T21:05:38.636Z 19 Acked 1018 Write 5 Done Done Sent false
189952024-03-06T21:05:38.636Z 20 Acked 1019 Write 3 Done Done Sent false
189962024-03-06T21:05:38.636Z 21 Acked 1020 Read 6 Done Done Sent false
189972024-03-06T21:05:38.636Z 22 Acked 1021 Write 4 Done Done Sent false
189982024-03-06T21:05:38.636Z 23 Acked 1022 Write 7 Done Done Sent false
189992024-03-06T21:05:38.636Z 24 Acked 1023 Read 10 Done Done Sent false
190002024-03-06T21:05:38.636Z 25 Acked 1024 Read 7 Done Done Sent false
190012024-03-06T21:05:38.636Z 26 Acked 1025 Write 3 Done Done Sent false
190022024-03-06T21:05:38.636Z 27 Acked 1026 Write 7 Done Done Sent false
190032024-03-06T21:05:38.636Z 28 Acked 1027 Read 7 Done Done Sent false
190042024-03-06T21:05:38.636Z 29 Acked 1028 Write 7 Sent Sent Sent false
190052024-03-06T21:05:38.636Z 30 Acked 1029 Write 4 Sent Sent Sent false
190062024-03-06T21:05:38.637Z STATES DS:0 DS:1 DS:2 TOTAL
190072024-03-06T21:05:38.637Z New 0 0 0 0
190082024-03-06T21:05:38.637Z Sent 2 2 26 30
190092024-03-06T21:05:38.637Z Done 28 28 4 60
190102024-03-06T21:05:38.637Z Skipped 0 0 0 0
190112024-03-06T21:05:38.637Z Error 0 0 0 0
190122024-03-06T21:05:38.637ZLast Flush: 0 0 0
190132024-03-06T21:05:38.637ZDownstairs last five completed:
190142024-03-06T21:05:38.637ZUpstairs last five completed: 30 29 28 27 26
190152024-03-06T21:05:38.637Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:05:36.379571839Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1843}
190162024-03-06T21:05:38.637ZWrote out file "/tmp/test_repair_verify.data"
190172024-03-06T21:05:38.637Z
190182024-03-06T21:05:38.638ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
190192024-03-06T21:05:38.638ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
190202024-03-06T21:05:38.638Z 0 000-019 174 174 174 307 307 307 F F F
190212024-03-06T21:05:38.638Z 1 020-039 174 174 174 307 307 307 F F F
190222024-03-06T21:05:38.638Z 2 040-059 175 175 175 308 308 308 F F F
190232024-03-06T21:05:38.638Z 3 060-079 175 175 175 308 308 308 F F F
190242024-03-06T21:05:38.638Z 4 080-099 175 175 175 308 308 308 F F F
190252024-03-06T21:05:38.638Z 5 100-119 174 174 174 307 307 307 F F F
190262024-03-06T21:05:38.638Z 6 120-139 175 175 175 308 308 308 T T F <---
190272024-03-06T21:05:38.638Z 7 140-159 172 172 172 303 303 303 T T T
190282024-03-06T21:05:38.638Z 8 160-179 175 175 175 308 308 308 F F F
190292024-03-06T21:05:38.638Z 9 180-199 172 172 172 302 302 302 F F F
190302024-03-06T21:05:38.639Z 10 200-219 172 172 172 303 303 303 F F F
190312024-03-06T21:05:38.639Z 11 220-239 174 174 174 307 307 307 F F F
190322024-03-06T21:05:38.639Z 12 240-259 170 170 170 300 300 300 F F F
190332024-03-06T21:05:38.639Z 13 260-279 171 171 171 301 301 301 F F F
190342024-03-06T21:05:38.639Z 14 280-299 175 175 175 308 308 308 F F F
190352024-03-06T21:05:38.639Z 15 300-319 171 171 171 301 301 301 T T F <---
190362024-03-06T21:05:38.639Z 16 320-339 173 173 173 306 306 306 T T F <---
190372024-03-06T21:05:38.639Z 17 340-359 173 173 173 306 306 306 F F F
190382024-03-06T21:05:38.639Z 18 360-379 175 175 175 308 308 308 T T F <---
190392024-03-06T21:05:38.639Z 19 380-399 175 175 175 308 308 308 T T F <---
190402024-03-06T21:05:38.639Z 20 400-419 168 168 168 296 296 296 T T F <---
190412024-03-06T21:05:38.639Z 21 420-439 175 175 175 308 308 308 T T F <---
190422024-03-06T21:05:38.640Z 22 440-459 173 173 173 306 306 306 T T T
190432024-03-06T21:05:38.640Z 23 460-479 172 172 172 304 304 304 T T F <---
190442024-03-06T21:05:38.640Z 24 480-499 172 172 172 304 304 304 T T F <---
190452024-03-06T21:05:38.640Z 25 500-519 175 175 175 308 308 308 F F F
190462024-03-06T21:05:38.640Z 26 520-539 168 168 168 296 296 296 F F F
190472024-03-06T21:05:38.640Z 27 540-559 175 175 175 308 308 308 T T F <---
190482024-03-06T21:05:38.640Z 28 560-579 175 175 175 308 308 308 F F F
190492024-03-06T21:05:38.640Z 29 580-599 173 173 173 306 306 306 F F F
190502024-03-06T21:05:38.640ZMax gen: 175, Max flush: 308
190512024-03-06T21:05:38.640ZError: Difference in extent metadata found!
190522024-03-06T21:05:38.640ZA Difference in extent metadata is expected here
190532024-03-06T21:05:38.662ZOn loop 88
190542024-03-06T21:05:38.662Z
190552024-03-06T21:05:38.662ZVerifying data now
190562024-03-06T21:05:43.350ZNo change after verify
190572024-03-06T21:05:43.350ZLoop: 88 Downstairs dump after verify (and repair):
190582024-03-06T21:05:43.366ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
190592024-03-06T21:05:43.366Z 0 000-019 174 174 174 307 307 307 F F F
190602024-03-06T21:05:43.366Z 1 020-039 174 174 174 307 307 307 F F F
190612024-03-06T21:05:43.366Z 2 040-059 175 175 175 308 308 308 F F F
190622024-03-06T21:05:43.366Z 3 060-079 175 175 175 308 308 308 F F F
190632024-03-06T21:05:43.366Z 4 080-099 175 175 175 308 308 308 F F F
190642024-03-06T21:05:43.366Z 5 100-119 174 174 174 307 307 307 F F F
190652024-03-06T21:05:43.367Z 6 120-139 176 176 176 309 309 309 F F F
190662024-03-06T21:05:43.367Z 7 140-159 176 176 176 309 309 309 F F F
190672024-03-06T21:05:43.367Z 8 160-179 175 175 175 308 308 308 F F F
190682024-03-06T21:05:43.367Z 9 180-199 172 172 172 302 302 302 F F F
190692024-03-06T21:05:43.367Z 10 200-219 172 172 172 303 303 303 F F F
190702024-03-06T21:05:43.367Z 11 220-239 174 174 174 307 307 307 F F F
190712024-03-06T21:05:43.367Z 12 240-259 170 170 170 300 300 300 F F F
190722024-03-06T21:05:43.367Z 13 260-279 171 171 171 301 301 301 F F F
190732024-03-06T21:05:43.367Z 14 280-299 175 175 175 308 308 308 F F F
190742024-03-06T21:05:43.368Z 15 300-319 176 176 176 309 309 309 F F F
190752024-03-06T21:05:43.368Z 16 320-339 176 176 176 309 309 309 F F F
190762024-03-06T21:05:43.372Z 17 340-359 173 173 173 306 306 306 F F F
190772024-03-06T21:05:43.373Z 18 360-379 176 176 176 309 309 309 F F F
190782024-03-06T21:05:43.373Z 19 380-399 176 176 176 309 309 309 F F F
190792024-03-06T21:05:43.373Z 20 400-419 176 176 176 309 309 309 F F F
190802024-03-06T21:05:43.373Z 21 420-439 176 176 176 309 309 309 F F F
190812024-03-06T21:05:43.373Z 22 440-459 176 176 176 309 309 309 F F F
190822024-03-06T21:05:43.373Z 23 460-479 176 176 176 309 309 309 F F F
190832024-03-06T21:05:43.373Z 24 480-499 176 176 176 309 309 309 F F F
190842024-03-06T21:05:43.373Z 25 500-519 175 175 175 308 308 308 F F F
190852024-03-06T21:05:43.373Z 26 520-539 168 168 168 296 296 296 F F F
190862024-03-06T21:05:43.373Z 27 540-559 176 176 176 309 309 309 F F F
190872024-03-06T21:05:43.374Z 28 560-579 175 175 175 308 308 308 F F F
190882024-03-06T21:05:43.374Z 29 580-599 173 173 173 306 306 306 F F F
190892024-03-06T21:05:43.374ZMax gen: 176, Max flush: 309
190902024-03-06T21:05:43.374Z
190912024-03-06T21:05:43.374ZBegin loop 89
190922024-03-06T21:05:43.374ZDownstairs to restart: 1
190932024-03-06T21:05:43.394Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.132207984Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851}
190942024-03-06T21:05:43.394Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.132541681Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851}
190952024-03-06T21:05:43.395Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.132573809Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851}
190962024-03-06T21:05:43.402Z{"msg":"Crucible a2451847-9625-48b9-9bb3-a1b27bee420e has session id: 7034d3ce-051a-43bf-8d40-1e1bc58732d8","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.132602797Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851Crucible runtime is spawned
190972024-03-06T21:05:43.402Z,"session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
190982024-03-06T21:05:43.402Z{"msg":"Upstairs opts: Upstairs UUID: a2451847-9625-48b9-9bb3-a1b27bee420e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.132663253Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
190992024-03-06T21:05:43.402Z{"msg":"Crucible stats registered with UUID: a2451847-9625-48b9-9bb3-a1b27bee420e","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.1327095Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191002024-03-06T21:05:43.402Z{"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.132754476Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851}
191012024-03-06T21:05:43.402Z{"msg":"a2451847-9625-48b9-9bb3-a1b27bee420e active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.132794124Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191022024-03-06T21:05:43.403Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.132826432Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"client":"0","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191032024-03-06T21:05:43.403Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.13284676Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"client":"0","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191042024-03-06T21:05:43.403Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.132882817Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"client":"1","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191052024-03-06T21:05:43.403Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.132906546Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"client":"1","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191062024-03-06T21:05:43.403Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.132947063Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"client":"2","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191072024-03-06T21:05:43.403Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.132971591Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"client":"2","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191082024-03-06T21:05:43.403Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.133057565Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"":"io task","client":"0","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191092024-03-06T21:05:43.403Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.133120471Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"":"io task","client":"1","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191102024-03-06T21:05:43.404Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.133149099Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"":"io task","client":"2","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191112024-03-06T21:05:43.404Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.133182427Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"":"io task","client":"0","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191122024-03-06T21:05:43.404Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.133325457Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"":"io task","client":"1","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191132024-03-06T21:05:43.404Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.133378083Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"":"io task","client":"2","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191142024-03-06T21:05:43.404Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.133921536Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"client":"2","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191152024-03-06T21:05:43.404Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.133957203Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"client":"2","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191162024-03-06T21:05:43.404Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.133973222Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"client":"0","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191172024-03-06T21:05:43.404Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.134018149Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"client":"0","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191182024-03-06T21:05:43.405Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.134501136Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"client":"0","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191192024-03-06T21:05:43.405Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.134661325Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"client":"2","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191202024-03-06T21:05:43.405Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.135076976Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"client":"0","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191212024-03-06T21:05:43.405Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.135117854Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"client":"0","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191222024-03-06T21:05:43.405Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.135144352Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191232024-03-06T21:05:43.405Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.13517424Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"client":"2","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191242024-03-06T21:05:43.405Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.135198058Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"client":"2","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191252024-03-06T21:05:43.405Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.135226346Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191262024-03-06T21:05:43.405Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.135247734Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"client":"1","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191272024-03-06T21:05:43.406Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.135275403Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"client":"1","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191282024-03-06T21:05:43.406Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.135719262Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"client":"1","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191292024-03-06T21:05:43.406Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.136133304Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"client":"1","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191302024-03-06T21:05:43.406Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.13618447Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"client":"1","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191312024-03-06T21:05:43.406Z{"msg":"[0]R flush_numbers[0..12]: [307, 307, 308, 308, 308, 307, 309, 309, 308, 302, 303, 307]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.136218368Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"":"downstairs","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191322024-03-06T21:05:43.406Z{"msg":"[0]R generation[0..12]: [174, 174, 175, 175, 175, 174, 176, 176, 175, 172, 172, 174]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.136244136Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"":"downstairs","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191332024-03-06T21:05:43.406Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.136273394Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"":"downstairs","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191342024-03-06T21:05:43.407Z{"msg":"[1]R flush_numbers[0..12]: [307, 307, 308, 308, 308, 307, 309, 309, 308, 302, 303, 307]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
191352024-03-06T21:05:43.407Z,"time":"2024-03-06T21:05:42.136299652Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"":"downstairs","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191362024-03-06T21:05:43.407Z{"msg":"[1]R generation[0..12]: [174, 174, 175, 175, 175, 174, 176, 176, 175, 172, 172, 174]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.13632582Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"":"downstairs","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191372024-03-06T21:05:43.407Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.136358298Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"":"downstairs","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191382024-03-06T21:05:43.407Z{"msg":"[2]R flush_numbers[0..12]: [307, 307, 308, 308, 308, 307, 309, 309, 308, 302, 303, 307]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.136383376Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"":"downstairs","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191392024-03-06T21:05:43.407Z{"msg":"[2]R generation[0..12]: [174, 174, 175, 175, 175, 174, 176, 176, 175, 172, 172, 174]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.136415204Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"":"downstairs","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191402024-03-06T21:05:43.407Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.136447722Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"":"downstairs","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191412024-03-06T21:05:43.407Z{"msg":"Max found gen is 177","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.136468041Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"":"downstairs","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191422024-03-06T21:05:43.407Z{"msg":"Generation requested: 178 >= found:177","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.136498908Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"":"downstairs","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191432024-03-06T21:05:43.408Z{"msg":"Next flush: 310","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.136522877Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"":"downstairs","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191442024-03-06T21:05:43.408ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
191452024-03-06T21:05:43.408Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.136556654Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"":"downstairs","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191462024-03-06T21:05:43.408Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.136580343Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191472024-03-06T21:05:43.408Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.136600621Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"client":"0","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191482024-03-06T21:05:43.408Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.13662324Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"client":"1","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191492024-03-06T21:05:43.408Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.136653528Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"client":"2","":"downstairs","session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191502024-03-06T21:05:43.408Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.136676506Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191512024-03-06T21:05:43.408Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.136702564Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191522024-03-06T21:05:43.409Z{"msg":"a2451847-9625-48b9-9bb3-a1b27bee420e is now active with session: 7034d3ce-051a-43bf-8d40-1e1bc58732d8","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.136725503Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191532024-03-06T21:05:43.409Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.136750211Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851,"session_id":"7034d3ce-051a-43bf-8d40-1e1bc58732d8"}
191542024-03-06T21:05:43.409Z{"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:42.136771369Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851}
191552024-03-06T21:05:43.414ZLoading write count information from file "/tmp/test_repair_verify.data"
191562024-03-06T21:05:43.414ZRun Repair workload
191572024-03-06T21:05:43.414Z01/30 Write block 91 len 12288 data: 10 10 9
191582024-03-06T21:05:43.414Z02/30 Write block 242 len 8192 data: 16 15
191592024-03-06T21:05:43.414Z03/30 Read block 567 len 28672
191602024-03-06T21:05:43.415Z04/30 Read block 390 len 36864
191612024-03-06T21:05:43.415Z05/30 Read block 134 len 16384
191622024-03-06T21:05:43.415Z06/30 Write block 179 len 24576 data: 17 17 19 17 16 16
191632024-03-06T21:05:43.415Z07/30 Write block 484 len 12288 data: 13 15 12
191642024-03-06T21:05:43.415Z08/30 Read block 413 len 20480
191652024-03-06T21:05:43.415Z09/30 Write block 366 len 16384 data: 13 13 11 15
191662024-03-06T21:05:43.415Z10/30 Read block 463 len 40960
191672024-03-06T21:05:43.415Z11/30 Flush
191682024-03-06T21:05:43.415Z12/30 Write block 344 len 4096 data: 17
191692024-03-06T21:05:43.416Z13/30 Read block 323 len 12288
191702024-03-06T21:05:43.416Z14/30 Read block 245 len 32768
191712024-03-06T21:05:43.423Z15/30 Read block 324 len 24576
191722024-03-06T21:05:43.423Z16/30 Read block 516 len 36864
191732024-03-06T21:05:43.423Z17/30 Write block 532 len 24576 data: 13 13 14 16 12 14
191742024-03-06T21:05:43.423Z18/30 Write block 576 len 36864 data: 15 11 10 12 11 13 12 12 13
191752024-03-06T21:05:43.424Z19/30 Read block 506 len 16384
191762024-03-06T21:05:43.424Z20/30 Write block 386 len 36864 data: 18 16 17 15 17 12 10 9 8
191772024-03-06T21:05:43.424Z21/30 Flush
191782024-03-06T21:05:43.427Z22/30 Write block 83 len 4096 data: 12
191792024-03-06T21:05:43.428Z23/30 Write block 19 len 8192 data: 19 19
191802024-03-06T21:05:43.428Z24/30 Read block 508 len 12288
191812024-03-06T21:05:43.428Z25/30 Read block 289 len 36864
191822024-03-06T21:05:43.428Z26/30 Read block 137 len 28672
191832024-03-06T21:05:43.432Z27/30 Write block 478 len 40960 data: 11 6 9 10 11 13 14 16 13 12
191842024-03-06T21:05:43.432Z28/30 Write block 529 len 4096 data: 17
191852024-03-06T21:05:43.432Z29/30 Write block 45 len 40960 data: 10 11 14 13 11 7 9 10 10 8
191862024-03-06T21:05:43.432Z30/30 Read block 279 len 12288
191872024-03-06T21:05:43.436Z----------------------------------------------------------------
191882024-03-06T21:05:43.436Z Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30
191892024-03-06T21:05:43.437ZStates: Active Active Active
191902024-03-06T21:05:43.437ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
191912024-03-06T21:05:43.437Z 1 Acked 1000 Write 3 Done Sent Done false
191922024-03-06T21:05:43.437Z 2 Acked 1001 Write 2 Done Sent Done false
191932024-03-06T21:05:43.437Z 3 Acked 1002 Read 7 Done Sent Done false
191942024-03-06T21:05:43.437Z 4 Acked 1003 Read 9 Done Sent Done false
191952024-03-06T21:05:43.437Z 5 Acked 1004 Read 4 Done Sent Done false
191962024-03-06T21:05:43.437Z 6 Acked 1005 Write 6 Done Sent Done false
191972024-03-06T21:05:43.437Z 7 Acked 1006 Write 3 Done Sent Done false
191982024-03-06T21:05:43.437Z 8 Acked 1007 Read 5 Done Sent Done false
191992024-03-06T21:05:43.437Z 9 Acked 1008 Write 4 Done Sent Done false
192002024-03-06T21:05:43.438Z 10 Acked 1009 Read 10 Done Sent Done false
192012024-03-06T21:05:43.438Z 11 Acked 1010 Flush 0 Done Sent Done false
192022024-03-06T21:05:43.438Z 12 Acked 1011 Write 1 Done Sent Done false
192032024-03-06T21:05:43.438Z 13 Acked 1012 Read 3 Done Sent Done false
192042024-03-06T21:05:43.438Z 14 Acked 1013 Read 8 Done Sent Done false
192052024-03-06T21:05:43.438Z 15 Acked 1014 Read 6 Done Sent Done false
192062024-03-06T21:05:43.438Z 16 Acked 1015 Read 9 Done Sent Done false
192072024-03-06T21:05:43.438Z 17 Acked 1016 Write 6 Done Sent Done false
192082024-03-06T21:05:43.438Z 18 Acked 1017 Write 9 Done Sent Done false
192092024-03-06T21:05:43.438Z 19 Acked 1018 Read 4 Done Sent Done false
192102024-03-06T21:05:43.438Z 20 Acked 1019 Write 9 Done Sent Done false
192112024-03-06T21:05:43.438Z 21 Acked 1020 Flush 0 Done Sent Done false
192122024-03-06T21:05:43.439Z 22 Acked 1021 Write 1 Done Sent Done false
192132024-03-06T21:05:43.439Z 23 Acked 1022 Write 2 Done Sent Done false
192142024-03-06T21:05:43.439Z 24 Acked 1023 Read 3 Done Sent Done false
192152024-03-06T21:05:43.439Z 25 Acked 1024 Read 9 Done Sent Done false
192162024-03-06T21:05:43.439Z 26 Acked 1025 Read 7 Done Sent Done false
192172024-03-06T21:05:43.439Z 27 Acked 1026 Write 10 Done Sent Done false
192182024-03-06T21:05:43.439Z 28 Acked 1027 Write 1 Done Sent Done false
192192024-03-06T21:05:43.439Z 29 Acked 1028 Write 10 Done Sent Done false
192202024-03-06T21:05:43.439Z 30 Acked 1029 Read 3 Sent Sent Done false
192212024-03-06T21:05:43.439Z STATES DS:0 DS:1 DS:2 TOTAL
192222024-03-06T21:05:43.439Z New 0 0 0 0
192232024-03-06T21:05:43.440Z Sent 1 30 0 31
192242024-03-06T21:05:43.440Z Done 29 0 30 59
192252024-03-06T21:05:43.440Z Skipped 0 0 0 0
192262024-03-06T21:05:43.440Z Error 0 0 0 0
192272024-03-06T21:05:43.440ZLast Flush: 1020 0 1020
192282024-03-06T21:05:43.440ZDownstairs last five completed:
192292024-03-06T21:05:43.440ZUpstairs last five completed: 30 29 28 27 26
192302024-03-06T21:05:43.440Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:05:42.170950094Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1851}
192312024-03-06T21:05:43.440ZWrote out file "/tmp/test_repair_verify.data"
192322024-03-06T21:05:43.445Z
192332024-03-06T21:05:43.445ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
192342024-03-06T21:05:43.462ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
192352024-03-06T21:05:43.462Z 0 000-019 174 174 174 307 307 307 T F T <---
192362024-03-06T21:05:43.462Z 1 020-039 174 174 174 307 307 307 T F T <---
192372024-03-06T21:05:43.462Z 2 040-059 175 175 175 308 308 308 T F T <---
192382024-03-06T21:05:43.462Z 3 060-079 175 175 175 308 308 308 F F F
192392024-03-06T21:05:43.462Z 4 080-099 178 175 178 310 308 310 T F T <---
192402024-03-06T21:05:43.462Z 5 100-119 174 174 174 307 307 307 F F F
192412024-03-06T21:05:43.462Z 6 120-139 176 176 176 309 309 309 F F F
192422024-03-06T21:05:43.462Z 7 140-159 176 176 176 309 309 309 F F F
192432024-03-06T21:05:43.462Z 8 160-179 178 175 178 310 308 310 F F F <---
192442024-03-06T21:05:43.463Z 9 180-199 178 172 178 310 302 310 F F F <---
192452024-03-06T21:05:43.463Z 10 200-219 172 172 172 303 303 303 F F F
192462024-03-06T21:05:43.463Z 11 220-239 174 174 174 307 307 307 F F F
192472024-03-06T21:05:43.463Z 12 240-259 178 170 178 310 300 310 F F F <---
192482024-03-06T21:05:43.463Z 13 260-279 171 171 171 301 301 301 F F F
192492024-03-06T21:05:43.463Z 14 280-299 175 175 175 308 308 308 F F F
192502024-03-06T21:05:43.463Z 15 300-319 176 176 176 309 309 309 F F F
192512024-03-06T21:05:43.463Z 16 320-339 176 176 176 309 309 309 F F F
192522024-03-06T21:05:43.463Z 17 340-359 178 173 178 311 306 311 F F F <---
192532024-03-06T21:05:43.464Z 18 360-379 178 176 178 310 309 310 F F F <---
192542024-03-06T21:05:43.464Z 19 380-399 178 176 178 311 309 311 F F F <---
192552024-03-06T21:05:43.464Z 20 400-419 176 176 176 309 309 309 F F F
192562024-03-06T21:05:43.464Z 21 420-439 176 176 176 309 309 309 F F F
192572024-03-06T21:05:43.469Z 22 440-459 176 176 176 309 309 309 F F F
192582024-03-06T21:05:43.469Z 23 460-479 176 176 176 309 309 309 T F T <---
192592024-03-06T21:05:43.469Z 24 480-499 178 176 178 310 309 310 T F T <---
192602024-03-06T21:05:43.469Z 25 500-519 175 175 175 308 308 308 F F F
192612024-03-06T21:05:43.469Z 26 520-539 178 168 178 311 296 311 T F T <---
192622024-03-06T21:05:43.469Z 27 540-559 176 176 176 309 309 309 F F F
192632024-03-06T21:05:43.469Z 28 560-579 178 175 178 311 308 311 F F F <---
192642024-03-06T21:05:43.469Z 29 580-599 178 173 178 311 306 311 F F F <---
192652024-03-06T21:05:43.469ZMax gen: 178, Max flush: 311
192662024-03-06T21:05:43.470ZError: Difference in extent metadata found!
192672024-03-06T21:05:43.470ZA Difference in extent metadata is expected here
192682024-03-06T21:05:43.470ZOn loop 89
192692024-03-06T21:05:43.470Z
192702024-03-06T21:05:43.474ZVerifying data now
192712024-03-06T21:05:48.237ZNo change after verify
192722024-03-06T21:05:48.237ZLoop: 89 Downstairs dump after verify (and repair):
192732024-03-06T21:05:48.253ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
192742024-03-06T21:05:48.253Z 0 000-019 179 179 179 312 312 312 F F F
192752024-03-06T21:05:48.253Z 1 020-039 179 179 179 312 312 312 F F F
192762024-03-06T21:05:48.253Z 2 040-059 179 179 179 312 312 312 F F F
192772024-03-06T21:05:48.253Z 3 060-079 175 175 175 308 308 308 F F F
192782024-03-06T21:05:48.253Z 4 080-099 179 179 179 312 312 312 F F F
192792024-03-06T21:05:48.253Z 5 100-119 174 174 174 307 307 307 F F F
192802024-03-06T21:05:48.254Z 6 120-139 176 176 176 309 309 309 F F F
192812024-03-06T21:05:48.254Z 7 140-159 176 176 176 309 309 309 F F F
192822024-03-06T21:05:48.254Z 8 160-179 178 178 178 310 310 310 F F F
192832024-03-06T21:05:48.254Z 9 180-199 178 178 178 310 310 310 F F F
192842024-03-06T21:05:48.254Z 10 200-219 172 172 172 303 303 303 F F F
192852024-03-06T21:05:48.254Z 11 220-239 174 174 174 307 307 307 F F F
192862024-03-06T21:05:48.254Z 12 240-259 178 178 178 310 310 310 F F F
192872024-03-06T21:05:48.254Z 13 260-279 171 171 171 301 301 301 F F F
192882024-03-06T21:05:48.254Z 14 280-299 175 175 175 308 308 308 F F F
192892024-03-06T21:05:48.254Z 15 300-319 176 176 176 309 309 309 F F F
192902024-03-06T21:05:48.254Z 16 320-339 176 176 176 309 309 309 F F F
192912024-03-06T21:05:48.254Z 17 340-359 178 178 178 311 311 311 F F F
192922024-03-06T21:05:48.254Z 18 360-379 178 178 178 310 310 310 F F F
192932024-03-06T21:05:48.262Z 19 380-399 178 178 178 311 311 311 F F F
192942024-03-06T21:05:48.263Z 20 400-419 176 176 176 309 309 309 F F F
192952024-03-06T21:05:48.263Z 21 420-439 176 176 176 309 309 309 F F F
192962024-03-06T21:05:48.263Z 22 440-459 176 176 176 309 309 309 F F F
192972024-03-06T21:05:48.264Z 23 460-479 179 179 179 312 312 312 F F F
192982024-03-06T21:05:48.264Z 24 480-499 179 179 179 312 312 312 F F F
192992024-03-06T21:05:48.264Z 25 500-519 175 175 175 308 308 308 F F F
193002024-03-06T21:05:48.264Z 26 520-539 179 179 179 312 312 312 F F F
193012024-03-06T21:05:48.264Z 27 540-559 176 176 176 309 309 309 F F F
193022024-03-06T21:05:48.264Z 28 560-579 178 178 178 311 311 311 F F F
193032024-03-06T21:05:48.265Z 29 580-599 178 178 178 311 311 311 F F F
193042024-03-06T21:05:48.265ZMax gen: 179, Max flush: 312
193052024-03-06T21:05:48.265Z
193062024-03-06T21:05:48.265ZBegin loop 90
193072024-03-06T21:05:48.265ZDownstairs to restart: 1
193082024-03-06T21:05:48.283Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.018803545Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859}
193092024-03-06T21:05:48.292Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.019099094Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859}
193102024-03-06T21:05:48.292Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.019134912Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859}
193112024-03-06T21:05:48.292Z{"msg":"Crucible f3a00967-30d8-4d3b-898e-b5613fc1c694 has session id: fb1c97c6-2950-44e7-a706-4724620608dc","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.019187438Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193122024-03-06T21:05:48.292Z{"msg":"Upstairs opts: Upstairs UUID: f3a00967-30d8-4d3b-898e-b5613fc1c694, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
193132024-03-06T21:05:48.292Z,"time":"2024-03-06T21:05:47.019240624Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193142024-03-06T21:05:48.293Z{"msg":"Crucible stats registered with UUID: f3a00967-30d8-4d3b-898e-b5613fc1c694","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.019294431Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193152024-03-06T21:05:48.293Z{"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.019326369Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859}
193162024-03-06T21:05:48.293Z{"msg":"f3a00967-30d8-4d3b-898e-b5613fc1c694 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.01944897Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193172024-03-06T21:05:48.293Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.019486287Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"client":"0","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193182024-03-06T21:05:48.293Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.019518625Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"client":"0","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193192024-03-06T21:05:48.293Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.019567562Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"client":"1","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193202024-03-06T21:05:48.293Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.01960072Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"client":"1","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193212024-03-06T21:05:48.293Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.019640947Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"client":"2","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193222024-03-06T21:05:48.293Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.019666975Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"client":"2","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193232024-03-06T21:05:48.293Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.019703432Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"":"io task","client":"2","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193242024-03-06T21:05:48.293Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.019757539Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"":"io task","client":"0","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193252024-03-06T21:05:48.293Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.019793016Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"":"io task","client":"1","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193262024-03-06T21:05:48.294Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.019842213Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"":"io task","client":"0","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193272024-03-06T21:05:48.294Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.019869811Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"":"io task","client":"1","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193282024-03-06T21:05:48.294Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.019977894Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"":"io task","client":"2","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193292024-03-06T21:05:48.294Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.020690024Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"client":"2","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193302024-03-06T21:05:48.294Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.020732222Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"client":"2","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193312024-03-06T21:05:48.294Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.02076165Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"client":"0","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193322024-03-06T21:05:48.294Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.020805776Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"client":"0","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193332024-03-06T21:05:48.294Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.021345159Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"client":"0","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193342024-03-06T21:05:48.294Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.02148898Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"client":"2","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193352024-03-06T21:05:48.294Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.021660688Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"client":"1","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193362024-03-06T21:05:48.295Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.021764881Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"client":"1","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193372024-03-06T21:05:48.295Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.021850175Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"client":"0","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193382024-03-06T21:05:48.295Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.021898471Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"client":"0","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193392024-03-06T21:05:48.295Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.021936819Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193402024-03-06T21:05:48.295Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.021982495Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"client":"2","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193412024-03-06T21:05:48.295Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.022014843Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"client":"2","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193422024-03-06T21:05:48.295Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.022039682Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193432024-03-06T21:05:48.295Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.022256416Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"client":"1","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193442024-03-06T21:05:48.295Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.022691037Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"client":"1","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193452024-03-06T21:05:48.295Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.022723914Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"client":"1","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193462024-03-06T21:05:48.296Z{"msg":"[0]R flush_numbers[0..12]: [312, 312, 312, 308, 312, 307, 309, 309, 310, 310, 303, 307]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.022750102Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"":"downstairs","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193472024-03-06T21:05:48.296Z{"msg":"[0]R generation[0..12]: [179, 179, 179, 175, 179, 174, 176, 176, 178, 178, 172, 174]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.022779351Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"":"downstairs","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193482024-03-06T21:05:48.296Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.022804499Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"":"downstairs","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193492024-03-06T21:05:48.296Z{"msg":"[1]R flush_numbers[0..12]: [312, 312, 312, 308, 312, 307, 309, 309, 310, 310, 303, 307]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.022831637Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"":"downstairs","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193502024-03-06T21:05:48.296Z{"msg":"[1]R generation[0..12]: [179, 179, 179, 175, 179, 174, 176, 176, 178, 178, 172, 174]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.022855775Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"":"downstairs",Wait for a query_work_queue command to finish before sending IO
193512024-03-06T21:05:48.296Z"":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193522024-03-06T21:05:48.296Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.022889943Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"":"downstairs","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193532024-03-06T21:05:48.296Z{"msg":"[2]R flush_numbers[0..12]: [312, 312, 312, 308, 312, 307, 309, 309, 310, 310, 303, 307]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.022914591Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"":"downstairs","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193542024-03-06T21:05:48.296Z{"msg":"[2]R generation[0..12]: [179, 179, 179, 175, 179, 174, 176, 176, 178, 178, 172, 174]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.022946969Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"":"downstairs","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193552024-03-06T21:05:48.296Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.022972107Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"":"downstairs","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193562024-03-06T21:05:48.296Z{"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.022999885Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"":"downstairs","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193572024-03-06T21:05:48.296Z{"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.023024474Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"":"downstairs","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193582024-03-06T21:05:48.296Z{"msg":"Next flush: 313","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.023055181Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"":"downstairs","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193592024-03-06T21:05:48.297Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.02307815Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"":"downstairs","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193602024-03-06T21:05:48.297Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.023104218Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193612024-03-06T21:05:48.297Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.023125157Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"client":"0","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193622024-03-06T21:05:48.297Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
193632024-03-06T21:05:48.297Z,"time":"2024-03-06T21:05:47.023152525Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"client":"1","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193642024-03-06T21:05:48.297Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.023179063Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"client":"2","":"downstairs","session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193652024-03-06T21:05:48.297Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.023205351Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193662024-03-06T21:05:48.297Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.02322719Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193672024-03-06T21:05:48.297Z{"msg":"f3a00967-30d8-4d3b-898e-b5613fc1c694 is now active with session: fb1c97c6-2950-44e7-a706-4724620608dc","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.023254738Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193682024-03-06T21:05:48.297Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.023275456Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859,"session_id":"fb1c97c6-2950-44e7-a706-4724620608dc"}
193692024-03-06T21:05:48.297Z{"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:47.023300515Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859}
193702024-03-06T21:05:48.297ZLoading write count information from file "/tmp/test_repair_verify.data"
193712024-03-06T21:05:48.298ZRun Repair workload
193722024-03-06T21:05:48.298Z01/30 Write block 49 len 32768 data: 12 8 10 11 11 9 11 8
193732024-03-06T21:05:48.298Z02/30 Read block 86 len 8192
193742024-03-06T21:05:48.310Z03/30 Read block 28 len 40960
193752024-03-06T21:05:48.311Z04/30 Read block 572 len 8192
193762024-03-06T21:05:48.311Z05/30 Flush
193772024-03-06T21:05:48.311Z06/30 Write block 1 len 40960 data: 5 9 11 14 14 14 15 14 14 14
193782024-03-06T21:05:48.311Z07/30 Read block 463 len 28672
193792024-03-06T21:05:48.311Z08/30 Flush
193802024-03-06T21:05:48.311Z09/30 Flush
193812024-03-06T21:05:48.311Z10/30 Write block 173 len 4096 data: 17
193822024-03-06T21:05:48.311Z11/30 Read block 99 len 36864
193832024-03-06T21:05:48.311Z12/30 Read block 471 len 28672
193842024-03-06T21:05:48.311Z13/30 Write block 570 len 32768 data: 12 15 14 13 12 13 16 12
193852024-03-06T21:05:48.311Z14/30 Flush
193862024-03-06T21:05:48.311Z15/30 Write block 540 len 8192 data: 12 13
193872024-03-06T21:05:48.311Z16/30 Read block 472 len 8192
193882024-03-06T21:05:48.312Z17/30 Write block 372 len 40960 data: 15 16 15 19 15 14 14 20 21 20
193892024-03-06T21:05:48.328Z18/30 Write block 124 len 32768 data: 13 18 16 16 15 16 12 10
193902024-03-06T21:05:48.328Z19/30 Write block 41 len 28672 data: 17 18 17 16 11 12 15
193912024-03-06T21:05:48.328Z20/30 Flush
193922024-03-06T21:05:48.328Z21/30 Write block 573 len 16384 data: 14 13 14 17
193932024-03-06T21:05:48.328Z22/30 Write block 174 len 12288 data: 15 16 14
193942024-03-06T21:05:48.328Z23/30 Flush
193952024-03-06T21:05:48.328Z24/30 Write block 303 len 40960 data: 15 17 18 20 19 18 17 16 16 13
193962024-03-06T21:05:48.328Z25/30 Read block 377 len 32768
193972024-03-06T21:05:48.336Z26/30 Read block 331 len 28672
193982024-03-06T21:05:48.336Z27/30 Read block 88 len 24576
193992024-03-06T21:05:48.337Z28/30 Read block 43 len 36864
194002024-03-06T21:05:48.337Z29/30 Write block 10 len 40960 data: 15 17 14 15 17 19 17 19 18 20
194012024-03-06T21:05:48.337Z30/30 Write block 540 len 36864 data: 13 14 16 18 20 21 20 18 15
194022024-03-06T21:05:48.337Z----------------------------------------------------------------
194032024-03-06T21:05:48.337Z Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:30
194042024-03-06T21:05:48.337ZStates: Active Active Active
194052024-03-06T21:05:48.337ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
194062024-03-06T21:05:48.337Z 1 Acked 1000 Write 8 Done Sent Done false
194072024-03-06T21:05:48.338Z 2 Acked 1001 Read 2 Done Sent Done false
194082024-03-06T21:05:48.338Z 3 Acked 1002 Read 10 Done Sent Done false
194092024-03-06T21:05:48.338Z 4 Acked 1003 Read 2 Done Sent Done false
194102024-03-06T21:05:48.338Z 5 Acked 1004 Flush 0 Done Sent Done false
194112024-03-06T21:05:48.338Z 6 Acked 1005 Write 10 Done Sent Done false
194122024-03-06T21:05:48.338Z 7 Acked 1006 Read 7 Done Sent Done false
194132024-03-06T21:05:48.338Z 8 Acked 1007 Flush 0 Done Sent Done false
194142024-03-06T21:05:48.338Z 9 Acked 1008 Flush 0 Done Sent Done false
194152024-03-06T21:05:48.338Z 10 Acked 1009 Write 1 Done Sent Done false
194162024-03-06T21:05:48.338Z 11 Acked 1010 Read 9 Done Sent Done false
194172024-03-06T21:05:48.338Z 12 Acked 1011 Read 7 Done Sent Done false
194182024-03-06T21:05:48.339Z 13 Acked 1012 Write 8 Done Sent Done false
194192024-03-06T21:05:48.339Z 14 Acked 1013 Flush 0 Done Sent Done false
194202024-03-06T21:05:48.339Z 15 Acked 1014 Write 2 Done Sent Done false
194212024-03-06T21:05:48.339Z 16 Acked 1015 Read 2 Done Sent Done false
194222024-03-06T21:05:48.339Z 17 Acked 1016 Write 10 Done Sent Done false
194232024-03-06T21:05:48.339Z 18 Acked 1017 Write 8 Done Sent Done false
194242024-03-06T21:05:48.339Z 19 Acked 1018 Write 7 Done Sent Done false
194252024-03-06T21:05:48.339Z 20 Acked 1019 Flush 0 Done Sent Done false
194262024-03-06T21:05:48.339Z 21 Acked 1020 Write 4 Done Sent Done false
194272024-03-06T21:05:48.339Z 22 Acked 1021 Write 3 Done Sent Done false
194282024-03-06T21:05:48.339Z 23 Acked 1022 Flush 0 Done Sent Done false
194292024-03-06T21:05:48.339Z 24 Acked 1023 Write 10 Done Sent Done false
194302024-03-06T21:05:48.340Z 25 Acked 1024 Read 8 Done Sent Done false
194312024-03-06T21:05:48.340Z 26 Acked 1025 Read 7 Done Sent Done false
194322024-03-06T21:05:48.340Z 27 Acked 1026 Read 6 Done Sent Done false
194332024-03-06T21:05:48.340Z 28 Acked 1027 Read 9 Done Sent Done false
194342024-03-06T21:05:48.340Z 29 Acked 1028 Write 10 Done Sent Done false
194352024-03-06T21:05:48.340Z 30 Acked 1029 Write 9 Sent Sent Sent false
194362024-03-06T21:05:48.340Z STATES DS:0 DS:1 DS:2 TOTAL
194372024-03-06T21:05:48.340Z New 0 0 0 0
194382024-03-06T21:05:48.340Z Sent 1 30 1 32
194392024-03-06T21:05:48.340Z Done 29 0 29 58
194402024-03-06T21:05:48.340Z Skipped 0 0 0 0
194412024-03-06T21:05:48.340Z Error 0 0 0 0
194422024-03-06T21:05:48.341ZLast Flush: 1022 0 1022
194432024-03-06T21:05:48.341ZDownstairs last five completed:
194442024-03-06T21:05:48.341ZUpstairs last five completed: 30 29 28 27 26
194452024-03-06T21:05:48.341Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:05:47.063082703Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1859}
194462024-03-06T21:05:48.341ZWrote out file "/tmp/test_repair_verify.data"
194472024-03-06T21:05:48.341Z
194482024-03-06T21:05:48.347ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
194492024-03-06T21:05:48.402ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
194502024-03-06T21:05:48.403Z 0 000-019 180 179 180 314 312 314 T F T <---
194512024-03-06T21:05:48.403Z 1 020-039 179 179 179 312 312 312 F F F
194522024-03-06T21:05:48.403Z 2 040-059 180 179 180 317 312 317 F F F <---
194532024-03-06T21:05:48.403Z 3 060-079 175 175 175 308 308 308 F F F
194542024-03-06T21:05:48.404Z 4 080-099 179 179 179 312 312 312 F F F
194552024-03-06T21:05:48.404Z 5 100-119 174 174 174 307 307 307 F F F
194562024-03-06T21:05:48.404Z 6 120-139 180 176 180 317 309 317 F F F <---
194572024-03-06T21:05:48.404Z 7 140-159 176 176 176 309 309 309 F F F
194582024-03-06T21:05:48.404Z 8 160-179 180 178 180 318 310 318 F F F <---
194592024-03-06T21:05:48.405Z 9 180-199 178 178 178 310 310 310 F F F
194602024-03-06T21:05:48.405Z 10 200-219 172 172 172 303 303 303 F F F
194612024-03-06T21:05:48.405Z 11 220-239 174 174 174 307 307 307 F F F
194622024-03-06T21:05:48.425Z 12 240-259 178 178 178 310 310 310 F F F
194632024-03-06T21:05:48.425Z 13 260-279 171 171 171 301 301 301 F F F
194642024-03-06T21:05:48.425Z 14 280-299 175 175 175 308 308 308 F F F
194652024-03-06T21:05:48.425Z 15 300-319 176 176 176 309 309 309 T F T <---
194662024-03-06T21:05:48.425Z 16 320-339 176 176 176 309 309 309 F F F
194672024-03-06T21:05:48.425Z 17 340-359 178 178 178 311 311 311 F F F
194682024-03-06T21:05:48.425Z 18 360-379 180 178 180 317 310 317 F F F <---
194692024-03-06T21:05:48.425Z 19 380-399 180 178 180 317 311 317 F F F <---
194702024-03-06T21:05:48.425Z 20 400-419 176 176 176 309 309 309 F F F
194712024-03-06T21:05:48.426Z 21 420-439 176 176 176 309 309 309 F F F
194722024-03-06T21:05:48.426Z 22 440-459 176 176 176 309 309 309 F F F
194732024-03-06T21:05:48.426Z 23 460-479 179 179 179 312 312 312 F F F
194742024-03-06T21:05:48.426Z 24 480-499 179 179 179 312 312 312 F F F
194752024-03-06T21:05:48.426Z 25 500-519 175 175 175 308 308 308 F F F
194762024-03-06T21:05:48.426Z 26 520-539 179 179 179 312 312 312 F F F
194772024-03-06T21:05:48.426Z 27 540-559 180 176 180 317 309 317 T F T <---
194782024-03-06T21:05:48.426Z 28 560-579 180 178 180 318 311 318 F F F <---
194792024-03-06T21:05:48.426Z 29 580-599 178 178 178 311 311 311 F F F
194802024-03-06T21:05:48.426ZMax gen: 180, Max flush: 318
194812024-03-06T21:05:48.426ZError: Difference in extent metadata found!
194822024-03-06T21:05:48.426ZA Difference in extent metadata is expected here
194832024-03-06T21:05:48.426ZOn loop 90
194842024-03-06T21:05:48.426Z
194852024-03-06T21:05:48.426ZVerifying data now
194862024-03-06T21:05:52.878ZNo change after verify
194872024-03-06T21:05:52.878ZLoop: 90 Downstairs dump after verify (and repair):
194882024-03-06T21:05:52.893ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
194892024-03-06T21:05:52.894Z 0 000-019 181 181 181 319 319 319 F F F
194902024-03-06T21:05:52.894Z 1 020-039 179 179 179 312 312 312 F F F
194912024-03-06T21:05:52.894Z 2 040-059 180 180 180 317 317 317 F F F
194922024-03-06T21:05:52.894Z 3 060-079 175 175 175 308 308 308 F F F
194932024-03-06T21:05:52.894Z 4 080-099 179 179 179 312 312 312 F F F
194942024-03-06T21:05:52.894Z 5 100-119 174 174 174 307 307 307 F F F
194952024-03-06T21:05:52.894Z 6 120-139 180 180 180 317 317 317 F F F
194962024-03-06T21:05:52.894Z 7 140-159 176 176 176 309 309 309 F F F
194972024-03-06T21:05:52.894Z 8 160-179 180 180 180 318 318 318 F F F
194982024-03-06T21:05:52.894Z 9 180-199 178 178 178 310 310 310 F F F
194992024-03-06T21:05:52.894Z 10 200-219 172 172 172 303 303 303 F F F
195002024-03-06T21:05:52.894Z 11 220-239 174 174 174 307 307 307 F F F
195012024-03-06T21:05:52.894Z 12 240-259 178 178 178 310 310 310 F F F
195022024-03-06T21:05:52.894Z 13 260-279 171 171 171 301 301 301 F F F
195032024-03-06T21:05:52.895Z 14 280-299 175 175 175 308 308 308 F F F
195042024-03-06T21:05:52.895Z 15 300-319 181 181 181 319 319 319 F F F
195052024-03-06T21:05:52.895Z 16 320-339 176 176 176 309 309 309 F F F
195062024-03-06T21:05:52.895Z 17 340-359 178 178 178 311 311 311 F F F
195072024-03-06T21:05:52.899Z 18 360-379 180 180 180 317 317 317 F F F
195082024-03-06T21:05:52.899Z 19 380-399 180 180 180 317 317 317 F F F
195092024-03-06T21:05:52.899Z 20 400-419 176 176 176 309 309 309 F F F
195102024-03-06T21:05:52.899Z 21 420-439 176 176 176 309 309 309 F F F
195112024-03-06T21:05:52.899Z 22 440-459 176 176 176 309 309 309 F F F
195122024-03-06T21:05:52.899Z 23 460-479 179 179 179 312 312 312 F F F
195132024-03-06T21:05:52.899Z 24 480-499 179 179 179 312 312 312 F F F
195142024-03-06T21:05:52.899Z 25 500-519 175 175 175 308 308 308 F F F
195152024-03-06T21:05:52.899Z 26 520-539 179 179 179 312 312 312 F F F
195162024-03-06T21:05:52.899Z 27 540-559 181 181 181 319 319 319 F F F
195172024-03-06T21:05:52.899Z 28 560-579 180 180 180 318 318 318 F F F
195182024-03-06T21:05:52.899Z 29 580-599 178 178 178 311 311 311 F F F
195192024-03-06T21:05:52.899ZMax gen: 181, Max flush: 319
195202024-03-06T21:05:52.900Z
195212024-03-06T21:05:52.900ZBegin loop 91
195222024-03-06T21:05:52.900ZDownstairs to restart: 2
195232024-03-06T21:05:52.922Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.658642261Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867}
195242024-03-06T21:05:52.922Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.658925252Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867}
195252024-03-06T21:05:52.922Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.65894998Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867}
195262024-03-06T21:05:52.922Z{"msg":"Crucible 857adc29-c090-4739-a993-29fb4d29b464 has session id: 75b68ddb-db4c-43cd-abdb-24bd51c54a48","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.658969849Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195272024-03-06T21:05:52.923Z{"msg":"Crucible runtime is spawned
195282024-03-06T21:05:53.361ZUpstairs opts: Upstairs UUID: 857adc29-c090-4739-a993-29fb4d29b464, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.659012626Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195292024-03-06T21:05:53.361Z{"msg":"Crucible stats registered with UUID: 857adc29-c090-4739-a993-29fb4d29b464","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.659086951Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195302024-03-06T21:05:53.361Z{"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.659124568Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867}
195312024-03-06T21:05:53.362Z{"msg":"857adc29-c090-4739-a993-29fb4d29b464 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.659143867Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195322024-03-06T21:05:53.362Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.659165635Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"client":"0","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195332024-03-06T21:05:53.362Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.659185704Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"client":"0","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195342024-03-06T21:05:53.362Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.659218111Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"client":"1","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195352024-03-06T21:05:53.362Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.659242Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"client":"1","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195362024-03-06T21:05:53.362Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.659278317Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"client":"2","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195372024-03-06T21:05:53.362Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.659303336Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"client":"2","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195382024-03-06T21:05:53.363Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.659341523Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"":"io task","client":"0","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195392024-03-06T21:05:53.363Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.65938239Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"":"io task","client":"1","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195402024-03-06T21:05:53.363Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.659416638Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"":"io task","client":"2","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195412024-03-06T21:05:53.363Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.659512721Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"":"io task","client":"0","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195422024-03-06T21:05:53.363Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.659563118Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"":"io task","client":"2","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195432024-03-06T21:05:53.363Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.659593526Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"":"io task","client":"1","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195442024-03-06T21:05:53.363Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.660330525Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"client":"0","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195452024-03-06T21:05:53.363Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.660373712Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"client":"0","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195462024-03-06T21:05:53.363Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.660394421Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"client":"1","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195472024-03-06T21:05:53.364Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.660479415Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"client":"1","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195482024-03-06T21:05:53.364Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.661038466Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"client":"1","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195492024-03-06T21:05:53.364Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.661114101Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"client":"0","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195502024-03-06T21:05:53.364Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.661618876Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"client":"2","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195512024-03-06T21:05:53.364Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.661659103Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"client":"2","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195522024-03-06T21:05:53.364Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.661844961Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"client":"1","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195532024-03-06T21:05:53.364Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.661874848Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"client":"1","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195542024-03-06T21:05:53.364Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.661890777Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195552024-03-06T21:05:53.365Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.661938884Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"client":"0","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195562024-03-06T21:05:53.365Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.661953873Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"client":"0","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195572024-03-06T21:05:53.365Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.661977781Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195582024-03-06T21:05:53.365Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.662198276Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"client":"2","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195592024-03-06T21:05:53.365Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.662588859Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"client":"2","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195602024-03-06T21:05:53.365Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.662614897Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"client":"2","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195612024-03-06T21:05:53.365Z{"msg":"[0]R flush_numbers[0..12]: [319, 312, 317, 308, 312, 307, 317, 309, 318, 310, 303, 307]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.662630016Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"":"downstairs","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195622024-03-06T21:05:53.365Z{"msg":"[0]R generation[0..12]: [181, 179, 180, 175, 179, 174, 180, 176, 180, 178, 172, 174]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.662644595Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"":"downstairs","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195632024-03-06T21:05:53.365Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.662676053Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"":"downstairs","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195642024-03-06T21:05:53.365Z{"msg":"[1]R flush_numbers[0..12]: [319, 312, 317, 308, 312, 307, 317, 309, 318, 310, 303, 307]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.662703631Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"":"downstairs","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195652024-03-06T21:05:53.366Z{"msg":"[1]R generation[0..12]: [181, 179, 180, 175, 179, 174, 180, 176, 180, 178, 172, 174]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.66272724Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"":"downstairs","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195662024-03-06T21:05:53.366Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.662741979Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"":"downstairs","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195672024-03-06T21:05:53.366Z{"msg":"[2]R flush_numbers[0..12]: [319, 312, 317, 308, 312, 307, 317, 309, 318, 310, 303, 307]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.662767687Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"":"downstairs","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195682024-03-06T21:05:53.366Z{"msg":"[2]R generation[0..12]: [181, 179, 180, 175, 179, 174, 180, 176, 180, 178, 172, 174]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.662787456Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"":"downstairs","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195692024-03-06T21:05:53.366Z{Wait for a query_work_queue command to finish before sending IO
195702024-03-06T21:05:53.366Z"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.662820623Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"":"downstairs","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195712024-03-06T21:05:53.366Z{"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.662849211Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"":"downstairs","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195722024-03-06T21:05:53.366Z{"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.66287289Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"":"downstairs","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195732024-03-06T21:05:53.366Z{"msg":"Next flush: 320","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.662899598Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"":"downstairs","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195742024-03-06T21:05:53.366Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.662922016Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"":"downstairs","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195752024-03-06T21:05:53.366Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.662949474Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195762024-03-06T21:05:53.367Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.662970313Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"client":"0","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195772024-03-06T21:05:53.367Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.662997671Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"client":"1","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195782024-03-06T21:05:53.367Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.6630208Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"client":"2","":"downstairs","session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195792024-03-06T21:05:53.367Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.663057647Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195802024-03-06T21:05:53.367Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.663080835Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195812024-03-06T21:05:53.367Z{"msg":"857adc29-c090-4739-a993-29fb4d29b464 is now active with session: 75b68ddb-db4c-43cd-abdb-24bd51c54a48","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.663107503Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195822024-03-06T21:05:53.367Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.663128682Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867,"session_id":"75b68ddb-db4c-43cd-abdb-24bd51c54a48"}
195832024-03-06T21:05:53.367Z{"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:51.66315449Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867}
195842024-03-06T21:05:53.367ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
195852024-03-06T21:05:53.367ZLoading write count information from file "/tmp/test_repair_verify.data"
195862024-03-06T21:05:53.367ZRun Repair workload
195872024-03-06T21:05:53.368Z01/30 Write block 126 len 20480 data: 17 17 16 17 13
195882024-03-06T21:05:53.381Z02/30 Write block 172 len 24576 data: 15 18 16 17 15 12
195892024-03-06T21:05:53.381Z03/30 Write block 287 len 20480 data: 21 21 20 18 17
195902024-03-06T21:05:53.381Z04/30 Read block 541 len 36864
195912024-03-06T21:05:53.381Z05/30 Read block 521 len 28672
195922024-03-06T21:05:53.382Z06/30 Read block 18 len 12288
195932024-03-06T21:05:53.382Z07/30 Read block 1 len 20480
195942024-03-06T21:05:53.382Z08/30 Write block 110 len 40960 data: 11 12 10 7 10 10 9 12 13 14
195952024-03-06T21:05:53.382Z09/30 Write block 31 len 24576 data: 11 12 12 12 13 11
195962024-03-06T21:05:53.382Z10/30 Read block 564 len 28672
195972024-03-06T21:05:53.382Z11/30 Write block 247 len 20480 data: 11 9 8 7 8
195982024-03-06T21:05:53.382Z12/30 Read block 561 len 8192
195992024-03-06T21:05:53.382Z13/30 Read block 84 len 40960
196002024-03-06T21:05:53.382Z14/30 Read block 141 len 32768
196012024-03-06T21:05:53.382Z15/30 Read block 145 len 24576
196022024-03-06T21:05:53.383Z16/30 Read block 207 len 4096
196032024-03-06T21:05:53.383Z17/30 Read block 32 len 8192
196042024-03-06T21:05:53.383Z18/30 Read block 245 len 12288
196052024-03-06T21:05:53.383Z19/30 Write block 125 len 24576 data: 19 18 18 17 18 14
196062024-03-06T21:05:53.383Z20/30 Read block 73 len 28672
196072024-03-06T21:05:53.383Z21/30 Flush
196082024-03-06T21:05:53.383Z22/30 Write block 356 len 24576 data: 19 16 18 16 17 21
196092024-03-06T21:05:53.383Z23/30 Write block 115 len 32768 data: 11 10 13 14 15 11 10 11
196102024-03-06T21:05:53.383Z24/30 Write block 581 len 32768 data: 14 13 13 14 14 16 16 15
196112024-03-06T21:05:53.383Z25/30 Flush
196122024-03-06T21:05:53.383Z26/30 Write block 490 len 4096 data: 13
196132024-03-06T21:05:53.383Z27/30 Write block 237 len 20480 data: 19 22 21 16 16
196142024-03-06T21:05:53.383Z28/30 Read block 596 len 16384
196152024-03-06T21:05:53.384Z29/30 Write block 55 len 16384 data: 12 9 11 14
196162024-03-06T21:05:53.384Z30/30 Read block 293 len 4096
196172024-03-06T21:05:53.384Z----------------------------------------------------------------
196182024-03-06T21:05:53.384Z Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30
196192024-03-06T21:05:53.384ZStates: Active Active Active
196202024-03-06T21:05:53.384ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
196212024-03-06T21:05:53.384Z 1 Acked 1000 Write 5 Done Done Done false
196222024-03-06T21:05:53.384Z 2 Acked 1001 Write 6 Done Done Done false
196232024-03-06T21:05:53.384Z 3 Acked 1002 Write 5 Done Done Done false
196242024-03-06T21:05:53.384Z 4 Acked 1003 Read 9 Done Done Done false
196252024-03-06T21:05:53.384Z 5 Acked 1004 Read 7 Done Done Done false
196262024-03-06T21:05:53.384Z 6 Acked 1005 Read 3 Done Done Done false
196272024-03-06T21:05:53.385Z 7 Acked 1006 Read 5 Done Done Sent false
196282024-03-06T21:05:53.385Z 8 Acked 1007 Write 10 Done Done Sent false
196292024-03-06T21:05:53.385Z 9 Acked 1008 Write 6 Done Done Sent false
196302024-03-06T21:05:53.385Z 10 Acked 1009 Read 7 Done Done Sent false
196312024-03-06T21:05:53.385Z 11 Acked 1010 Write 5 Done Done Sent false
196322024-03-06T21:05:53.385Z 12 Acked 1011 Read 2 Done Done Sent false
196332024-03-06T21:05:53.385Z 13 Acked 1012 Read 10 Done Done Sent false
196342024-03-06T21:05:53.385Z 14 Acked 1013 Read 8 Done Done Sent false
196352024-03-06T21:05:53.385Z 15 Acked 1014 Read 6 Done Done Sent false
196362024-03-06T21:05:53.385Z 16 Acked 1015 Read 1 Done Done Sent false
196372024-03-06T21:05:53.385Z 17 Acked 1016 Read 2 Done Done Sent false
196382024-03-06T21:05:53.386Z 18 Acked 1017 Read 3 Done Done Sent false
196392024-03-06T21:05:53.386Z 19 Acked 1018 Write 6 Done Done Sent false
196402024-03-06T21:05:53.386Z 20 Acked 1019 Read 7 Done Done Sent false
196412024-03-06T21:05:53.386Z 21 Acked 1020 Flush 0 Done Done Sent false
196422024-03-06T21:05:53.386Z 22 Acked 1021 Write 6 Done Done Sent false
196432024-03-06T21:05:53.386Z 23 Acked 1022 Write 8 Done Done Sent false
196442024-03-06T21:05:53.386Z 24 Acked 1023 Write 8 Done Done Sent false
196452024-03-06T21:05:53.386Z 25 Acked 1024 Flush 0 Done Done Sent false
196462024-03-06T21:05:53.386Z 26 Acked 1025 Write 1 Done Done Sent false
196472024-03-06T21:05:53.387Z 27 Acked 1026 Write 5 Done Done Sent false
196482024-03-06T21:05:53.387Z 28 Acked 1027 Read 4 Done Done Sent false
196492024-03-06T21:05:53.387Z 29 Acked 1028 Write 4 Done Done Sent false
196502024-03-06T21:05:53.387Z 30 Acked 1029 Read 1 Done Done Sent false
196512024-03-06T21:05:53.387Z STATES DS:0 DS:1 DS:2 TOTAL
196522024-03-06T21:05:53.387Z New 0 0 0 0
196532024-03-06T21:05:53.387Z Sent 0 0 24 24
196542024-03-06T21:05:53.387Z Done 30 30 6 66
196552024-03-06T21:05:53.387Z Skipped 0 0 0 0
196562024-03-06T21:05:53.387Z Error 0 0 0 0
196572024-03-06T21:05:53.388ZLast Flush: 1024 1024 0
196582024-03-06T21:05:53.388ZDownstairs last five completed:
196592024-03-06T21:05:53.388ZUpstairs last five completed: 30 29 28 27 26
196602024-03-06T21:05:53.388Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:05:51.699523464Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1867}
196612024-03-06T21:05:53.388ZWrote out file "/tmp/test_repair_verify.data"
196622024-03-06T21:05:53.388Z
196632024-03-06T21:05:53.388ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
196642024-03-06T21:05:53.388ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
196652024-03-06T21:05:53.388Z 0 000-019 181 181 181 319 319 319 F F F
196662024-03-06T21:05:53.388Z 1 020-039 182 182 179 320 320 312 F F F <---
196672024-03-06T21:05:53.389Z 2 040-059 180 180 180 317 317 317 T T F <---
196682024-03-06T21:05:53.389Z 3 060-079 175 175 175 308 308 308 F F F
196692024-03-06T21:05:53.389Z 4 080-099 179 179 179 312 312 312 F F F
196702024-03-06T21:05:53.389Z 5 100-119 182 182 174 321 321 307 F F F <---
196712024-03-06T21:05:53.389Z 6 120-139 182 182 180 321 321 317 F F T <---
196722024-03-06T21:05:53.389Z 7 140-159 176 176 176 309 309 309 F F F
196732024-03-06T21:05:53.389Z 8 160-179 182 182 180 320 320 318 F F T <---
196742024-03-06T21:05:53.389Z 9 180-199 178 178 178 310 310 310 F F F
196752024-03-06T21:05:53.390Z 10 200-219 172 172 172 303 303 303 F F F
196762024-03-06T21:05:53.390Z 11 220-239 174 174 174 307 307 307 T T F <---
196772024-03-06T21:05:53.390Z 12 240-259 182 182 178 320 320 310 T T F <---
196782024-03-06T21:05:53.390Z 13 260-279 171 171 171 301 301 301 F F F
196792024-03-06T21:05:53.390Z 14 280-299 182 182 175 320 320 308 F F T <---
196802024-03-06T21:05:53.390Z 15 300-319 181 181 181 319 319 319 F F F
196812024-03-06T21:05:53.390Z 16 320-339 176 176 176 309 309 309 F F F
196822024-03-06T21:05:53.390Z 17 340-359 182 182 178 321 321 311 F F F <---
196832024-03-06T21:05:53.391Z 18 360-379 182 182 180 321 321 317 F F F <---
196842024-03-06T21:05:53.391Z 19 380-399 180 180 180 317 317 317 F F F
196852024-03-06T21:05:53.391Z 20 400-419 176 176 176 309 309 309 F F F
196862024-03-06T21:05:53.391Z 21 420-439 176 176 176 309 309 309 F F F
196872024-03-06T21:05:53.391Z 22 440-459 176 176 176 309 309 309 F F F
196882024-03-06T21:05:53.496Z 23 460-479 179 179 179 312 312 312 F F F
196892024-03-06T21:05:53.497Z 24 480-499 179 179 179 312 312 312 T T F <---
196902024-03-06T21:05:53.497Z 25 500-519 175 175 175 308 308 308 F F F
196912024-03-06T21:05:53.497Z 26 520-539 179 179 179 312 312 312 F F F
196922024-03-06T21:05:53.497Z 27 540-559 181 181 181 319 319 319 F F F
196932024-03-06T21:05:53.497Z 28 560-579 180 180 180 318 318 318 F F F
196942024-03-06T21:05:53.497Z 29 580-599 182 182 178 321 321 311 F F F <---
196952024-03-06T21:05:53.497ZMax gen: 182, Max flush: 321
196962024-03-06T21:05:53.497ZError: Difference in extent metadata found!
196972024-03-06T21:05:53.497ZA Difference in extent metadata is expected here
196982024-03-06T21:05:53.498ZOn loop 91
196992024-03-06T21:05:53.498Z
197002024-03-06T21:05:53.498ZVerifying data now
197012024-03-06T21:05:58.652ZNo change after verify
197022024-03-06T21:05:58.652ZLoop: 91 Downstairs dump after verify (and repair):
197032024-03-06T21:05:58.667ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
197042024-03-06T21:05:58.667Z 0 000-019 181 181 181 319 319 319 F F F
197052024-03-06T21:05:58.667Z 1 020-039 182 182 182 320 320 320 F F F
197062024-03-06T21:05:58.667Z 2 040-059 183 183 183 322 322 322 F F F
197072024-03-06T21:05:58.667Z 3 060-079 175 175 175 308 308 308 F F F
197082024-03-06T21:05:58.667Z 4 080-099 179 179 179 312 312 312 F F F
197092024-03-06T21:05:58.667Z 5 100-119 182 182 182 321 321 321 F F F
197102024-03-06T21:05:58.667Z 6 120-139 182 182 182 321 321 321 F F F
197112024-03-06T21:05:58.667Z 7 140-159 176 176 176 309 309 309 F F F
197122024-03-06T21:05:58.667Z 8 160-179 182 182 182 320 320 320 F F F
197132024-03-06T21:05:58.668Z 9 180-199 178 178 178 310 310 310 F F F
197142024-03-06T21:05:58.668Z 10 200-219 172 172 172 303 303 303 F F F
197152024-03-06T21:05:58.668Z 11 220-239 183 183 183 322 322 322 F F F
197162024-03-06T21:05:58.668Z 12 240-259 183 183 183 322 322 322 F F F
197172024-03-06T21:05:58.668Z 13 260-279 171 171 171 301 301 301 F F F
197182024-03-06T21:05:58.668Z 14 280-299 182 182 182 320 320 320 F F F
197192024-03-06T21:05:58.668Z 15 300-319 181 181 181 319 319 319 F F F
197202024-03-06T21:05:58.668Z 16 320-339 176 176 176 309 309 309 F F F
197212024-03-06T21:05:58.668Z 17 340-359 182 182 182 321 321 321 F F F
197222024-03-06T21:05:58.672Z 18 360-379 182 182 182 321 321 321 F F F
197232024-03-06T21:05:58.672Z 19 380-399 180 180 180 317 317 317 F F F
197242024-03-06T21:05:58.672Z 20 400-419 176 176 176 309 309 309 F F F
197252024-03-06T21:05:58.672Z 21 420-439 176 176 176 309 309 309 F F F
197262024-03-06T21:05:58.672Z 22 440-459 176 176 176 309 309 309 F F F
197272024-03-06T21:05:58.672Z 23 460-479 179 179 179 312 312 312 F F F
197282024-03-06T21:05:58.672Z 24 480-499 183 183 183 322 322 322 F F F
197292024-03-06T21:05:58.672Z 25 500-519 175 175 175 308 308 308 F F F
197302024-03-06T21:05:58.672Z 26 520-539 179 179 179 312 312 312 F F F
197312024-03-06T21:05:58.672Z 27 540-559 181 181 181 319 319 319 F F F
197322024-03-06T21:05:58.672Z 28 560-579 180 180 180 318 318 318 F F F
197332024-03-06T21:05:58.672Z 29 580-599 182 182 182 321 321 321 F F F
197342024-03-06T21:05:58.672ZMax gen: 183, Max flush: 322
197352024-03-06T21:05:58.672Z
197362024-03-06T21:05:58.672ZBegin loop 92
197372024-03-06T21:05:58.672ZDownstairs to restart: 1
197382024-03-06T21:05:58.695Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.432213985Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875}
197392024-03-06T21:05:58.695Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.432508865Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875}
197402024-03-06T21:05:58.695Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
197412024-03-06T21:05:58.700Z,"time":"2024-03-06T21:05:57.432548372Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875}
197422024-03-06T21:05:58.700Z{"msg":"Crucible 34de6207-ef1f-454e-92bb-eef5c5dd3845 has session id: c29411fb-2be3-4d60-a13b-7d3654db0aec","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.432648745Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197432024-03-06T21:05:58.700Z{"msg":"Upstairs opts: Upstairs UUID: 34de6207-ef1f-454e-92bb-eef5c5dd3845, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.43271386Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197442024-03-06T21:05:58.700Z{"msg":"Crucible stats registered with UUID: 34de6207-ef1f-454e-92bb-eef5c5dd3845","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.432757037Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197452024-03-06T21:05:58.700Z{"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.432796555Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875}
197462024-03-06T21:05:58.700Z{"msg":"34de6207-ef1f-454e-92bb-eef5c5dd3845 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.432821093Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197472024-03-06T21:05:58.700Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.43285521Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"client":"0","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197482024-03-06T21:05:58.700Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.432875769Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"client":"0","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197492024-03-06T21:05:58.701Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.432902237Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"client":"1","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197502024-03-06T21:05:58.701Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.432920636Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"client":"1","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197512024-03-06T21:05:58.701Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.432957754Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"client":"2","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197522024-03-06T21:05:58.701Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.433027999Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"client":"2","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197532024-03-06T21:05:58.701Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.433058967Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"":"io task","client":"2","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197542024-03-06T21:05:58.701Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.433105923Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"":"io task","client":"0","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197552024-03-06T21:05:58.701Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.433134341Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"":"io task","client":"1","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197562024-03-06T21:05:58.701Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.433162729Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"":"io task","client":"2","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197572024-03-06T21:05:58.701Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.433189067Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"":"io task","client":"0","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197582024-03-06T21:05:58.701Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.433323618Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"":"io task","client":"1","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197592024-03-06T21:05:58.701Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.434111344Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"client":"2","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197602024-03-06T21:05:58.701Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.434148812Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"client":"2","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197612024-03-06T21:05:58.702Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.43416491Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"client":"0","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197622024-03-06T21:05:58.702Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.434225046Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"client":"0","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197632024-03-06T21:05:58.702Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.434781518Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"client":"2","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197642024-03-06T21:05:58.702Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.434945586Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"client":"0","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197652024-03-06T21:05:58.702Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.43533136Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"client":"2","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197662024-03-06T21:05:58.702Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.435376867Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"client":"2","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197672024-03-06T21:05:58.702Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.435399235Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197682024-03-06T21:05:58.702Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.435518437Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"client":"0","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197692024-03-06T21:05:58.702Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.435551575Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"client":"0","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197702024-03-06T21:05:58.702Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.435566704Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197712024-03-06T21:05:58.702Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.435579873Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"client":"1","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197722024-03-06T21:05:58.702Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.435593472Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"client":"1","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197732024-03-06T21:05:58.722Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.436101777Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"client":"1","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197742024-03-06T21:05:58.722Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.436520838Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"client":"1","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197752024-03-06T21:05:58.722Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.436562035Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"client":"1","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197762024-03-06T21:05:58.722Z{"msg":"[0]R flush_numbers[0..12]: [319, 320, 322, 308, 312, 321, 321, 309, 320, 310, 303, 322]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.436584024Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"":"downstairs","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197772024-03-06T21:05:58.722Z{"msg":"[0]R generation[0..12]: [181, 182, 183, 175, 179, 182, 182, 176, 182, 178, 172, 183]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.436598972Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"":"downstairs","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197782024-03-06T21:05:58.723Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.436622661Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"":"downstairs","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197792024-03-06T21:05:58.723Z{"msg":"[1]R flush_numbers[0..12]: [319, 320, 322, 308, 312, 321, 321, 309, 320, 310, 303, 322]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.43663746Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"":"downstairs","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197802024-03-06T21:05:58.723Z{"msg":"[1]R generation[0..12]: [181, 182, 183, 175, 179, 182, 182, 176, 182, 178, 172, 183]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.436658559Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"":"downstairs","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197812024-03-06T21:05:58.723Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.436673107Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"":"downstairs","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197822024-03-06T21:05:58.723Z{"msg":"[2]R flush_numbers[0..12]: [319, 320, 322, 308, 312, 321, 321, 309, 320, 310, 303, 322]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.436694616Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"":"downstairs","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197832024-03-06T21:05:58.723Z{"msg":"[2]R generation[0..12]: [181, 182, 183, 175, 179, 182, 182, 176, 182, 178, 172, 183]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.436709575Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"":"downstairs","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197842024-03-06T21:05:58.723Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.436730294Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"":"downstairs","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197852024-03-06T21:05:58.723Z{"msg":"Max found gen is 184","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
197862024-03-06T21:05:58.723Z,"time":"2024-03-06T21:05:57.436743653Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"":"downstairs","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197872024-03-06T21:05:58.723Z{"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.436759361Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"":"downstairs","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197882024-03-06T21:05:58.723Z{"msg":"Next flush: 323","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.43677256Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"":"downstairs","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197892024-03-06T21:05:58.723Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.436792329Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"":"downstairs","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197902024-03-06T21:05:58.723Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.436805988Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197912024-03-06T21:05:58.723Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.436824307Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"client":"0","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197922024-03-06T21:05:58.723Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.436837866Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"client":"1","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197932024-03-06T21:05:58.724Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.436851665Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"client":"2","":"downstairs","session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197942024-03-06T21:05:58.724Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.436865634Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197952024-03-06T21:05:58.724Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.436878603Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197962024-03-06T21:05:58.724Z{"msg":"34de6207-ef1f-454e-92bb-eef5c5dd3845 is now active with session: c29411fb-2be3-4d60-a13b-7d3654db0aec","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.436891362Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197972024-03-06T21:05:58.724Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.43693063Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875,"session_id":"c29411fb-2be3-4d60-a13b-7d3654db0aec"}
197982024-03-06T21:05:58.724Z{"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:05:57.436966297Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875}
197992024-03-06T21:05:58.724ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
198002024-03-06T21:05:58.724ZLoading write count information from file "/tmp/test_repair_verify.data"
198012024-03-06T21:05:58.724ZRun Repair workload
198022024-03-06T21:05:58.724Z01/30 Write block 584 len 40960 data: 15 15 17 17 16 15 15 17 17 12
198032024-03-06T21:05:58.724Z02/30 Write block 12 len 16384 data: 15 16 18 20
198042024-03-06T21:05:58.724Z03/30 Read block 133 len 40960
198052024-03-06T21:05:58.724Z04/30 Read block 172 len 8192
198062024-03-06T21:05:58.725Z05/30 Read block 590 len 24576
198072024-03-06T21:05:58.725Z06/30 Read block 391 len 4096
198082024-03-06T21:05:58.725Z07/30 Read block 561 len 4096
198092024-03-06T21:05:58.725Z08/30 Read block 121 len 28672
198102024-03-06T21:05:58.725Z09/30 Write block 189 len 4096 data: 18
198112024-03-06T21:05:58.741Z10/30 Read block 360 len 4096
198122024-03-06T21:05:58.741Z11/30 Write block 555 len 12288 data: 8 7 9
198132024-03-06T21:05:58.741Z12/30 Read block 194 len 24576
198142024-03-06T21:05:58.741Z13/30 Read block 37 len 24576
198152024-03-06T21:05:58.741Z14/30 Read block 487 len 36864
198162024-03-06T21:05:58.741Z15/30 Write block 98 len 4096 data: 14
198172024-03-06T21:05:58.741Z16/30 Write block 497 len 24576 data: 17 14 12 11 7 8
198182024-03-06T21:05:58.741Z17/30 Read block 161 len 4096
198192024-03-06T21:05:58.742Z18/30 Flush
198202024-03-06T21:05:58.742Z19/30 Write block 338 len 24576 data: 16 14 18 17 16 18
198212024-03-06T21:05:58.742Z20/30 Write block 481 len 4096 data: 11
198222024-03-06T21:05:58.742Z21/30 Read block 156 len 20480
198232024-03-06T21:05:58.742Z22/30 Read block 35 len 40960
198242024-03-06T21:05:58.742Z23/30 Read block 517 len 40960
198252024-03-06T21:05:58.742Z24/30 Read block 321 len 4096
198262024-03-06T21:05:58.742Z25/30 Read block 216 len 4096
198272024-03-06T21:05:58.742Z26/30 Write block 398 len 36864 data: 11 7 9 9 9 12 10 13 13
198282024-03-06T21:05:58.742Z27/30 Read block 288 len 20480
198292024-03-06T21:05:58.742Z28/30 Read block 509 len 20480
198302024-03-06T21:05:58.742Z29/30 Read block 342 len 16384
198312024-03-06T21:05:58.742Z30/30 Read block 33 len 24576
198322024-03-06T21:05:58.742Z----------------------------------------------------------------
198332024-03-06T21:05:58.742Z Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:30
198342024-03-06T21:05:58.742ZStates: Active Active Active
198352024-03-06T21:05:58.742ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
198362024-03-06T21:05:58.742Z 1 Acked 1000 Write 10 Done Done Done false
198372024-03-06T21:05:58.742Z 2 Acked 1001 Write 4 Done Done Done false
198382024-03-06T21:05:58.742Z 3 Acked 1002 Read 10 Done Done Done false
198392024-03-06T21:05:58.742Z 4 Acked 1003 Read 2 Done Sent Done false
198402024-03-06T21:05:58.742Z 5 Acked 1004 Read 6 Done Sent Done false
198412024-03-06T21:05:58.742Z 6 Acked 1005 Read 1 Done Sent Done false
198422024-03-06T21:05:58.742Z 7 Acked 1006 Read 1 Done Sent Done false
198432024-03-06T21:05:58.743Z 8 Acked 1007 Read 7 Done Sent Done false
198442024-03-06T21:05:58.743Z 9 Acked 1008 Write 1 Done Sent Done false
198452024-03-06T21:05:58.743Z 10 Acked 1009 Read 1 Done Sent Done false
198462024-03-06T21:05:58.743Z 11 Acked 1010 Write 3 Done Sent Done false
198472024-03-06T21:05:58.743Z 12 Acked 1011 Read 6 Done Sent Done false
198482024-03-06T21:05:58.743Z 13 Acked 1012 Read 6 Done Sent Done false
198492024-03-06T21:05:58.743Z 14 Acked 1013 Read 9 Done Sent Done false
198502024-03-06T21:05:58.743Z 15 Acked 1014 Write 1 Done Sent Done false
198512024-03-06T21:05:58.743Z 16 Acked 1015 Write 6 Done Sent Done false
198522024-03-06T21:05:58.743Z 17 Acked 1016 Read 1 Done Sent Done false
198532024-03-06T21:05:58.743Z 18 Acked 1017 Flush 0 Done Sent Done false
198542024-03-06T21:05:58.743Z 19 Acked 1018 Write 6 Done Sent Done false
198552024-03-06T21:05:58.743Z 20 Acked 1019 Write 1 Done Sent Done false
198562024-03-06T21:05:58.743Z 21 Acked 1020 Read 5 Done Sent Done false
198572024-03-06T21:05:58.743Z 22 Acked 1021 Read 10 Done Sent Done false
198582024-03-06T21:05:58.743Z 23 Acked 1022 Read 10 Done Sent Done false
198592024-03-06T21:05:58.743Z 24 Acked 1023 Read 1 Done Sent Done false
198602024-03-06T21:05:58.743Z 25 Acked 1024 Read 1 Done Sent Done false
198612024-03-06T21:05:58.743Z 26 Acked 1025 Write 9 Done Sent Done false
198622024-03-06T21:05:58.743Z 27 Acked 1026 Read 5 Done Sent Done false
198632024-03-06T21:05:58.743Z 28 Acked 1027 Read 5 Done Sent Done false
198642024-03-06T21:05:58.743Z 29 Acked 1028 Read 4 Done Sent Done false
198652024-03-06T21:05:58.743Z 30 Acked 1029 Read 6 Done Sent Done false
198662024-03-06T21:05:58.744Z STATES DS:0 DS:1 DS:2 TOTAL
198672024-03-06T21:05:58.744Z New 0 0 0 0
198682024-03-06T21:05:58.744Z Sent 0 27 0 27
198692024-03-06T21:05:58.744Z Done 30 3 30 63
198702024-03-06T21:05:58.744Z Skipped 0 0 0 0
198712024-03-06T21:05:58.744Z Error 0 0 0 0
198722024-03-06T21:05:58.744ZLast Flush: 1017 0 1017
198732024-03-06T21:05:58.744ZDownstairs last five completed:
198742024-03-06T21:05:58.744ZUpstairs last five completed: 30 29 28 27 26
198752024-03-06T21:05:58.744Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:05:57.468438588Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1875}
198762024-03-06T21:05:58.744ZWrote out file "/tmp/test_repair_verify.data"
198772024-03-06T21:05:58.748Z
198782024-03-06T21:05:58.748ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
198792024-03-06T21:05:58.759ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
198802024-03-06T21:05:58.759Z 0 000-019 184 181 184 323 319 323 F T F <---
198812024-03-06T21:05:58.759Z 1 020-039 182 182 182 320 320 320 F F F
198822024-03-06T21:05:58.759Z 2 040-059 183 183 183 322 322 322 F F F
198832024-03-06T21:05:58.759Z 3 060-079 175 175 175 308 308 308 F F F
198842024-03-06T21:05:58.759Z 4 080-099 184 179 184 323 312 323 F F F <---
198852024-03-06T21:05:58.759Z 5 100-119 182 182 182 321 321 321 F F F
198862024-03-06T21:05:58.759Z 6 120-139 182 182 182 321 321 321 F F F
198872024-03-06T21:05:58.759Z 7 140-159 176 176 176 309 309 309 F F F
198882024-03-06T21:05:58.759Z 8 160-179 182 182 182 320 320 320 F F F
198892024-03-06T21:05:58.760Z 9 180-199 184 178 184 323 310 323 F F F <---
198902024-03-06T21:05:58.760Z 10 200-219 172 172 172 303 303 303 F F F
198912024-03-06T21:05:58.760Z 11 220-239 183 183 183 322 322 322 F F F
198922024-03-06T21:05:58.760Z 12 240-259 183 183 183 322 322 322 F F F
198932024-03-06T21:05:58.760Z 13 260-279 171 171 171 301 301 301 F F F
198942024-03-06T21:05:58.760Z 14 280-299 182 182 182 320 320 320 F F F
198952024-03-06T21:05:58.760Z 15 300-319 181 181 181 319 319 319 F F F
198962024-03-06T21:05:58.760Z 16 320-339 176 176 176 309 309 309 T F T <---
198972024-03-06T21:05:58.760Z 17 340-359 182 182 182 321 321 321 T F T <---
198982024-03-06T21:05:58.760Z 18 360-379 182 182 182 321 321 321 F F F
198992024-03-06T21:05:58.760Z 19 380-399 180 180 180 317 317 317 T F T <---
199002024-03-06T21:05:58.760Z 20 400-419 176 176 176 309 309 309 T F T <---
199012024-03-06T21:05:58.760Z 21 420-439 176 176 176 309 309 309 F F F
199022024-03-06T21:05:58.760Z 22 440-459 176 176 176 309 309 309 F F F
199032024-03-06T21:05:58.760Z 23 460-479 179 179 179 312 312 312 F F F
199042024-03-06T21:05:58.760Z 24 480-499 184 183 184 323 322 323 T F T <---
199052024-03-06T21:05:58.760Z 25 500-519 184 175 184 323 308 323 F F F <---
199062024-03-06T21:05:58.760Z 26 520-539 179 179 179 312 312 312 F F F
199072024-03-06T21:05:58.764Z 27 540-559 184 181 184 323 319 323 F F F <---
199082024-03-06T21:05:58.764Z 28 560-579 180 180 180 318 318 318 F F F
199092024-03-06T21:05:58.764Z 29 580-599 184 182 184 323 321 323 F T F <---
199102024-03-06T21:05:58.764ZMax gen: 184, Max flush: 323
199112024-03-06T21:05:58.764ZError: Difference in extent metadata found!
199122024-03-06T21:05:58.767ZA Difference in extent metadata is expected here
199132024-03-06T21:05:58.767ZOn loop 92
199142024-03-06T21:05:58.767Z
199152024-03-06T21:05:58.771ZVerifying data now
199162024-03-06T21:06:04.399ZNo change after verify
199172024-03-06T21:06:04.400ZLoop: 92 Downstairs dump after verify (and repair):
199182024-03-06T21:06:04.410ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
199192024-03-06T21:06:04.410Z 0 000-019 184 184 184 323 323 323 F F F
199202024-03-06T21:06:04.410Z 1 020-039 182 182 182 320 320 320 F F F
199212024-03-06T21:06:04.410Z 2 040-059 183 183 183 322 322 322 F F F
199222024-03-06T21:06:04.411Z 3 060-079 175 175 175 308 308 308 F F F
199232024-03-06T21:06:04.411Z 4 080-099 184 184 184 323 323 323 F F F
199242024-03-06T21:06:04.411Z 5 100-119 182 182 182 321 321 321 F F F
199252024-03-06T21:06:04.411Z 6 120-139 182 182 182 321 321 321 F F F
199262024-03-06T21:06:04.411Z 7 140-159 176 176 176 309 309 309 F F F
199272024-03-06T21:06:04.411Z 8 160-179 182 182 182 320 320 320 F F F
199282024-03-06T21:06:04.411Z 9 180-199 184 184 184 323 323 323 F F F
199292024-03-06T21:06:04.411Z 10 200-219 172 172 172 303 303 303 F F F
199302024-03-06T21:06:04.411Z 11 220-239 183 183 183 322 322 322 F F F
199312024-03-06T21:06:04.411Z 12 240-259 183 183 183 322 322 322 F F F
199322024-03-06T21:06:04.412Z 13 260-279 171 171 171 301 301 301 F F F
199332024-03-06T21:06:04.412Z 14 280-299 182 182 182 320 320 320 F F F
199342024-03-06T21:06:04.412Z 15 300-319 181 181 181 319 319 319 F F F
199352024-03-06T21:06:04.412Z 16 320-339 185 185 185 324 324 324 F F F
199362024-03-06T21:06:04.412Z 17 340-359 185 185 185 324 324 324 F F F
199372024-03-06T21:06:04.412Z 18 360-379 182 182 182 321 321 321 F F F
199382024-03-06T21:06:04.412Z 19 380-399 185 185 185 324 324 324 F F F
199392024-03-06T21:06:04.412Z 20 400-419 185 185 185 324 324 324 F F F
199402024-03-06T21:06:04.412Z 21 420-439 176 176 176 309 309 309 F F F
199412024-03-06T21:06:04.412Z 22 440-459 176 176 176 309 309 309 F F F
199422024-03-06T21:06:04.413Z 23 460-479 179 179 179 312 312 312 F F F
199432024-03-06T21:06:04.413Z 24 480-499 185 185 185 324 324 324 F F F
199442024-03-06T21:06:04.413Z 25 500-519 184 184 184 323 323 323 F F F
199452024-03-06T21:06:04.413Z 26 520-539 179 179 179 312 312 312 F F F
199462024-03-06T21:06:04.413Z 27 540-559 184 184 184 323 323 323 F F F
199472024-03-06T21:06:04.413Z 28 560-579 180 180 180 318 318 318 F F F
199482024-03-06T21:06:04.413Z 29 580-599 184 184 184 323 323 323 F F F
199492024-03-06T21:06:04.413ZMax gen: 185, Max flush: 324
199502024-03-06T21:06:04.414Z
199512024-03-06T21:06:04.414ZBegin loop 93
199522024-03-06T21:06:04.414ZDownstairs to restart: 1
199532024-03-06T21:06:04.459Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.161030182Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883}
199542024-03-06T21:06:04.460Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.161332911Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883}
199552024-03-06T21:06:04.460Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.161379578Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883}
199562024-03-06T21:06:04.460Z{"msg":"Crucible runtime is spawned
199572024-03-06T21:06:04.460ZCrucible 77f25ded-b9d7-4087-9085-ebbfe0778578 has session id: 385335b9-013f-46b7-9683-39d8d042e429","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.161414765Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199582024-03-06T21:06:04.460Z{"msg":"Upstairs opts: Upstairs UUID: 77f25ded-b9d7-4087-9085-ebbfe0778578, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.161468722Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199592024-03-06T21:06:04.461Z{"msg":"Crucible stats registered with UUID: 77f25ded-b9d7-4087-9085-ebbfe0778578","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.161516188Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199602024-03-06T21:06:04.461Z{"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.161536517Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883}
199612024-03-06T21:06:04.461Z{"msg":"77f25ded-b9d7-4087-9085-ebbfe0778578 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.161706315Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199622024-03-06T21:06:04.461Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.161732703Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"client":"0","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199632024-03-06T21:06:04.461Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.161749692Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"client":"0","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199642024-03-06T21:06:04.462Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.161772271Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"client":"1","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199652024-03-06T21:06:04.462Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.16178832Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"client":"1","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199662024-03-06T21:06:04.462Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.161818438Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"client":"2","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199672024-03-06T21:06:04.462Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.161842806Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"client":"2","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199682024-03-06T21:06:04.462Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.161865834Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"":"io task","client":"2","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199692024-03-06T21:06:04.462Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.161897212Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"":"io task","client":"0","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199702024-03-06T21:06:04.462Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.161942659Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"":"io task","client":"1","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199712024-03-06T21:06:04.462Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.162109667Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"":"io task","client":"2","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199722024-03-06T21:06:04.463Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.162169523Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"":"io task","client":"0","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199732024-03-06T21:06:04.463Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.16221505Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"":"io task","client":"1","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199742024-03-06T21:06:04.463Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.162867355Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"client":"0","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199752024-03-06T21:06:04.463Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.162914302Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"client":"0","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199762024-03-06T21:06:04.463Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.16293976Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"client":"2","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199772024-03-06T21:06:04.463Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.162972488Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"client":"2","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199782024-03-06T21:06:04.464Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.163470944Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"client":"2","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199792024-03-06T21:06:04.464Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.163603884Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"client":"0","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199802024-03-06T21:06:04.464Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.16395395Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"client":"2","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199812024-03-06T21:06:04.464Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.163989238Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"client":"2","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199822024-03-06T21:06:04.464Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.164039754Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199832024-03-06T21:06:04.465Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.164135168Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"client":"1","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199842024-03-06T21:06:04.465Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.164170635Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"client":"1","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199852024-03-06T21:06:04.465Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.164202053Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"client":"0","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199862024-03-06T21:06:04.465Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.164223102Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"client":"0","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199872024-03-06T21:06:04.465Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.16425352Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199882024-03-06T21:06:04.465Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.164702009Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"client":"1","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199892024-03-06T21:06:04.466Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.165169547Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"client":"1","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199902024-03-06T21:06:04.466Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.165206864Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"client":"1","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199912024-03-06T21:06:04.466Z{"msg":"[0]R flush_numbers[0..12]: [323, 320, 322, 308, 323, 321, 321, 309, 320, 323, 303, 322]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.165224553Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"":"downstairs","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199922024-03-06T21:06:04.466Z{"msg":"[0]R generation[0..12]: [184, 182, 183, 175, 184, 182, 182, 176, 182, 184, 172, 183]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.165248351Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"":"downstairs","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199932024-03-06T21:06:04.466Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.16526393Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"":"downstairs","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199942024-03-06T21:06:04.466Z{"msg":"[1]R flush_numbers[0..12]: [323, 320, 322, 308, 323, 321, 321, 309, 320, 323, 303, 322]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.165286059Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"":"downstairs","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199952024-03-06T21:06:04.466Z{"msg":"[1]R generation[0..12]: [184, 182, 183, 175, 184, 182, 182, 176, 182, 184, 172, 183]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.165300367Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"":"downstairs","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199962024-03-06T21:06:04.466ZWait for a query_work_queue command to finish before sending IO
199972024-03-06T21:06:04.466Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.165327176Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"":"downstairs","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199982024-03-06T21:06:04.467Z{"msg":"[2]R flush_numbers[0..12]: [323, 320, 322, 308, 323, 321, 321, 309, 320, 323, 303, 322]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.165342975Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"":"downstairs","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
199992024-03-06T21:06:04.467Z{"msg":"[2]R generation[0..12]: [184, 182, 183, 175, 184, 182, 182, 176, 182, 184, 172, 183]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.165363223Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"":"downstairs","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
200002024-03-06T21:06:04.467Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.165377502Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"":"downstairs","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
200012024-03-06T21:06:04.467Z{"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.165403211Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"":"downstairs","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
200022024-03-06T21:06:04.467Z{"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.165438458Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"":"downstairs","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
200032024-03-06T21:06:04.467Z{"msg":"Next flush: 325","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.165472946Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"":"downstairs","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
200042024-03-06T21:06:04.467Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.165502933Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"":"downstairs","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
200052024-03-06T21:06:04.467Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.165525932Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
200062024-03-06T21:06:04.467Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.16555756Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"client":"0","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
200072024-03-06T21:06:04.467Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
200082024-03-06T21:06:04.467Z2024-03-06T21:06:03.165590718Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"client":"1","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
200092024-03-06T21:06:04.467Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.165631765Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"client":"2","":"downstairs","session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
200102024-03-06T21:06:04.468Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.165656263Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
200112024-03-06T21:06:04.468Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.165670002Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
200122024-03-06T21:06:04.468Z{"msg":"77f25ded-b9d7-4087-9085-ebbfe0778578 is now active with session: 385335b9-013f-46b7-9683-39d8d042e429","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.165690701Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
200132024-03-06T21:06:04.468Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.16570304Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883,"session_id":"385335b9-013f-46b7-9683-39d8d042e429"}
200142024-03-06T21:06:04.468Z{"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:03.165721929Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883}
200152024-03-06T21:06:04.468ZLoading write count information from file "/tmp/test_repair_verify.data"
200162024-03-06T21:06:04.468ZRun Repair workload
200172024-03-06T21:06:04.468Z01/30 Write block 475 len 32768 data: 8 9 9 12 7 10 12 12
200182024-03-06T21:06:04.468Z02/30 Write block 457 len 16384 data: 17 16 18 18
200192024-03-06T21:06:04.469Z03/30 Read block 233 len 40960
200202024-03-06T21:06:04.469Z04/30 Write block 554 len 32768 data: 8 9 8 10 9 9 13 13
200212024-03-06T21:06:04.469Z05/30 Write block 475 len 12288 data: 9 10 10
200222024-03-06T21:06:04.469Z06/30 Read block 215 len 20480
200232024-03-06T21:06:04.469Z07/30 Read block 507 len 8192
200242024-03-06T21:06:04.469Z08/30 Read block 424 len 36864
200252024-03-06T21:06:04.469Z09/30 Write block 84 len 36864 data: 9 9 10 10 10 10 8 11 11
200262024-03-06T21:06:04.469Z10/30 Write block 179 len 40960 data: 18 18 20 18 17 17 15 13 12 12
200272024-03-06T21:06:04.470Z11/30 Read block 479 len 36864
200282024-03-06T21:06:04.470Z12/30 Flush
200292024-03-06T21:06:04.470Z13/30 Write block 593 len 24576 data: 13 13 10 8 5 4
200302024-03-06T21:06:04.470Z14/30 Read block 302 len 28672
200312024-03-06T21:06:04.485Z15/30 Flush
200322024-03-06T21:06:04.485Z16/30 Write block 583 len 12288 data: 14 16 16
200332024-03-06T21:06:04.485Z17/30 Write block 40 len 24576 data: 14 18 19 18 17 12
200342024-03-06T21:06:04.485Z18/30 Write block 316 len 40960 data: 12 10 14 14 13 16 15 12 14 13
200352024-03-06T21:06:04.485Z19/30 Write block 220 len 4096 data: 8
200362024-03-06T21:06:04.485Z20/30 Write block 60 len 8192 data: 18 21
200372024-03-06T21:06:04.485Z21/30 Write block 37 len 24576 data: 7 10 12 15 19 20
200382024-03-06T21:06:04.485Z22/30 Read block 37 len 12288
200392024-03-06T21:06:04.486Z23/30 Write block 230 len 24576 data: 15 16 14 13 13 16
200402024-03-06T21:06:04.486Z24/30 Write block 81 len 28672 data: 12 12 13 10 10 11 11
200412024-03-06T21:06:04.486Z25/30 Write block 364 len 8192 data: 17 15
200422024-03-06T21:06:04.486Z26/30 Write block 225 len 24576 data: 7 10 13 14 13 16
200432024-03-06T21:06:04.486Z27/30 Read block 23 len 16384
200442024-03-06T21:06:04.486Z28/30 Read block 54 len 36864
200452024-03-06T21:06:04.486Z29/30 Read block 580 len 8192
200462024-03-06T21:06:04.486Z30/30 Write block 168 len 8192 data: 14 14
200472024-03-06T21:06:04.486Z----------------------------------------------------------------
200482024-03-06T21:06:04.487Z Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:30
200492024-03-06T21:06:04.487ZStates: Active Active Active
200502024-03-06T21:06:04.487ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
200512024-03-06T21:06:04.487Z 1 Acked 1000 Write 8 Done Sent Done false
200522024-03-06T21:06:04.487Z 2 Acked 1001 Write 4 Done Sent Done false
200532024-03-06T21:06:04.487Z 3 Acked 1002 Read 10 Done Sent Done false
200542024-03-06T21:06:04.487Z 4 Acked 1003 Write 8 Done Sent Done false
200552024-03-06T21:06:04.487Z 5 Acked 1004 Write 3 Done Sent Done false
200562024-03-06T21:06:04.488Z 6 Acked 1005 Read 5 Done Sent Done false
200572024-03-06T21:06:04.488Z 7 Acked 1006 Read 2 Done Sent Done false
200582024-03-06T21:06:04.488Z 8 Acked 1007 Read 9 Done Sent Done false
200592024-03-06T21:06:04.488Z 9 Acked 1008 Write 9 Done Sent Done false
200602024-03-06T21:06:04.488Z 10 Acked 1009 Write 10 Done Sent Done false
200612024-03-06T21:06:04.488Z 11 Acked 1010 Read 9 Done Sent Done false
200622024-03-06T21:06:04.488Z 12 Acked 1011 Flush 0 Done Sent Done false
200632024-03-06T21:06:04.488Z 13 Acked 1012 Write 6 Done Sent Done false
200642024-03-06T21:06:04.488Z 14 Acked 1013 Read 7 Done Sent Done false
200652024-03-06T21:06:04.488Z 15 Acked 1014 Flush 0 Done Sent Done false
200662024-03-06T21:06:04.488Z 16 Acked 1015 Write 3 Done Sent Done false
200672024-03-06T21:06:04.488Z 17 Acked 1016 Write 6 Done Sent Done false
200682024-03-06T21:06:04.488Z 18 Acked 1017 Write 10 Done Sent Done false
200692024-03-06T21:06:04.488Z 19 Acked 1018 Write 1 Done Sent Done false
200702024-03-06T21:06:04.489Z 20 Acked 1019 Write 2 Done Sent Done false
200712024-03-06T21:06:04.489Z 21 Acked 1020 Write 6 Done Sent Done false
200722024-03-06T21:06:04.489Z 22 Acked 1021 Read 3 Done Sent Done false
200732024-03-06T21:06:04.489Z 23 Acked 1022 Write 6 Done Sent Done false
200742024-03-06T21:06:04.489Z 24 Acked 1023 Write 7 Done Sent Done false
200752024-03-06T21:06:04.489Z 25 Acked 1024 Write 2 Done Sent Done false
200762024-03-06T21:06:04.489Z 26 Acked 1025 Write 6 Done Sent Done false
200772024-03-06T21:06:04.489Z 27 Acked 1026 Read 4 Done Sent Done false
200782024-03-06T21:06:04.489Z 28 Acked 1027 Read 9 Done Sent Done false
200792024-03-06T21:06:04.489Z 29 Acked 1028 Read 2 Done Sent Done false
200802024-03-06T21:06:04.489Z 30 Acked 1029 Write 2 Sent Sent Sent false
200812024-03-06T21:06:04.490Z STATES DS:0 DS:1 DS:2 TOTAL
200822024-03-06T21:06:04.490Z New 0 0 0 0
200832024-03-06T21:06:04.490Z Sent 1 30 1 32
200842024-03-06T21:06:04.490Z Done 29 0 29 58
200852024-03-06T21:06:04.490Z Skipped 0 0 0 0
200862024-03-06T21:06:04.490Z Error 0 0 0 0
200872024-03-06T21:06:04.490ZLast Flush: 1014 0 1014
200882024-03-06T21:06:04.490ZDownstairs last five completed:
200892024-03-06T21:06:04.490ZUpstairs last five completed: 30 29 28 27 26
200902024-03-06T21:06:04.490Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:06:03.201791313Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1883}
200912024-03-06T21:06:04.490ZWrote out file "/tmp/test_repair_verify.data"
200922024-03-06T21:06:04.490Z
200932024-03-06T21:06:04.490ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
200942024-03-06T21:06:04.509ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
200952024-03-06T21:06:04.510Z 0 000-019 184 184 184 323 323 323 F F F
200962024-03-06T21:06:04.510Z 1 020-039 182 182 182 320 320 320 T F T <---
200972024-03-06T21:06:04.510Z 2 040-059 183 183 183 322 322 322 T F T <---
200982024-03-06T21:06:04.510Z 3 060-079 175 175 175 308 308 308 T F T <---
200992024-03-06T21:06:04.510Z 4 080-099 186 184 186 325 323 325 T F T <---
201002024-03-06T21:06:04.510Z 5 100-119 182 182 182 321 321 321 F F F
201012024-03-06T21:06:04.510Z 6 120-139 182 182 182 321 321 321 F F F
201022024-03-06T21:06:04.510Z 7 140-159 176 176 176 309 309 309 F F F
201032024-03-06T21:06:04.510Z 8 160-179 186 182 186 325 320 325 T F T <---
201042024-03-06T21:06:04.510Z 9 180-199 186 184 186 325 323 325 F F F <---
201052024-03-06T21:06:04.510Z 10 200-219 172 172 172 303 303 303 F F F
201062024-03-06T21:06:04.510Z 11 220-239 183 183 183 322 322 322 T F T <---
201072024-03-06T21:06:04.511Z 12 240-259 183 183 183 322 322 322 F F F
201082024-03-06T21:06:04.511Z 13 260-279 171 171 171 301 301 301 F F F
201092024-03-06T21:06:04.511Z 14 280-299 182 182 182 320 320 320 F F F
201102024-03-06T21:06:04.511Z 15 300-319 181 181 181 319 319 319 T F T <---
201112024-03-06T21:06:04.511Z 16 320-339 185 185 185 324 324 324 T F T <---
201122024-03-06T21:06:04.511Z 17 340-359 185 185 185 324 324 324 F F F
201132024-03-06T21:06:04.511Z 18 360-379 182 182 182 321 321 321 T F T <---
201142024-03-06T21:06:04.511Z 19 380-399 185 185 185 324 324 324 F F F
201152024-03-06T21:06:04.511Z 20 400-419 185 185 185 324 324 324 F F F
201162024-03-06T21:06:04.511Z 21 420-439 176 176 176 309 309 309 F F F
201172024-03-06T21:06:04.511Z 22 440-459 186 176 186 325 309 325 F F F <---
201182024-03-06T21:06:04.511Z 23 460-479 186 179 186 325 312 325 F F F <---
201192024-03-06T21:06:04.511Z 24 480-499 186 185 186 325 324 325 F F F <---
201202024-03-06T21:06:04.512Z 25 500-519 184 184 184 323 323 323 F F F
201212024-03-06T21:06:04.512Z 26 520-539 179 179 179 312 312 312 F F F
201222024-03-06T21:06:04.512Z 27 540-559 186 184 186 325 323 325 F F F <---
201232024-03-06T21:06:04.512Z 28 560-579 186 180 186 325 318 325 F F F <---
201242024-03-06T21:06:04.512Z 29 580-599 186 184 186 326 323 326 T F T <---
201252024-03-06T21:06:04.512ZMax gen: 186, Max flush: 326
201262024-03-06T21:06:04.512ZError: Difference in extent metadata found!
201272024-03-06T21:06:04.529ZA Difference in extent metadata is expected here
201282024-03-06T21:06:04.529ZOn loop 93
201292024-03-06T21:06:04.529Z
201302024-03-06T21:06:04.529ZVerifying data now
201312024-03-06T21:06:13.617ZNo change after verify
201322024-03-06T21:06:13.617ZLoop: 93 Downstairs dump after verify (and repair):
201332024-03-06T21:06:13.769ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
201342024-03-06T21:06:13.770Z 0 000-019 184 184 184 323 323 323 F F F
201352024-03-06T21:06:13.770Z 1 020-039 187 187 187 327 327 327 F F F
201362024-03-06T21:06:13.770Z 2 040-059 187 187 187 327 327 327 F F F
201372024-03-06T21:06:13.770Z 3 060-079 187 187 187 327 327 327 F F F
201382024-03-06T21:06:13.770Z 4 080-099 187 187 187 327 327 327 F F F
201392024-03-06T21:06:13.770Z 5 100-119 182 182 182 321 321 321 F F F
201402024-03-06T21:06:13.770Z 6 120-139 182 182 182 321 321 321 F F F
201412024-03-06T21:06:13.770Z 7 140-159 176 176 176 309 309 309 F F F
201422024-03-06T21:06:13.770Z 8 160-179 187 187 187 327 327 327 F F F
201432024-03-06T21:06:13.770Z 9 180-199 186 186 186 325 325 325 F F F
201442024-03-06T21:06:13.770Z 10 200-219 172 172 172 303 303 303 F F F
201452024-03-06T21:06:13.771Z 11 220-239 187 187 187 327 327 327 F F F
201462024-03-06T21:06:13.771Z 12 240-259 183 183 183 322 322 322 F F F
201472024-03-06T21:06:13.771Z 13 260-279 171 171 171 301 301 301 F F F
201482024-03-06T21:06:13.771Z 14 280-299 182 182 182 320 320 320 F F F
201492024-03-06T21:06:13.771Z 15 300-319 187 187 187 327 327 327 F F F
201502024-03-06T21:06:13.771Z 16 320-339 187 187 187 327 327 327 F F F
201512024-03-06T21:06:13.771Z 17 340-359 185 185 185 324 324 324 F F F
201522024-03-06T21:06:13.771Z 18 360-379 187 187 187 327 327 327 F F F
201532024-03-06T21:06:13.771Z 19 380-399 185 185 185 324 324 324 F F F
201542024-03-06T21:06:13.771Z 20 400-419 185 185 185 324 324 324 F F F
201552024-03-06T21:06:13.771Z 21 420-439 176 176 176 309 309 309 F F F
201562024-03-06T21:06:13.771Z 22 440-459 186 186 186 325 325 325 F F F
201572024-03-06T21:06:13.772Z 23 460-479 186 186 186 325 325 325 F F F
201582024-03-06T21:06:13.772Z 24 480-499 186 186 186 325 325 325 F F F
201592024-03-06T21:06:13.772Z 25 500-519 184 184 184 323 323 323 F F F
201602024-03-06T21:06:13.772Z 26 520-539 179 179 179 312 312 312 F F F
201612024-03-06T21:06:13.772Z 27 540-559 186 186 186 325 325 325 F F F
201622024-03-06T21:06:13.772Z 28 560-579 186 186 186 325 325 325 F F F
201632024-03-06T21:06:13.772Z 29 580-599 187 187 187 327 327 327 F F F
201642024-03-06T21:06:13.772ZMax gen: 187, Max flush: 327
201652024-03-06T21:06:13.772Z
201662024-03-06T21:06:13.772ZBegin loop 94
201672024-03-06T21:06:13.772ZDownstairs to restart: 0
201682024-03-06T21:06:13.772Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.12166546Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891}
201692024-03-06T21:06:13.772Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.121993077Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891Crucible runtime is spawned
201702024-03-06T21:06:13.773Z}
201712024-03-06T21:06:13.773Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.122043664Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891}
201722024-03-06T21:06:13.773Z{"msg":"Crucible 1a6bbbc1-0781-408c-91a8-6039d662e59a has session id: 3b12d90b-4d26-42e9-b910-8183edf787b0","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.122074621Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
201732024-03-06T21:06:13.773Z{"msg":"Upstairs opts: Upstairs UUID: 1a6bbbc1-0781-408c-91a8-6039d662e59a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.122137917Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
201742024-03-06T21:06:13.773Z{"msg":"Crucible stats registered with UUID: 1a6bbbc1-0781-408c-91a8-6039d662e59a","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.122180304Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
201752024-03-06T21:06:13.773Z{"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.122208442Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891}
201762024-03-06T21:06:13.773Z{"msg":"1a6bbbc1-0781-408c-91a8-6039d662e59a active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.1222396Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
201772024-03-06T21:06:13.773Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.122272478Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"client":"0","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
201782024-03-06T21:06:13.773Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.122313055Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"client":"0","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
201792024-03-06T21:06:13.773Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.122360922Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"client":"1","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
201802024-03-06T21:06:13.774Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.12239044Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"client":"1","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
201812024-03-06T21:06:13.774Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.122418548Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"client":"2","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
201822024-03-06T21:06:13.774Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.122442986Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"client":"2","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
201832024-03-06T21:06:13.774Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.122467385Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"":"io task","client":"2","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
201842024-03-06T21:06:13.774Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.122513091Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"":"io task","client":"0","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
201852024-03-06T21:06:13.774Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.122553348Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"":"io task","client":"1","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
201862024-03-06T21:06:13.774Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.122600025Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"":"io task","client":"2","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
201872024-03-06T21:06:13.774Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.122640322Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"":"io task","client":"1","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
201882024-03-06T21:06:13.774Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.122664871Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"":"io task","client":"0","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
201892024-03-06T21:06:13.774Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.12340621Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"client":"2","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
201902024-03-06T21:06:13.774Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.123453437Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"client":"2","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
201912024-03-06T21:06:13.774Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.123485264Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"client":"1","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
201922024-03-06T21:06:13.774Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.12354921Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"client":"1","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
201932024-03-06T21:06:13.775Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.124107021Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"client":"2","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
201942024-03-06T21:06:13.775Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.124262901Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"client":"1","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
201952024-03-06T21:06:13.775Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.12441642Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"client":"0","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
201962024-03-06T21:06:13.775Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.124466197Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"client":"0","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
201972024-03-06T21:06:13.775Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.12471288Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"client":"2","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
201982024-03-06T21:06:13.775Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.124756297Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"client":"2","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
201992024-03-06T21:06:13.775Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.124782475Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
202002024-03-06T21:06:13.775Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.124843171Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"client":"1","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
202012024-03-06T21:06:13.775Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.124878278Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"client":"1","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
202022024-03-06T21:06:13.775Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.124894177Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
202032024-03-06T21:06:13.775Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.125114652Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"client":"0","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
202042024-03-06T21:06:13.775Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.125679233Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"client":"0","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
202052024-03-06T21:06:13.776Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.12572568Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"client":"0","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
202062024-03-06T21:06:13.776Z{"msg":"[0]R flush_numbers[0..12]: [323, 327, 327, 327, 327, 321, 321, 309, 327, 325, 303, 327]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.125748268Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"":"downstairs","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
202072024-03-06T21:06:13.776Z{"msg":"[0]R generation[0..12]: [184, 187, 187, 187, 187, 182, 182, 176, 187, 186, 172, 187]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.125805735Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB",Wait for a query_work_queue command to finish before sending IO
202082024-03-06T21:06:13.776Z"pid":1891,"":"downstairs","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
202092024-03-06T21:06:13.776Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.125879419Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"":"downstairs","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
202102024-03-06T21:06:13.776Z{"msg":"[1]R flush_numbers[0..12]: [323, 327, 327, 327, 327, 321, 321, 309, 327, 325, 303, 327]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.125897758Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"":"downstairs","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
202112024-03-06T21:06:13.776Z{"msg":"[1]R generation[0..12]: [184, 187, 187, 187, 187, 182, 182, 176, 187, 186, 172, 187]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.125919317Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"":"downstairs","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
202122024-03-06T21:06:13.776Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.125934015Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"":"downstairs","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
202132024-03-06T21:06:13.776Z{"msg":"[2]R flush_numbers[0..12]: [323, 327, 327, 327, 327, 321, 321, 309, 327, 325, 303, 327]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.125960894Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"":"downstairs","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
202142024-03-06T21:06:13.776Z{"msg":"[2]R generation[0..12]: [184, 187, 187, 187, 187, 182, 182, 176, 187, 186, 172, 187]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.125977123Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"":"downstairs","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
202152024-03-06T21:06:13.776Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.125998241Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"":"downstairs","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
202162024-03-06T21:06:13.776Z{"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.12601227Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"":"downstairs","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
202172024-03-06T21:06:13.776Z{"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.126032509Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"":"downstairs","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
202182024-03-06T21:06:13.776Z{"msg":"Next flush: 328","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.126046268Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"":"downstairs","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
202192024-03-06T21:06:13.776Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.126073586Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"":"downstairs","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
202202024-03-06T21:06:13.776Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.126094475Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
202212024-03-06T21:06:13.776Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.126107334Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"client":"0","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
202222024-03-06T21:06:13.777Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.126127182Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"client":"1","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
202232024-03-06T21:06:13.777Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.126140901Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"client":"2","":"downstairs","session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
202242024-03-06T21:06:13.777Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.12616059Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
202252024-03-06T21:06:13.777Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.126173069Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
202262024-03-06T21:06:13.777Z{"msg":"1a6bbbc1-0781-408c-91a8-6039d662e59a is now active with session: 3b12d90b-4d26-42e9-b910-8183edf787b0","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.126192078Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
202272024-03-06T21:06:13.777Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.126204247Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891,"session_id":"3b12d90b-4d26-42e9-b910-8183edf787b0"}
202282024-03-06T21:06:13.777Z{"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:08.126216896Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891}
202292024-03-06T21:06:13.777ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
202302024-03-06T21:06:13.777ZLoading write count information from file "/tmp/test_repair_verify.data"
202312024-03-06T21:06:13.777ZRun Repair workload
202322024-03-06T21:06:13.777Z01/30 Write block 514 len 12288 data: 17 19 18
202332024-03-06T21:06:13.816Z02/30 Read block 303 len 28672
202342024-03-06T21:06:13.816Z03/30 Write block 385 len 36864 data: 19 19 17 18 16 18 13 11 10
202352024-03-06T21:06:13.816Z04/30 Read block 274 len 4096
202362024-03-06T21:06:13.816Z05/30 Read block 584 len 16384
202372024-03-06T21:06:13.816Z06/30 Write block 189 len 4096 data: 19
202382024-03-06T21:06:13.816Z07/30 Read block 62 len 40960
202392024-03-06T21:06:13.816Z08/30 Write block 259 len 4096 data: 7
202402024-03-06T21:06:13.817Z09/30 Read block 209 len 40960
202412024-03-06T21:06:13.817Z10/30 Read block 470 len 4096
202422024-03-06T21:06:13.817Z11/30 Read block 557 len 32768
202432024-03-06T21:06:13.817Z12/30 Write block 431 len 24576 data: 12 12 13 17 16 16
202442024-03-06T21:06:13.817Z13/30 Read block 590 len 20480
202452024-03-06T21:06:13.817Z14/30 Write block 590 len 32768 data: 16 18 18 14 14 11 9 6
202462024-03-06T21:06:13.817Z15/30 Write block 181 len 40960 data: 21 19 18 18 16 14 13 13 20 21
202472024-03-06T21:06:13.817Z16/30 Read block 161 len 16384
202482024-03-06T21:06:13.817Z17/30 Read block 200 len 16384
202492024-03-06T21:06:13.817Z18/30 Write block 386 len 8192 data: 20 18
202502024-03-06T21:06:13.817Z19/30 Read block 506 len 4096
202512024-03-06T21:06:13.817Z20/30 Write block 396 len 4096 data: 6
202522024-03-06T21:06:13.817Z21/30 Write block 4 len 24576 data: 15 15 15 16 15 15
202532024-03-06T21:06:13.817Z22/30 Read block 471 len 36864
202542024-03-06T21:06:13.817Z23/30 Read block 97 len 16384
202552024-03-06T21:06:13.817Z24/30 Write block 42 len 20480 data: 21 19 18 13 13
202562024-03-06T21:06:13.817Z25/30 Read block 336 len 8192
202572024-03-06T21:06:13.817Z26/30 Write block 50 len 12288 data: 9 11 12
202582024-03-06T21:06:13.817Z27/30 Write block 37 len 4096 data: 8
202592024-03-06T21:06:13.818Z28/30 Write block 250 len 20480 data: 8 9 9 10 9
202602024-03-06T21:06:13.818Z29/30 Write block 93 len 32768 data: 10 11 11 14 12 15 14 11
202612024-03-06T21:06:13.818Z30/30 Write block 335 len 4096 data: 9
202622024-03-06T21:06:13.818Z----------------------------------------------------------------
202632024-03-06T21:06:13.818Z Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:30
202642024-03-06T21:06:13.818ZStates: Active Active Active
202652024-03-06T21:06:13.818ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
202662024-03-06T21:06:13.818Z 1 Acked 1000 Write 3 Done Done Done false
202672024-03-06T21:06:13.818Z 2 Acked 1001 Read 7 Done Done Done false
202682024-03-06T21:06:13.818Z 3 Acked 1002 Write 9 Done Done Done false
202692024-03-06T21:06:13.818Z 4 Acked 1003 Read 1 Done Done Done false
202702024-03-06T21:06:13.818Z 5 Acked 1004 Read 4 Done Done Done false
202712024-03-06T21:06:13.818Z 6 Acked 1005 Write 1 Sent Done Done false
202722024-03-06T21:06:13.818Z 7 Acked 1006 Read 10 Sent Done Done false
202732024-03-06T21:06:13.818Z 8 Acked 1007 Write 1 Sent Done Done false
202742024-03-06T21:06:13.818Z 9 Acked 1008 Read 10 Sent Done Done false
202752024-03-06T21:06:13.818Z 10 Acked 1009 Read 1 Sent Done Done false
202762024-03-06T21:06:13.818Z 11 Acked 1010 Read 8 Sent Done Done false
202772024-03-06T21:06:13.818Z 12 Acked 1011 Write 6 Sent Done Done false
202782024-03-06T21:06:13.818Z 13 Acked 1012 Read 5 Sent Done Done false
202792024-03-06T21:06:13.819Z 14 Acked 1013 Write 8 Sent Done Done false
202802024-03-06T21:06:13.819Z 15 Acked 1014 Write 10 Sent Done Done false
202812024-03-06T21:06:13.819Z 16 Acked 1015 Read 4 Sent Done Done false
202822024-03-06T21:06:13.819Z 17 Acked 1016 Read 4 Sent Done Done false
202832024-03-06T21:06:13.819Z 18 Acked 1017 Write 2 Sent Done Done false
202842024-03-06T21:06:13.819Z 19 Acked 1018 Read 1 Sent Done Done false
202852024-03-06T21:06:13.819Z 20 Acked 1019 Write 1 Sent Done Done false
202862024-03-06T21:06:13.819Z 21 Acked 1020 Write 6 Sent Done Done false
202872024-03-06T21:06:13.819Z 22 Acked 1021 Read 9 Sent Done Done false
202882024-03-06T21:06:13.819Z 23 Acked 1022 Read 4 Sent Done Done false
202892024-03-06T21:06:13.819Z 24 Acked 1023 Write 5 Sent Done Done false
202902024-03-06T21:06:13.819Z 25 Acked 1024 Read 2 Sent Done Done false
202912024-03-06T21:06:13.819Z 26 Acked 1025 Write 3 Sent Done Done false
202922024-03-06T21:06:13.819Z 27 Acked 1026 Write 1 Sent Done Done false
202932024-03-06T21:06:13.819Z 28 Acked 1027 Write 5 Sent Done Done false
202942024-03-06T21:06:13.819Z 29 Acked 1028 Write 8 Sent Sent Sent false
202952024-03-06T21:06:13.819Z 30 Acked 1029 Write 1 Sent Sent Sent false
202962024-03-06T21:06:13.819Z STATES DS:0 DS:1 DS:2 TOTAL
202972024-03-06T21:06:13.819Z New 0 0 0 0
202982024-03-06T21:06:13.819Z Sent 25 2 2 29
202992024-03-06T21:06:13.819Z Done 5 28 28 61
203002024-03-06T21:06:13.819Z Skipped 0 0 0 0
203012024-03-06T21:06:13.819Z Error 0 0 0 0
203022024-03-06T21:06:13.819ZLast Flush: 0 0 0
203032024-03-06T21:06:13.820ZDownstairs last five completed:
203042024-03-06T21:06:13.820ZUpstairs last five completed: 30 29 28 27 26
203052024-03-06T21:06:13.820Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:06:08.156494009Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1891}
203062024-03-06T21:06:13.820ZWrote out file "/tmp/test_repair_verify.data"
203072024-03-06T21:06:13.820Z
203082024-03-06T21:06:13.820ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
203092024-03-06T21:06:13.820ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
203102024-03-06T21:06:13.820Z 0 000-019 184 184 184 323 323 323 F T T <---
203112024-03-06T21:06:13.820Z 1 020-039 187 187 187 327 327 327 F T T <---
203122024-03-06T21:06:13.820Z 2 040-059 187 187 187 327 327 327 F T T <---
203132024-03-06T21:06:13.820Z 3 060-079 187 187 187 327 327 327 F F F
203142024-03-06T21:06:13.820Z 4 080-099 187 187 187 327 327 327 F T T <---
203152024-03-06T21:06:13.820Z 5 100-119 182 182 182 321 321 321 F T T <---
203162024-03-06T21:06:13.820Z 6 120-139 182 182 182 321 321 321 F F F
203172024-03-06T21:06:13.820Z 7 140-159 176 176 176 309 309 309 F F F
203182024-03-06T21:06:13.820Z 8 160-179 187 187 187 327 327 327 F F F
203192024-03-06T21:06:13.820Z 9 180-199 186 186 186 325 325 325 F T T <---
203202024-03-06T21:06:13.820Z 10 200-219 172 172 172 303 303 303 F F F
203212024-03-06T21:06:13.820Z 11 220-239 187 187 187 327 327 327 F F F
203222024-03-06T21:06:13.820Z 12 240-259 183 183 183 322 322 322 F T T <---
203232024-03-06T21:06:13.820Z 13 260-279 171 171 171 301 301 301 F F F
203242024-03-06T21:06:13.820Z 14 280-299 182 182 182 320 320 320 F F F
203252024-03-06T21:06:13.820Z 15 300-319 187 187 187 327 327 327 F F F
203262024-03-06T21:06:13.820Z 16 320-339 187 187 187 327 327 327 F T T <---
203272024-03-06T21:06:13.821Z 17 340-359 185 185 185 324 324 324 F F F
203282024-03-06T21:06:13.821Z 18 360-379 187 187 187 327 327 327 F F F
203292024-03-06T21:06:13.821Z 19 380-399 185 185 185 324 324 324 T T T
203302024-03-06T21:06:13.821Z 20 400-419 185 185 185 324 324 324 F F F
203312024-03-06T21:06:13.821Z 21 420-439 176 176 176 309 309 309 F T T <---
203322024-03-06T21:06:13.821Z 22 440-459 186 186 186 325 325 325 F F F
203332024-03-06T21:06:13.834Z 23 460-479 186 186 186 325 325 325 F F F
203342024-03-06T21:06:13.834Z 24 480-499 186 186 186 325 325 325 F F F
203352024-03-06T21:06:13.834Z 25 500-519 184 184 184 323 323 323 T T T
203362024-03-06T21:06:13.834Z 26 520-539 179 179 179 312 312 312 F F F
203372024-03-06T21:06:13.834Z 27 540-559 186 186 186 325 325 325 F F F
203382024-03-06T21:06:13.835Z 28 560-579 186 186 186 325 325 325 F F F
203392024-03-06T21:06:13.835Z 29 580-599 187 187 187 327 327 327 F T T <---
203402024-03-06T21:06:13.835ZMax gen: 187, Max flush: 327
203412024-03-06T21:06:13.835ZError: Difference in extent metadata found!
203422024-03-06T21:06:13.835ZA Difference in extent metadata is expected here
203432024-03-06T21:06:13.835ZOn loop 94
203442024-03-06T21:06:13.835Z
203452024-03-06T21:06:13.836ZVerifying data now
203462024-03-06T21:06:16.941ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
203472024-03-06T21:06:16.942Z389,390c389
203482024-03-06T21:06:16.942Z< 19,
203492024-03-06T21:06:16.942Z< 17,
203502024-03-06T21:06:16.942Z---
203512024-03-06T21:06:16.942Z> 20,
203522024-03-06T21:06:16.943Z391a391
203532024-03-06T21:06:16.943Z> 18,
203542024-03-06T21:06:16.943Z399c399
203552024-03-06T21:06:16.943Z< 5,
203562024-03-06T21:06:16.943Z---
203572024-03-06T21:06:16.943Z> 6,
203582024-03-06T21:06:16.944Zdiff found after verify
203592024-03-06T21:06:16.944ZLoop: 94 Downstairs dump after verify (and repair):
203602024-03-06T21:06:16.944ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
203612024-03-06T21:06:16.944Z 0 000-019 188 188 188 328 328 328 F F F
203622024-03-06T21:06:16.944Z 1 020-039 188 188 188 328 328 328 F F F
203632024-03-06T21:06:16.944Z 2 040-059 188 188 188 328 328 328 F F F
203642024-03-06T21:06:16.945Z 3 060-079 187 187 187 327 327 327 F F F
203652024-03-06T21:06:16.945Z 4 080-099 188 188 188 328 328 328 F F F
203662024-03-06T21:06:16.945Z 5 100-119 188 188 188 328 328 328 F F F
203672024-03-06T21:06:16.945Z 6 120-139 182 182 182 321 321 321 F F F
203682024-03-06T21:06:16.946Z 7 140-159 176 176 176 309 309 309 F F F
203692024-03-06T21:06:16.946Z 8 160-179 187 187 187 327 327 327 F F F
203702024-03-06T21:06:16.946Z 9 180-199 188 188 188 328 328 328 F F F
203712024-03-06T21:06:16.946Z 10 200-219 172 172 172 303 303 303 F F F
203722024-03-06T21:06:16.946Z 11 220-239 187 187 187 327 327 327 F F F
203732024-03-06T21:06:16.947Z 12 240-259 188 188 188 328 328 328 F F F
203742024-03-06T21:06:16.947Z 13 260-279 171 171 171 301 301 301 F F F
203752024-03-06T21:06:16.948Z 14 280-299 182 182 182 320 320 320 F F F
203762024-03-06T21:06:16.948Z 15 300-319 187 187 187 327 327 327 F F F
203772024-03-06T21:06:16.948Z 16 320-339 188 188 188 328 328 328 F F F
203782024-03-06T21:06:16.948Z 17 340-359 185 185 185 324 324 324 F F F
203792024-03-06T21:06:16.948Z 18 360-379 187 187 187 327 327 327 F F F
203802024-03-06T21:06:16.948Z 19 380-399 188 188 188 328 328 328 F F F
203812024-03-06T21:06:16.949Z 20 400-419 185 185 185 324 324 324 F F F
203822024-03-06T21:06:16.949Z 21 420-439 188 188 188 328 328 328 F F F
203832024-03-06T21:06:16.949Z 22 440-459 186 186 186 325 325 325 F F F
203842024-03-06T21:06:16.949Z 23 460-479 186 186 186 325 325 325 F F F
203852024-03-06T21:06:16.949Z 24 480-499 186 186 186 325 325 325 F F F
203862024-03-06T21:06:16.949Z 25 500-519 188 188 188 328 328 328 F F F
203872024-03-06T21:06:16.949Z 26 520-539 179 179 179 312 312 312 F F F
203882024-03-06T21:06:16.950Z 27 540-559 186 186 186 325 325 325 F F F
203892024-03-06T21:06:16.950Z 28 560-579 186 186 186 325 325 325 F F F
203902024-03-06T21:06:16.950Z 29 580-599 188 188 188 328 328 328 F F F
203912024-03-06T21:06:16.950ZMax gen: 188, Max flush: 328
203922024-03-06T21:06:16.950Z
203932024-03-06T21:06:16.950ZBegin loop 95
203942024-03-06T21:06:16.950ZDownstairs to restart: 0
203952024-03-06T21:06:16.951Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.943879422Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902}
203962024-03-06T21:06:16.951Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.944163902Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902}
203972024-03-06T21:06:16.951Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.944183411Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902}
203982024-03-06T21:06:16.951Z{"msg":"Crucible 4cd33cf0-de2e-4861-a725-c36c5e7fd422 has session id: c794cb9c-49d0-40b6-bc76-a890b7cdc820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.94419875Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
203992024-03-06T21:06:16.951Z{"msg":"Upstairs opts: Upstairs UUID: 4cd33cf0-de2e-4861-a725-c36c5e7fd422, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.944233628Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204002024-03-06T21:06:16.951Z{"msg":"Crucible stats registered with UUID: 4cd33cf0-de2e-4861-a725-c36c5e7fd422","v":0,"name":"crucible","level"Crucible runtime is spawned
204012024-03-06T21:06:17.099Z:30,"time":"2024-03-06T21:06:13.944266675Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204022024-03-06T21:06:17.099Z{"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.944319511Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902}
204032024-03-06T21:06:17.099Z{"msg":"4cd33cf0-de2e-4861-a725-c36c5e7fd422 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.944471421Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204042024-03-06T21:06:17.099Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.944499009Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"client":"0","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204052024-03-06T21:06:17.099Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.944523088Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"client":"0","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204062024-03-06T21:06:17.099Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.944556705Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"client":"1","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204072024-03-06T21:06:17.099Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.944594093Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"client":"1","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204082024-03-06T21:06:17.099Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.94463061Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"client":"2","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204092024-03-06T21:06:17.099Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.944674347Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"client":"2","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204102024-03-06T21:06:17.099Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.944743412Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204112024-03-06T21:06:17.100Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.944840266Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204122024-03-06T21:06:17.100Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.944879863Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204132024-03-06T21:06:17.100Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.944913171Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204142024-03-06T21:06:17.100Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.944949848Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204152024-03-06T21:06:17.100Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.944988416Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204162024-03-06T21:06:17.100Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.945764852Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"client":"2","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204172024-03-06T21:06:17.100Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.945843206Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"client":"2","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204182024-03-06T21:06:17.101Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.945927831Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"client":"1","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204192024-03-06T21:06:17.101Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.945977827Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"client":"1","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204202024-03-06T21:06:17.101Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.94622733Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"client":"0","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204212024-03-06T21:06:17.101Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.946299795Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"client":"0","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204222024-03-06T21:06:17.101Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.946448265Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"client":"1","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204232024-03-06T21:06:17.101Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.946569387Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"client":"2","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204242024-03-06T21:06:17.101Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.946910913Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"client":"0","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204252024-03-06T21:06:17.101Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.94710749Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"client":"1","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204262024-03-06T21:06:17.101Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.947143697Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"client":"1","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204272024-03-06T21:06:17.101Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.947160436Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204282024-03-06T21:06:17.101Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.947174015Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"client":"2","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204292024-03-06T21:06:17.102Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.947195103Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"client":"2","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204302024-03-06T21:06:17.102Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.947208663Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204312024-03-06T21:06:17.102Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.94739677Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"client":"0","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204322024-03-06T21:06:17.102Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.947443436Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"client":"0","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204332024-03-06T21:06:17.102Z{"msg":"[0]R flush_numbers[0..12]: [328, 328, 328, 327, 328, 328, 321, 309, 327, 328, 303, 327]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.947478164Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"":"downstairs","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204342024-03-06T21:06:17.102Z{"msg":"[0]R generation[0..12]: [188, 188, 188, 187, 188, 188, 182, 176, 187, 188, 172, 187]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.947513411Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"":"downstairs","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204352024-03-06T21:06:17.102Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.947549479Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"":"downstairs","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204362024-03-06T21:06:17.102Z{"msg":"[1]R flush_numbers[0..12]: [328, 328, 328, 327, 328, 328, 321, 309, 327, 328, 303, 327]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.947584086Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"":"downstairs","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204372024-03-06T21:06:17.102ZWait for a query_work_queue command to finish before sending IO
204382024-03-06T21:06:17.103Z{"msg":"[1]R generation[0..12]: [188, 188, 188, 187, 188, 188, 182, 176, 187, 188, 172, 187]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.947623224Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"":"downstairs","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204392024-03-06T21:06:17.103Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.947661111Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"":"downstairs","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204402024-03-06T21:06:17.103Z{"msg":"[2]R flush_numbers[0..12]: [328, 328, 328, 327, 328, 328, 321, 309, 327, 328, 303, 327]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.947699629Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"":"downstairs","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204412024-03-06T21:06:17.103Z{"msg":"[2]R generation[0..12]: [188, 188, 188, 187, 188, 188, 182, 176, 187, 188, 172, 187]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.947739596Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"":"downstairs","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204422024-03-06T21:06:17.103Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.947777923Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"":"downstairs","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204432024-03-06T21:06:17.103Z{"msg":"Max found gen is 189","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.947813081Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"":"downstairs","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204442024-03-06T21:06:17.103Z{"msg":"Generation requested: 190 >= found:189","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.947839259Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"":"downstairs","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204452024-03-06T21:06:17.103Z{"msg":"Next flush: 329","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.947861798Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"":"downstairs","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204462024-03-06T21:06:17.103Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.947889516Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"":"downstairs","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204472024-03-06T21:06:17.103ZError: Difference in extent metadata found!
204482024-03-06T21:06:17.103Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.947916444Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204492024-03-06T21:06:17.103Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.947943932Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"client":"0","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204502024-03-06T21:06:17.103Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.947957621Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"client":"1","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204512024-03-06T21:06:17.103Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
204522024-03-06T21:06:17.104Z,"time":"2024-03-06T21:06:13.947975859Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"client":"2","":"downstairs","session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204532024-03-06T21:06:17.104Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.948002868Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204542024-03-06T21:06:17.104Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.948024916Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204552024-03-06T21:06:17.104Z{"msg":"4cd33cf0-de2e-4861-a725-c36c5e7fd422 is now active with session: c794cb9c-49d0-40b6-bc76-a890b7cdc820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.948045365Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204562024-03-06T21:06:17.104Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.948058044Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902,"session_id":"c794cb9c-49d0-40b6-bc76-a890b7cdc820"}
204572024-03-06T21:06:17.104Z{"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:13.948070233Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902}
204582024-03-06T21:06:17.104ZLoading write count information from file "/tmp/test_repair_verify.data"
204592024-03-06T21:06:17.104ZRun Repair workload
204602024-03-06T21:06:17.104Z01/30 Write block 133 len 8192 data: 12 11
204612024-03-06T21:06:17.104Z02/30 Read block 251 len 24576
204622024-03-06T21:06:17.104Z03/30 Read block 444 len 4096
204632024-03-06T21:06:17.105Z04/30 Write block 532 len 32768 data: 14 14 15 17 13 15 10 8
204642024-03-06T21:06:17.105Z05/30 Read block 433 len 16384
204652024-03-06T21:06:17.105Z06/30 Write block 311 len 36864 data: 17 14 11 11 11 13 11 15 15
204662024-03-06T21:06:17.105Z07/30 Write block 338 len 32768 data: 17 15 19 18 17 19 18 14
204672024-03-06T21:06:17.105Z08/30 Flush
204682024-03-06T21:06:17.105Z09/30 Write block 555 len 24576 data: 10 9 11 10 10 14
204692024-03-06T21:06:17.105Z10/30 Write block 512 len 16384 data: 15 14 18 20
204702024-03-06T21:06:17.105Z11/30 Read block 221 len 40960
204712024-03-06T21:06:17.105Z12/30 Write block 181 len 32768 data: 22 20 19 19 17 15 14 14
204722024-03-06T21:06:17.105Z13/30 Read block 385 len 28672
204732024-03-06T21:06:17.105Z14/30 Flush
204742024-03-06T21:06:17.105Z15/30 Write block 159 len 12288 data: 13 12 14
204752024-03-06T21:06:17.106Z16/30 Write block 306 len 4096 data: 21
204762024-03-06T21:06:17.106Z17/30 Read block 454 len 4096
204772024-03-06T21:06:17.106Z18/30 Read block 379 len 16384
204782024-03-06T21:06:17.106Z19/30 Read block 158 len 12288
204792024-03-06T21:06:17.106Z20/30 Write block 280 len 4096 data: 13
204802024-03-06T21:06:17.106Z21/30 Read block 23 len 12288
204812024-03-06T21:06:17.107Z22/30 Write block 184 len 12288 data: 20 18 16
204822024-03-06T21:06:17.107Z23/30 Write block 409 len 4096 data: 15
204832024-03-06T21:06:17.107Z24/30 Write block 242 len 40960 data: 17 16 15 14 13 12 10 9 9 10
204842024-03-06T21:06:17.107Z25/30 Read block 440 len 20480
204852024-03-06T21:06:17.107Z26/30 Write block 185 len 40960 data: 19 17 15 15 21 22 15 17 14 13
204862024-03-06T21:06:17.107Z27/30 Write block 511 len 24576 data: 15 16 15 19 21 19
204872024-03-06T21:06:17.107Z28/30 Write block 51 len 36864 data: 12 13 12 10 13 10 12 15 17
204882024-03-06T21:06:17.107Z29/30 Write block 334 len 16384 data: 10 10 12 13
204892024-03-06T21:06:17.108Z30/30 Read block 573 len 12288
204902024-03-06T21:06:17.108Z----------------------------------------------------------------
204912024-03-06T21:06:17.108Z Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:30
204922024-03-06T21:06:17.108ZStates: Active Active Active
204932024-03-06T21:06:17.108ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
204942024-03-06T21:06:17.108Z 1 Acked 1000 Write 2 Sent Done Done false
204952024-03-06T21:06:17.108Z 2 Acked 1001 Read 6 Sent Done Done false
204962024-03-06T21:06:17.109Z 3 Acked 1002 Read 1 Sent Done Done false
204972024-03-06T21:06:17.109Z 4 Acked 1003 Write 8 Sent Done Done false
204982024-03-06T21:06:17.109Z 5 Acked 1004 Read 4 Sent Done Done false
204992024-03-06T21:06:17.109Z 6 Acked 1005 Write 9 Sent Done Done false
205002024-03-06T21:06:17.109Z 7 Acked 1006 Write 8 Sent Done Done false
205012024-03-06T21:06:17.122Z 8 Acked 1007 Flush 0 Sent Done Done false
205022024-03-06T21:06:17.122Z 9 Acked 1008 Write 6 Sent Done Done false
205032024-03-06T21:06:17.122Z 10 Acked 1009 Write 4 Sent Done Done false
205042024-03-06T21:06:17.122Z 11 Acked 1010 Read 10 Sent Done Done false
205052024-03-06T21:06:17.122Z 12 Acked 1011 Write 8 Sent Done Done false
205062024-03-06T21:06:17.122Z 13 Acked 1012 Read 7 Sent Done Done false
205072024-03-06T21:06:17.123Z 14 Acked 1013 Flush 0 Sent Done Done false
205082024-03-06T21:06:17.123Z 15 Acked 1014 Write 3 Sent Done Done false
205092024-03-06T21:06:17.123Z 16 Acked 1015 Write 1 Sent Done Done false
205102024-03-06T21:06:17.123Z 17 Acked 1016 Read 1 Sent Done Done false
205112024-03-06T21:06:17.123Z 18 Acked 1017 Read 4 Sent Done Done false
205122024-03-06T21:06:17.123Z 19 Acked 1018 Read 3 Sent Done Done false
205132024-03-06T21:06:17.123Z 20 Acked 1019 Write 1 Sent Done Done false
205142024-03-06T21:06:17.123Z 21 Acked 1020 Read 3 Sent Done Done false
205152024-03-06T21:06:17.124Z 22 Acked 1021 Write 3 Sent Done Done false
205162024-03-06T21:06:17.124Z 23 Acked 1022 Write 1 Sent Done Done false
205172024-03-06T21:06:17.124Z 24 Acked 1023 Write 10 Sent Done Done false
205182024-03-06T21:06:17.124Z 25 Acked 1024 Read 5 Sent Done Done false
205192024-03-06T21:06:17.124Z 26 Acked 1025 Write 10 Sent Done Done false
205202024-03-06T21:06:17.124Z 27 Acked 1026 Write 6 Sent Done Done false
205212024-03-06T21:06:17.124Z 28 Acked 1027 Write 9 Sent Done Done false
205222024-03-06T21:06:17.124Z 29 Acked 1028 Write 4 Sent Done Done false
205232024-03-06T21:06:17.124Z 30 Acked 1029 Read 3 Sent Done Done false
205242024-03-06T21:06:17.124Z STATES DS:0 DS:1 DS:2 TOTAL
205252024-03-06T21:06:17.124Z New 0 0 0 0
205262024-03-06T21:06:17.124Z Sent 30 0 0 30
205272024-03-06T21:06:17.124Z Done 0 30 30 60
205282024-03-06T21:06:17.124Z Skipped 0 0 0 0
205292024-03-06T21:06:17.124Z Error 0 0 0 0
205302024-03-06T21:06:17.124ZLast Flush: 0 1013 1013
205312024-03-06T21:06:17.125ZDownstairs last five completed:
205322024-03-06T21:06:17.125ZUpstairs last five completed: 30 29 28 27 26
205332024-03-06T21:06:17.125Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:06:13.983030394Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1902}
205342024-03-06T21:06:17.125ZWrote out file "/tmp/test_repair_verify.data"
205352024-03-06T21:06:17.125Z
205362024-03-06T21:06:17.125ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
205372024-03-06T21:06:17.125ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
205382024-03-06T21:06:17.125Z 0 000-019 188 188 188 328 328 328 F F F
205392024-03-06T21:06:17.125Z 1 020-039 188 188 188 328 328 328 F F F
205402024-03-06T21:06:17.125Z 2 040-059 188 188 188 328 328 328 F T T <---
205412024-03-06T21:06:17.125Z 3 060-079 187 187 187 327 327 327 F F F
205422024-03-06T21:06:17.125Z 4 080-099 188 188 188 328 328 328 F F F
205432024-03-06T21:06:17.125Z 5 100-119 188 188 188 328 328 328 F F F
205442024-03-06T21:06:17.125Z 6 120-139 182 190 190 321 329 329 F F F <---
205452024-03-06T21:06:17.125Z 7 140-159 176 176 176 309 309 309 F T T <---
205462024-03-06T21:06:17.125Z 8 160-179 187 187 187 327 327 327 F T T <---
205472024-03-06T21:06:17.126Z 9 180-199 188 190 190 328 330 330 F T T <---
205482024-03-06T21:06:17.126Z 10 200-219 172 172 172 303 303 303 F F F
205492024-03-06T21:06:17.126Z 11 220-239 187 187 187 327 327 327 F F F
205502024-03-06T21:06:17.126Z 12 240-259 188 188 188 328 328 328 F T T <---
205512024-03-06T21:06:17.126Z 13 260-279 171 171 171 301 301 301 F F F
205522024-03-06T21:06:17.126Z 14 280-299 182 182 182 320 320 320 F T T <---
205532024-03-06T21:06:17.126Z 15 300-319 187 190 190 327 329 329 F T T <---
205542024-03-06T21:06:17.126Z 16 320-339 188 190 190 328 329 329 F T T <---
205552024-03-06T21:06:17.127Z 17 340-359 185 190 190 324 329 329 F F F <---
205562024-03-06T21:06:17.127Z 18 360-379 187 187 187 327 327 327 F F F
205572024-03-06T21:06:17.127Z 19 380-399 188 188 188 328 328 328 F F F
205582024-03-06T21:06:17.127Z 20 400-419 185 185 185 324 324 324 F T T <---
205592024-03-06T21:06:17.127Z 21 420-439 188 188 188 328 328 328 F F F
205602024-03-06T21:06:17.127Z 22 440-459 186 186 186 325 325 325 F F F
205612024-03-06T21:06:17.127Z 23 460-479 186 186 186 325 325 325 F F F
205622024-03-06T21:06:17.128Z 24 480-499 186 186 186 325 325 325 F F F
205632024-03-06T21:06:17.128Z 25 500-519 188 190 190 328 330 330 F T T <---
205642024-03-06T21:06:17.128Z 26 520-539 179 190 190 312 329 329 F F F <---
205652024-03-06T21:06:17.128Z 27 540-559 186 190 190 325 330 330 F F F <---
205662024-03-06T21:06:17.128Z 28 560-579 186 190 190 325 330 330 F F F <---
205672024-03-06T21:06:17.128Z 29 580-599 188 188 188 328 328 328 F F F
205682024-03-06T21:06:17.128ZMax gen: 190, Max flush: 330
205692024-03-06T21:06:17.128ZA Difference in extent metadata is expected here
205702024-03-06T21:06:17.128ZOn loop 95
205712024-03-06T21:06:17.129Z
205722024-03-06T21:06:17.129ZVerifying data now
205732024-03-06T21:06:20.096ZNo change after verify
205742024-03-06T21:06:20.097ZLoop: 95 Downstairs dump after verify (and repair):
205752024-03-06T21:06:20.224ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
205762024-03-06T21:06:20.225Z 0 000-019 188 188 188 328 328 328 F F F
205772024-03-06T21:06:20.225Z 1 020-039 188 188 188 328 328 328 F F F
205782024-03-06T21:06:20.225Z 2 040-059 191 191 191 331 331 331 F F F
205792024-03-06T21:06:20.225Z 3 060-079 187 187 187 327 327 327 F F F
205802024-03-06T21:06:20.225Z 4 080-099 188 188 188 328 328 328 F F F
205812024-03-06T21:06:20.225Z 5 100-119 188 188 188 328 328 328 F F F
205822024-03-06T21:06:20.226Z 6 120-139 190 190 190 329 329 329 F F F
205832024-03-06T21:06:20.226Z 7 140-159 191 191 191 331 331 331 F F F
205842024-03-06T21:06:20.226Z 8 160-179 191 191 191 331 331 331 F F F
205852024-03-06T21:06:20.226Z 9 180-199 191 191 191 331 331 331 F F F
205862024-03-06T21:06:20.226Z 10 200-219 172 172 172 303 303 303 F F F
205872024-03-06T21:06:20.227Z 11 220-239 187 187 187 327 327 327 F F F
205882024-03-06T21:06:20.227Z 12 240-259 191 191 191 331 331 331 F F F
205892024-03-06T21:06:20.227Z 13 260-279 171 171 171 301 301 301 F F F
205902024-03-06T21:06:20.227Z 14 280-299 191 191 191 331 331 331 F F F
205912024-03-06T21:06:20.227Z 15 300-319 191 191 191 331 331 331 F F F
205922024-03-06T21:06:20.227Z 16 320-339 191 191 191 331 331 331 F F F
205932024-03-06T21:06:20.227Z 17 340-359 190 190 190 329 329 329 F F F
205942024-03-06T21:06:20.228Z 18 360-379 187 187 187 327 327 327 F F F
205952024-03-06T21:06:20.228Z 19 380-399 188 188 188 328 328 328 F F F
205962024-03-06T21:06:20.228Z 20 400-419 191 191 191 331 331 331 F F F
205972024-03-06T21:06:20.228Z 21 420-439 188 188 188 328 328 328 F F F
205982024-03-06T21:06:20.228Z 22 440-459 186 186 186 325 325 325 F F F
205992024-03-06T21:06:20.229Z 23 460-479 186 186 186 325 325 325 F F F
206002024-03-06T21:06:20.229Z 24 480-499 186 186 186 325 325 325 F F F
206012024-03-06T21:06:20.229Z 25 500-519 191 191 191 331 331 331 F F F
206022024-03-06T21:06:20.229Z 26 520-539 190 190 190 329 329 329 F F F
206032024-03-06T21:06:20.229Z 27 540-559 190 190 190 330 330 330 F F F
206042024-03-06T21:06:20.230Z 28 560-579 190 190 190 330 330 330 F F F
206052024-03-06T21:06:20.230Z 29 580-599 188 188 188 328 328 328 F F F
206062024-03-06T21:06:20.230ZMax gen: 191, Max flush: 331
206072024-03-06T21:06:20.230Z
206082024-03-06T21:06:20.230ZBegin loop 96
206092024-03-06T21:06:20.231ZDownstairs to restart: 2
206102024-03-06T21:06:20.231Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.869433438Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910}
206112024-03-06T21:06:20.231Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.869733097Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910}
206122024-03-06T21:06:20.231ZCrucible runtime is spawned
206132024-03-06T21:06:20.231Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.869786363Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910}
206142024-03-06T21:06:20.231Z{"msg":"Crucible 05886c1e-68b5-4b31-be92-04b6e7717bfe has session id: cdd1d61a-5b1c-4553-be94-4261f7c4b11a","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.869825411Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206152024-03-06T21:06:20.232Z{"msg":"Upstairs opts: Upstairs UUID: 05886c1e-68b5-4b31-be92-04b6e7717bfe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.869867688Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206162024-03-06T21:06:20.232Z{"msg":"Crucible stats registered with UUID: 05886c1e-68b5-4b31-be92-04b6e7717bfe","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.869892416Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206172024-03-06T21:06:20.232Z{"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.869908305Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910}
206182024-03-06T21:06:20.232Z{"msg":"05886c1e-68b5-4b31-be92-04b6e7717bfe active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.869989599Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206192024-03-06T21:06:20.232Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.870020497Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"client":"0","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206202024-03-06T21:06:20.232Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.870057615Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"client":"0","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206212024-03-06T21:06:20.232Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.870096832Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"client":"1","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206222024-03-06T21:06:20.232Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.870134259Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"client":"1","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206232024-03-06T21:06:20.233Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.870170657Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"client":"2","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206242024-03-06T21:06:20.233Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.870208354Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"client":"2","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206252024-03-06T21:06:20.233Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.870234862Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206262024-03-06T21:06:20.233Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.87026607Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206272024-03-06T21:06:20.233Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.870308857Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206282024-03-06T21:06:20.233Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.870479146Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206292024-03-06T21:06:20.233Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.870549321Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206302024-03-06T21:06:20.233Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.870594638Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206312024-03-06T21:06:20.233Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.87127924Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"client":"0","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206322024-03-06T21:06:20.233Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.871326497Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"client":"0","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206332024-03-06T21:06:20.233Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.871393412Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"client":"1","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206342024-03-06T21:06:20.233Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.871446799Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"client":"1","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206352024-03-06T21:06:20.233Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.872098984Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"client":"1","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206362024-03-06T21:06:20.234Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.8721555Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"client":"0","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206372024-03-06T21:06:20.234Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.872205716Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"client":"2","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206382024-03-06T21:06:20.234Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.872244974Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"client":"2","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206392024-03-06T21:06:20.234Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.872718631Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"client":"1","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206402024-03-06T21:06:20.234Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.872765958Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"client":"1","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206412024-03-06T21:06:20.234Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.872802425Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206422024-03-06T21:06:20.234Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.872837623Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"client":"0","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206432024-03-06T21:06:20.234Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.87287648Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"client":"0","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206442024-03-06T21:06:20.234Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.872914828Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206452024-03-06T21:06:20.234Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.873038449Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"client":"2","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206462024-03-06T21:06:20.234Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.873440271Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"client":"2","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206472024-03-06T21:06:20.234Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.873476979Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"client":"2","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206482024-03-06T21:06:20.234Z{"msg":"[0]R flush_numbers[0..12]: [328, 328, 331, 327, 328, 328, 329, 331, 331, 331, 303, 327]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.873500277Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"":"downstairs","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206492024-03-06T21:06:20.235Z{"msg":"[0]R generation[0..12]: [188, 188, 191, 187, 188, 188, 190, 191, 191, 191, 172, 187]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.873515836Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"":"downstairs","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206502024-03-06T21:06:20.235Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.873537305Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"":"downstairs","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206512024-03-06T21:06:20.235Z{"msg":"[1]R flush_numbers[0..12]: [328, 328, 331, 327, 328, 328, 329, 331, 331, 331, 303, 327]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.873558133Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"":"downstairs","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206522024-03-06T21:06:20.235Z{"msg":"[1]R generation[0..12]: [188, 188, 191, 187, 188, 188, 190, 191, 191, 191, 172, 187]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.873585671Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"":"downstairs","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206532024-03-06T21:06:20.235ZWait for a query_work_queue command to finish before sending IO
206542024-03-06T21:06:20.235Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.873623449Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"":"downstairs","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206552024-03-06T21:06:20.235Z{"msg":"[2]R flush_numbers[0..12]: [328, 328, 331, 327, 328, 328, 329, 331, 331, 331, 303, 327]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.873661796Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"":"downstairs","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206562024-03-06T21:06:20.235Z{"msg":"[2]R generation[0..12]: [188, 188, 191, 187, 188, 188, 190, 191, 191, 191, 172, 187]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.873707783Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"":"downstairs","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206572024-03-06T21:06:20.235Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.873736671Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"":"downstairs","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206582024-03-06T21:06:20.259Z{"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.873765409Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"":"downstairs","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206592024-03-06T21:06:20.259Z{"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.873790627Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"":"downstairs","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206602024-03-06T21:06:20.259Z{"msg":"Next flush: 332","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.873826295Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"":"downstairs","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206612024-03-06T21:06:20.259Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.873854823Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"":"downstairs","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206622024-03-06T21:06:20.259Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.873876021Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206632024-03-06T21:06:20.259Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.87390325Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"client":"0","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206642024-03-06T21:06:20.259Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.873939277Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"client":"1","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206652024-03-06T21:06:20.259Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.873977174Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"client":"2","":"downstairs","session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206662024-03-06T21:06:20.259Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.874009792Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206672024-03-06T21:06:20.259Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.874030611Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206682024-03-06T21:06:20.260Z{"msg":"05886c1e-68b5-4b31-be92-04b6e7717bfe is now active with session: cdd1d61a-5b1c-4553-be94-4261f7c4b11a","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
206692024-03-06T21:06:20.260Z,"time":"2024-03-06T21:06:18.874060859Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206702024-03-06T21:06:20.260Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.874094276Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910,"session_id":"cdd1d61a-5b1c-4553-be94-4261f7c4b11a"}
206712024-03-06T21:06:20.260Z{"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:18.874123684Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910}
206722024-03-06T21:06:20.260ZLoading write count information from file "/tmp/test_repair_verify.data"
206732024-03-06T21:06:20.260ZRun Repair workload
206742024-03-06T21:06:20.260Z01/30 Write block 255 len 8192 data: 10 10
206752024-03-06T21:06:20.260Z02/30 Write block 27 len 32768 data: 14 13 15 13 12 13 13 13
206762024-03-06T21:06:20.260Z03/30 Flush
206772024-03-06T21:06:20.260Z04/30 Write block 583 len 36864 data: 15 17 17 18 18 17 16 17 19
206782024-03-06T21:06:20.260Z05/30 Write block 423 len 4096 data: 15
206792024-03-06T21:06:20.261Z06/30 Read block 46 len 12288
206802024-03-06T21:06:20.261Z07/30 Read block 271 len 24576
206812024-03-06T21:06:20.261Z08/30 Flush
206822024-03-06T21:06:20.261Z09/30 Write block 189 len 20480 data: 22 23 16 18 15
206832024-03-06T21:06:20.261Z10/30 Flush
206842024-03-06T21:06:20.261Z11/30 Write block 92 len 12288 data: 12 11 12
206852024-03-06T21:06:20.261Z12/30 Read block 586 len 8192
206862024-03-06T21:06:20.261Z13/30 Read block 66 len 28672
206872024-03-06T21:06:20.261Z14/30 Write block 423 len 40960 data: 16 13 13 14 15 14 14 14 13 13
206882024-03-06T21:06:20.261Z15/30 Read block 408 len 20480
206892024-03-06T21:06:20.262Z16/30 Write block 486 len 16384 data: 14 13 15 17
206902024-03-06T21:06:20.262Z17/30 Flush
206912024-03-06T21:06:20.262Z18/30 Write block 385 len 8192 data: 20 20
206922024-03-06T21:06:20.262Z19/30 Read block 98 len 36864
206932024-03-06T21:06:20.262Z20/30 Read block 164 len 24576
206942024-03-06T21:06:20.262Z21/30 Read block 285 len 8192
206952024-03-06T21:06:20.262Z22/30 Read block 478 len 32768
206962024-03-06T21:06:20.262Z23/30 Read block 347 len 8192
206972024-03-06T21:06:20.262Z24/30 Read block 386 len 8192
206982024-03-06T21:06:20.262Z25/30 Write block 262 len 4096 data: 7
206992024-03-06T21:06:20.262Z26/30 Read block 408 len 32768
207002024-03-06T21:06:20.263Z27/30 Read block 498 len 40960
207012024-03-06T21:06:20.263Z28/30 Read block 74 len 4096
207022024-03-06T21:06:20.263Z29/30 Read block 168 len 12288
207032024-03-06T21:06:20.263Z30/30 Read block 22 len 28672
207042024-03-06T21:06:20.263Z----------------------------------------------------------------
207052024-03-06T21:06:20.263Z Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:30
207062024-03-06T21:06:20.263ZStates: Active Active Active
207072024-03-06T21:06:20.263ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
207082024-03-06T21:06:20.263Z 1 Acked 1000 Write 2 Done Done Sent false
207092024-03-06T21:06:20.263Z 2 Acked 1001 Write 8 Done Done Sent false
207102024-03-06T21:06:20.263Z 3 Acked 1002 Flush 0 Done Done Sent false
207112024-03-06T21:06:20.263Z 4 Acked 1003 Write 9 Done Done Sent false
207122024-03-06T21:06:20.264Z 5 Acked 1004 Write 1 Done Done Sent false
207132024-03-06T21:06:20.264Z 6 Acked 1005 Read 3 Done Done Sent false
207142024-03-06T21:06:20.264Z 7 Acked 1006 Read 6 Done Done Sent false
207152024-03-06T21:06:20.264Z 8 Acked 1007 Flush 0 Done Done Sent false
207162024-03-06T21:06:20.264Z 9 Acked 1008 Write 5 Done Done Sent false
207172024-03-06T21:06:20.264Z 10 Acked 1009 Flush 0 Done Done Sent false
207182024-03-06T21:06:20.264Z 11 Acked 1010 Write 3 Done Done Sent false
207192024-03-06T21:06:20.264Z 12 Acked 1011 Read 2 Done Done Sent false
207202024-03-06T21:06:20.264Z 13 Acked 1012 Read 7 Done Done Sent false
207212024-03-06T21:06:20.264Z 14 Acked 1013 Write 10 Done Done Sent false
207222024-03-06T21:06:20.265Z 15 Acked 1014 Read 5 Done Done Sent false
207232024-03-06T21:06:20.265Z 16 Acked 1015 Write 4 Done Done Sent false
207242024-03-06T21:06:20.265Z 17 Acked 1016 Flush 0 Done Done Sent false
207252024-03-06T21:06:20.265Z 18 Acked 1017 Write 2 Done Done Sent false
207262024-03-06T21:06:20.265Z 19 Acked 1018 Read 9 Done Done Sent false
207272024-03-06T21:06:20.265Z 20 Acked 1019 Read 6 Done Done Sent false
207282024-03-06T21:06:20.265Z 21 Acked 1020 Read 2 Done Done Sent false
207292024-03-06T21:06:20.265Z 22 Acked 1021 Read 8 Done Done Sent false
207302024-03-06T21:06:20.265Z 23 Acked 1022 Read 2 Done Done Sent false
207312024-03-06T21:06:20.266Z 24 Acked 1023 Read 2 Done Done Sent false
207322024-03-06T21:06:20.266Z 25 Acked 1024 Write 1 Done Done Sent false
207332024-03-06T21:06:20.266Z 26 Acked 1025 Read 8 Done Done Sent false
207342024-03-06T21:06:20.266Z 27 Acked 1026 Read 10 Done Done Sent false
207352024-03-06T21:06:20.266Z 28 Acked 1027 Read 1 Done Done Sent false
207362024-03-06T21:06:20.266Z 29 Acked 1028 Read 3 Done Done Sent false
207372024-03-06T21:06:20.266Z 30 Acked 1029 Read 7 Done Done Sent false
207382024-03-06T21:06:20.266Z STATES DS:0 DS:1 DS:2 TOTAL
207392024-03-06T21:06:20.266Z New 0 0 0 0
207402024-03-06T21:06:20.267Z Sent 0 0 30 30
207412024-03-06T21:06:20.267Z Done 30 30 0 60
207422024-03-06T21:06:20.267Z Skipped 0 0 0 0
207432024-03-06T21:06:20.267Z Error 0 0 0 0
207442024-03-06T21:06:20.267ZLast Flush: 1016 1016 0
207452024-03-06T21:06:20.267ZDownstairs last five completed:
207462024-03-06T21:06:20.267ZUpstairs last five completed: 30 29 28 27 26
207472024-03-06T21:06:20.268Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:06:18.929592772Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1910}
207482024-03-06T21:06:20.268ZWrote out file "/tmp/test_repair_verify.data"
207492024-03-06T21:06:20.268Z
207502024-03-06T21:06:20.268ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
207512024-03-06T21:06:20.268ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
207522024-03-06T21:06:20.268Z 0 000-019 188 188 188 328 328 328 F F F
207532024-03-06T21:06:20.268Z 1 020-039 192 192 188 332 332 328 F F F <---
207542024-03-06T21:06:20.268Z 2 040-059 191 191 191 331 331 331 F F F
207552024-03-06T21:06:20.268Z 3 060-079 187 187 187 327 327 327 F F F
207562024-03-06T21:06:20.268Z 4 080-099 192 192 188 335 335 328 F F F <---
207572024-03-06T21:06:20.268Z 5 100-119 188 188 188 328 328 328 F F F
207582024-03-06T21:06:20.279Z 6 120-139 190 190 190 329 329 329 F F F
207592024-03-06T21:06:20.279Z 7 140-159 191 191 191 331 331 331 F F F
207602024-03-06T21:06:20.279Z 8 160-179 191 191 191 331 331 331 F F F
207612024-03-06T21:06:20.279Z 9 180-199 192 192 191 334 334 331 F F F <---
207622024-03-06T21:06:20.280Z 10 200-219 172 172 172 303 303 303 F F F
207632024-03-06T21:06:20.280Z 11 220-239 187 187 187 327 327 327 F F F
207642024-03-06T21:06:20.280Z 12 240-259 192 192 191 332 332 331 F F F <---
207652024-03-06T21:06:20.280Z 13 260-279 171 171 171 301 301 301 T T F <---
207662024-03-06T21:06:20.280Z 14 280-299 191 191 191 331 331 331 F F F
207672024-03-06T21:06:20.280Z 15 300-319 191 191 191 331 331 331 F F F
207682024-03-06T21:06:20.280Z 16 320-339 191 191 191 331 331 331 F F F
207692024-03-06T21:06:20.280Z 17 340-359 190 190 190 329 329 329 F F F
207702024-03-06T21:06:20.280Z 18 360-379 187 187 187 327 327 327 F F F
207712024-03-06T21:06:20.281Z 19 380-399 188 188 188 328 328 328 T T F <---
207722024-03-06T21:06:20.281Z 20 400-419 191 191 191 331 331 331 F F F
207732024-03-06T21:06:20.281Z 21 420-439 192 192 188 335 335 328 F F F <---
207742024-03-06T21:06:20.281Z 22 440-459 186 186 186 325 325 325 F F F
207752024-03-06T21:06:20.281Z 23 460-479 186 186 186 325 325 325 F F F
207762024-03-06T21:06:20.281Z 24 480-499 192 192 186 335 335 325 F F F <---
207772024-03-06T21:06:20.281Z 25 500-519 191 191 191 331 331 331 F F F
207782024-03-06T21:06:20.281Z 26 520-539 190 190 190 329 329 329 F F F
207792024-03-06T21:06:20.282Z 27 540-559 190 190 190 330 330 330 F F F
207802024-03-06T21:06:20.282Z 28 560-579 190 190 190 330 330 330 F F F
207812024-03-06T21:06:20.282Z 29 580-599 192 192 188 333 333 328 F F F <---
207822024-03-06T21:06:20.282ZMax gen: 192, Max flush: 335
207832024-03-06T21:06:20.282ZError: Difference in extent metadata found!
207842024-03-06T21:06:20.283ZA Difference in extent metadata is expected here
207852024-03-06T21:06:20.283ZOn loop 96
207862024-03-06T21:06:20.283Z
207872024-03-06T21:06:20.283ZVerifying data now
207882024-03-06T21:06:24.742ZNo change after verify
207892024-03-06T21:06:24.742ZLoop: 96 Downstairs dump after verify (and repair):
207902024-03-06T21:06:24.760ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
207912024-03-06T21:06:24.760Z 0 000-019 188 188 188 328 328 328 F F F
207922024-03-06T21:06:24.760Z 1 020-039 192 192 192 332 332 332 F F F
207932024-03-06T21:06:24.760Z 2 040-059 191 191 191 331 331 331 F F F
207942024-03-06T21:06:24.760Z 3 060-079 187 187 187 327 327 327 F F F
207952024-03-06T21:06:24.760Z 4 080-099 192 192 192 335 335 335 F F F
207962024-03-06T21:06:24.760Z 5 100-119 188 188 188 328 328 328 F F F
207972024-03-06T21:06:24.760Z 6 120-139 190 190 190 329 329 329 F F F
207982024-03-06T21:06:24.760Z 7 140-159 191 191 191 331 331 331 F F F
207992024-03-06T21:06:24.760Z 8 160-179 191 191 191 331 331 331 F F F
208002024-03-06T21:06:24.760Z 9 180-199 192 192 192 334 334 334 F F F
208012024-03-06T21:06:24.761Z 10 200-219 172 172 172 303 303 303 F F F
208022024-03-06T21:06:24.761Z 11 220-239 187 187 187 327 327 327 F F F
208032024-03-06T21:06:24.761Z 12 240-259 192 192 192 332 332 332 F F F
208042024-03-06T21:06:24.761Z 13 260-279 193 193 193 336 336 336 F F F
208052024-03-06T21:06:24.761Z 14 280-299 191 191 191 331 331 331 F F F
208062024-03-06T21:06:24.761Z 15 300-319 191 191 191 331 331 331 F F F
208072024-03-06T21:06:24.761Z 16 320-339 191 191 191 331 331 331 F F F
208082024-03-06T21:06:24.761Z 17 340-359 190 190 190 329 329 329 F F F
208092024-03-06T21:06:24.761Z 18 360-379 187 187 187 327 327 327 F F F
208102024-03-06T21:06:24.761Z 19 380-399 193 193 193 336 336 336 F F F
208112024-03-06T21:06:24.761Z 20 400-419 191 191 191 331 331 331 F F F
208122024-03-06T21:06:24.768Z 21 420-439 192 192 192 335 335 335 F F F
208132024-03-06T21:06:24.769Z 22 440-459 186 186 186 325 325 325 F F F
208142024-03-06T21:06:24.769Z 23 460-479 186 186 186 325 325 325 F F F
208152024-03-06T21:06:24.769Z 24 480-499 192 192 192 335 335 335 F F F
208162024-03-06T21:06:24.769Z 25 500-519 191 191 191 331 331 331 F F F
208172024-03-06T21:06:24.769Z 26 520-539 190 190 190 329 329 329 F F F
208182024-03-06T21:06:24.769Z 27 540-559 190 190 190 330 330 330 F F F
208192024-03-06T21:06:24.769Z 28 560-579 190 190 190 330 330 330 F F F
208202024-03-06T21:06:24.769Z 29 580-599 192 192 192 333 333 333 F F F
208212024-03-06T21:06:24.769ZMax gen: 193, Max flush: 336
208222024-03-06T21:06:24.769Z
208232024-03-06T21:06:24.769ZBegin loop 97
208242024-03-06T21:06:24.769ZDownstairs to restart: 2
208252024-03-06T21:06:24.778Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.510664878Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918}
208262024-03-06T21:06:24.788Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.510952698Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918}
208272024-03-06T21:06:24.788Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level"Crucible runtime is spawned
208282024-03-06T21:06:24.788Z:30,"time":"2024-03-06T21:06:23.511017554Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918}
208292024-03-06T21:06:24.788Z{"msg":"Crucible 053c74cb-254f-425a-a39c-ba283d2e7481 has session id: 59049fab-aa72-4176-8488-11fc4fdb91da","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.511049742Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208302024-03-06T21:06:24.789Z{"msg":"Upstairs opts: Upstairs UUID: 053c74cb-254f-425a-a39c-ba283d2e7481, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.511099958Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208312024-03-06T21:06:24.789Z{"msg":"Crucible stats registered with UUID: 053c74cb-254f-425a-a39c-ba283d2e7481","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.511121897Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208322024-03-06T21:06:24.789Z{"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.511174463Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918}
208332024-03-06T21:06:24.789Z{"msg":"053c74cb-254f-425a-a39c-ba283d2e7481 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.511196091Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208342024-03-06T21:06:24.789Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.511234149Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"client":"0","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208352024-03-06T21:06:24.789Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.511270937Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"client":"0","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208362024-03-06T21:06:24.789Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.511301614Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"client":"1","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208372024-03-06T21:06:24.789Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.511339672Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"client":"1","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208382024-03-06T21:06:24.789Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.511383119Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"client":"2","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208392024-03-06T21:06:24.790Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.511412367Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"client":"2","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208402024-03-06T21:06:24.790Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.511452384Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208412024-03-06T21:06:24.790Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.511499311Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208422024-03-06T21:06:24.790Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.511539878Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208432024-03-06T21:06:24.790Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.511565206Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208442024-03-06T21:06:24.790Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.511597224Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208452024-03-06T21:06:24.790Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.511623972Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208462024-03-06T21:06:24.790Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.512426507Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"client":"0","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208472024-03-06T21:06:24.790Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.512462944Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"client":"0","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208482024-03-06T21:06:24.790Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.512479623Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"client":"1","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208492024-03-06T21:06:24.790Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.512495202Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"client":"1","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208502024-03-06T21:06:24.790Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.51310979Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"client":"1","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208512024-03-06T21:06:24.790Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.513148587Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"client":"0","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208522024-03-06T21:06:24.791Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.513428978Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"client":"2","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208532024-03-06T21:06:24.791Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.513477964Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"client":"2","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208542024-03-06T21:06:24.791Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.513675471Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"client":"0","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208552024-03-06T21:06:24.791Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.513716378Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"client":"0","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208562024-03-06T21:06:24.791Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.513737567Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208572024-03-06T21:06:24.791Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.5138327Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"client":"1","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208582024-03-06T21:06:24.791Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.513866347Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"client":"1","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208592024-03-06T21:06:24.791Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.513881026Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208602024-03-06T21:06:24.791Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.514083073Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"client":"2","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208612024-03-06T21:06:24.791Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.514505703Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"client":"2","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208622024-03-06T21:06:24.791Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.51455513Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"client":"2","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208632024-03-06T21:06:24.791Z{"msg":"[0]R flush_numbers[0..12]: [328, 332, 331, 327, 335, 328, 329, 331, 331, 334, 303, 327]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.514572739Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"":"downstairs","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208642024-03-06T21:06:24.791Z{"msg":"[0]R generation[0..12]: [188, 192, 191, 187, 192, 188, 190, 191, 191, 192, 172, 187]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.514598317Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"":"downstairs","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208652024-03-06T21:06:24.791Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.514613026Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"":"downstairs","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208662024-03-06T21:06:24.791Z{"msg":"[1]R flush_numbers[0..12]: [328, 332, 331, 327, 335, 328, 329, 331, 331, 334, 303, 327]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.514634135Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"":"downstairs","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208672024-03-06T21:06:24.791Z{"msg":"[1]R generation[0..12]: [188, 192, 191, 187, 192, 188, 190, 191, 191, 192, 172, 187]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.514648704Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"":"downstairs","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208682024-03-06T21:06:24.791Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.514670652Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"":"downstairs","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208692024-03-06T21:06:24.792Z{"msg":"[2]R flush_numbers[0..12]: [328, 332, 331, 327, 335, 328, 329, 331, 331, 334, 303, 327]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.514685961Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"":"downstairs","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208702024-03-06T21:06:24.792ZWait for a query_work_queue command to finish before sending IO
208712024-03-06T21:06:24.792Z{"msg":"[2]R generation[0..12]: [188, 192, 191, 187, 192, 188, 190, 191, 191, 192, 172, 187]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.514713399Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"":"downstairs","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208722024-03-06T21:06:24.792Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.514729868Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"":"downstairs","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208732024-03-06T21:06:24.792Z{"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.514749357Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"":"downstairs","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208742024-03-06T21:06:24.792Z{"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.514763066Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"":"downstairs","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208752024-03-06T21:06:24.792Z{"msg":"Next flush: 337","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.514785074Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"":"downstairs","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208762024-03-06T21:06:24.792Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.514799033Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"":"downstairs","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208772024-03-06T21:06:24.792Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.514815282Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208782024-03-06T21:06:24.792Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.514828711Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"client":"0","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208792024-03-06T21:06:24.793Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.51484251Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"client":"1","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208802024-03-06T21:06:24.793Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.514856709Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"client":"2","":"downstairs","session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208812024-03-06T21:06:24.793Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.514871278Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208822024-03-06T21:06:24.793Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.514883477Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208832024-03-06T21:06:24.793Z{"msg":"053c74cb-254f-425a-a39c-ba283d2e7481 is now active with session: 59049fab-aa72-4176-8488-11fc4fdb91da","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.514897267Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208842024-03-06T21:06:24.793Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.514918655Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918,"session_id":"59049fab-aa72-4176-8488-11fc4fdb91da"}
208852024-03-06T21:06:24.793Z{"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:23.514931824Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918}
208862024-03-06T21:06:24.793ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
208872024-03-06T21:06:24.793ZLoading write count information from file "/tmp/test_repair_verify.data"
208882024-03-06T21:06:24.793ZRun Repair workload
208892024-03-06T21:06:24.794Z01/30 Write block 41 len 40960 data: 20 22 20 19 14 14 16 14 13 10
208902024-03-06T21:06:24.802Z02/30 Write block 134 len 32768 data: 12 12 12 9 7 8 8 6
208912024-03-06T21:06:24.802Z03/30 Read block 27 len 32768
208922024-03-06T21:06:24.803Z04/30 Write block 337 len 16384 data: 14 18 16 20
208932024-03-06T21:06:24.803Z05/30 Write block 419 len 36864 data: 14 14 18 15 17 14 14 15 16
208942024-03-06T21:06:24.803Z06/30 Read block 13 len 28672
208952024-03-06T21:06:24.803Z07/30 Write block 383 len 8192 data: 20 20
208962024-03-06T21:06:24.803Z08/30 Write block 23 len 32768 data: 19 16 15 15 15 14 16 14
208972024-03-06T21:06:24.804Z09/30 Write block 10 len 32768 data: 16 18 16 17 19 21 18 20
208982024-03-06T21:06:24.804Z10/30 Flush
208992024-03-06T21:06:24.804Z11/30 Write block 506 len 12288 data: 12 12 12
209002024-03-06T21:06:24.804Z12/30 Read block 226 len 40960
209012024-03-06T21:06:24.804Z13/30 Read block 102 len 8192
209022024-03-06T21:06:24.804Z14/30 Read block 375 len 36864
209032024-03-06T21:06:24.804Z15/30 Flush
209042024-03-06T21:06:24.812Z16/30 Write block 263 len 40960 data: 10 9 13 12 12 13 13 12 12 10
209052024-03-06T21:06:24.812Z17/30 Read block 579 len 8192
209062024-03-06T21:06:24.812Z18/30 Read block 382 len 24576
209072024-03-06T21:06:24.812Z19/30 Write block 229 len 40960 data: 14 17 17 15 14 14 17 16 20 23
209082024-03-06T21:06:24.813Z20/30 Read block 385 len 40960
209092024-03-06T21:06:24.813Z21/30 Read block 513 len 16384
209102024-03-06T21:06:24.813Z22/30 Read block 564 len 16384
209112024-03-06T21:06:24.843Z23/30 Write block 504 len 32768 data: 10 11 13 13 13 13 15 16
209122024-03-06T21:06:24.843Z24/30 Read block 160 len 28672
209132024-03-06T21:06:24.843Z25/30 Write block 169 len 36864 data: 15 13 13 16 19 17 18 16 13
209142024-03-06T21:06:24.844Z26/30 Flush
209152024-03-06T21:06:24.844Z27/30 Write block 477 len 16384 data: 11 13 8 11
209162024-03-06T21:06:24.844Z28/30 Write block 200 len 20480 data: 16 13 11 11 11
209172024-03-06T21:06:24.844Z29/30 Write block 486 len 12288 data: 15 14 16
209182024-03-06T21:06:24.844Z30/30 Write block 375 len 8192 data: 20 16
209192024-03-06T21:06:24.844Z----------------------------------------------------------------
209202024-03-06T21:06:24.844Z Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:30
209212024-03-06T21:06:24.845ZStates: Active Active Active
209222024-03-06T21:06:24.845ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
209232024-03-06T21:06:24.845Z 1 Acked 1000 Write 10 Done Done Done false
209242024-03-06T21:06:24.845Z 2 Acked 1001 Write 8 Done Done Done false
209252024-03-06T21:06:24.845Z 3 Acked 1002 Read 8 Done Done Done false
209262024-03-06T21:06:24.845Z 4 Acked 1003 Write 4 Done Done Sent false
209272024-03-06T21:06:24.845Z 5 Acked 1004 Write 9 Done Done Sent false
209282024-03-06T21:06:24.845Z 6 Acked 1005 Read 7 Done Done Sent false
209292024-03-06T21:06:24.845Z 7 Acked 1006 Write 2 Done Done Sent false
209302024-03-06T21:06:24.845Z 8 Acked 1007 Write 8 Done Done Sent false
209312024-03-06T21:06:24.846Z 9 Acked 1008 Write 8 Done Done Sent false
209322024-03-06T21:06:24.846Z 10 Acked 1009 Flush 0 Done Done Sent false
209332024-03-06T21:06:24.846Z 11 Acked 1010 Write 3 Done Done Sent false
209342024-03-06T21:06:24.846Z 12 Acked 1011 Read 10 Done Done Sent false
209352024-03-06T21:06:24.847Z 13 Acked 1012 Read 2 Done Done Sent false
209362024-03-06T21:06:24.847Z 14 Acked 1013 Read 9 Done Done Sent false
209372024-03-06T21:06:24.847Z 15 Acked 1014 Flush 0 Done Done Sent false
209382024-03-06T21:06:24.847Z 16 Acked 1015 Write 10 Done Done Sent false
209392024-03-06T21:06:24.847Z 17 Acked 1016 Read 2 Done Done Sent false
209402024-03-06T21:06:24.847Z 18 Acked 1017 Read 6 Done Done Sent false
209412024-03-06T21:06:24.847Z 19 Acked 1018 Write 10 Done Done Sent false
209422024-03-06T21:06:24.847Z 20 Acked 1019 Read 10 Done Done Sent false
209432024-03-06T21:06:24.848Z 21 Acked 1020 Read 4 Done Done Sent false
209442024-03-06T21:06:24.848Z 22 Acked 1021 Read 4 Done Done Sent false
209452024-03-06T21:06:24.848Z 23 Acked 1022 Write 8 Done Done Sent false
209462024-03-06T21:06:24.848Z 24 Acked 1023 Read 7 Done Done Sent false
209472024-03-06T21:06:24.848Z 25 Acked 1024 Write 9 Done Done Sent false
209482024-03-06T21:06:24.848Z 26 Acked 1025 Flush 0 Done Done Sent false
209492024-03-06T21:06:24.849Z 27 Acked 1026 Write 4 Done Done Sent false
209502024-03-06T21:06:24.849Z 28 Acked 1027 Write 5 Done Done Sent false
209512024-03-06T21:06:24.849Z 29 Acked 1028 Write 3 Done Done Sent false
209522024-03-06T21:06:24.849Z 30 Acked 1029 Write 2 Sent Sent Sent false
209532024-03-06T21:06:24.849Z STATES DS:0 DS:1 DS:2 TOTAL
209542024-03-06T21:06:24.850Z New 0 0 0 0
209552024-03-06T21:06:24.850Z Sent 1 1 27 29
209562024-03-06T21:06:24.850Z Done 29 29 3 61
209572024-03-06T21:06:24.850Z Skipped 0 0 0 0
209582024-03-06T21:06:24.850Z Error 0 0 0 0
209592024-03-06T21:06:24.851ZLast Flush: 1025 1025 0
209602024-03-06T21:06:24.851ZDownstairs last five completed:
209612024-03-06T21:06:24.851ZUpstairs last five completed: 30 29 28 27 26
209622024-03-06T21:06:24.851Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:06:23.55533519Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1918}
209632024-03-06T21:06:24.851ZWrote out file "/tmp/test_repair_verify.data"
209642024-03-06T21:06:24.862Z
209652024-03-06T21:06:24.862ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
209662024-03-06T21:06:24.863ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
209672024-03-06T21:06:24.863Z 0 000-019 194 194 188 337 337 328 F F F <---
209682024-03-06T21:06:24.863Z 1 020-039 194 194 192 337 337 332 F F F <---
209692024-03-06T21:06:24.863Z 2 040-059 194 194 191 337 337 331 F F T <---
209702024-03-06T21:06:24.864Z 3 060-079 187 187 187 327 327 327 F F F
209712024-03-06T21:06:24.864Z 4 080-099 192 192 192 335 335 335 F F F
209722024-03-06T21:06:24.864Z 5 100-119 188 188 188 328 328 328 F F F
209732024-03-06T21:06:24.864Z 6 120-139 194 194 190 337 337 329 F F T <---
209742024-03-06T21:06:24.864Z 7 140-159 194 194 191 337 337 331 F F T <---
209752024-03-06T21:06:24.864Z 8 160-179 194 194 191 339 339 331 F F F <---
209762024-03-06T21:06:24.864Z 9 180-199 192 192 192 334 334 334 F F F
209772024-03-06T21:06:24.865Z 10 200-219 172 172 172 303 303 303 T T F <---
209782024-03-06T21:06:24.865Z 11 220-239 194 194 187 339 339 327 F F F <---
209792024-03-06T21:06:24.865Z 12 240-259 192 192 192 332 332 332 F F F
209802024-03-06T21:06:24.865Z 13 260-279 194 194 193 339 339 336 F F F <---
209812024-03-06T21:06:24.865Z 14 280-299 191 191 191 331 331 331 F F F
209822024-03-06T21:06:24.866Z 15 300-319 191 191 191 331 331 331 F F F
209832024-03-06T21:06:24.866Z 16 320-339 194 194 191 337 337 331 F F F <---
209842024-03-06T21:06:24.866Z 17 340-359 194 194 190 337 337 329 F F F <---
209852024-03-06T21:06:24.866Z 18 360-379 187 187 187 327 327 327 T T F <---
209862024-03-06T21:06:24.866Z 19 380-399 194 194 193 337 337 336 F F F <---
209872024-03-06T21:06:24.866Z 20 400-419 194 194 191 337 337 331 F F F <---
209882024-03-06T21:06:24.867Z 21 420-439 194 194 192 337 337 335 F F F <---
209892024-03-06T21:06:24.867Z 22 440-459 186 186 186 325 325 325 F F F
209902024-03-06T21:06:24.867Z 23 460-479 186 186 186 325 325 325 T T F <---
209912024-03-06T21:06:24.867Z 24 480-499 192 192 192 335 335 335 T T F <---
209922024-03-06T21:06:24.867Z 25 500-519 194 194 191 339 339 331 F F F <---
209932024-03-06T21:06:24.868Z 26 520-539 190 190 190 329 329 329 F F F
209942024-03-06T21:06:24.868Z 27 540-559 190 190 190 330 330 330 F F F
209952024-03-06T21:06:24.868Z 28 560-579 190 190 190 330 330 330 F F F
209962024-03-06T21:06:24.868Z 29 580-599 192 192 192 333 333 333 F F F
209972024-03-06T21:06:24.868ZMax gen: 194, Max flush: 339
209982024-03-06T21:06:24.868ZError: Difference in extent metadata found!
209992024-03-06T21:06:24.868ZA Difference in extent metadata is expected here
210002024-03-06T21:06:24.869ZOn loop 97
210012024-03-06T21:06:24.869Z
210022024-03-06T21:06:24.869ZVerifying data now
210032024-03-06T21:06:30.763ZNo change after verify
210042024-03-06T21:06:30.763ZLoop: 97 Downstairs dump after verify (and repair):
210052024-03-06T21:06:30.776ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
210062024-03-06T21:06:30.776Z 0 000-019 194 194 194 337 337 337 F F F
210072024-03-06T21:06:30.776Z 1 020-039 194 194 194 337 337 337 F F F
210082024-03-06T21:06:30.776Z 2 040-059 194 194 194 337 337 337 F F F
210092024-03-06T21:06:30.776Z 3 060-079 187 187 187 327 327 327 F F F
210102024-03-06T21:06:30.776Z 4 080-099 192 192 192 335 335 335 F F F
210112024-03-06T21:06:30.776Z 5 100-119 188 188 188 328 328 328 F F F
210122024-03-06T21:06:30.776Z 6 120-139 194 194 194 337 337 337 F F F
210132024-03-06T21:06:30.776Z 7 140-159 194 194 194 337 337 337 F F F
210142024-03-06T21:06:30.777Z 8 160-179 194 194 194 339 339 339 F F F
210152024-03-06T21:06:30.777Z 9 180-199 192 192 192 334 334 334 F F F
210162024-03-06T21:06:30.777Z 10 200-219 195 195 195 340 340 340 F F F
210172024-03-06T21:06:30.777Z 11 220-239 194 194 194 339 339 339 F F F
210182024-03-06T21:06:30.777Z 12 240-259 192 192 192 332 332 332 F F F
210192024-03-06T21:06:30.781Z 13 260-279 194 194 194 339 339 339 F F F
210202024-03-06T21:06:30.781Z 14 280-299 191 191 191 331 331 331 F F F
210212024-03-06T21:06:30.781Z 15 300-319 191 191 191 331 331 331 F F F
210222024-03-06T21:06:30.781Z 16 320-339 194 194 194 337 337 337 F F F
210232024-03-06T21:06:30.781Z 17 340-359 194 194 194 337 337 337 F F F
210242024-03-06T21:06:30.781Z 18 360-379 195 195 195 340 340 340 F F F
210252024-03-06T21:06:30.781Z 19 380-399 194 194 194 337 337 337 F F F
210262024-03-06T21:06:30.781Z 20 400-419 194 194 194 337 337 337 F F F
210272024-03-06T21:06:30.781Z 21 420-439 194 194 194 337 337 337 F F F
210282024-03-06T21:06:30.781Z 22 440-459 186 186 186 325 325 325 F F F
210292024-03-06T21:06:30.781Z 23 460-479 195 195 195 340 340 340 F F F
210302024-03-06T21:06:30.781Z 24 480-499 195 195 195 340 340 340 F F F
210312024-03-06T21:06:30.781Z 25 500-519 194 194 194 339 339 339 F F F
210322024-03-06T21:06:30.781Z 26 520-539 190 190 190 329 329 329 F F F
210332024-03-06T21:06:30.781Z 27 540-559 190 190 190 330 330 330 F F F
210342024-03-06T21:06:30.781Z 28 560-579 190 190 190 330 330 330 F F F
210352024-03-06T21:06:30.781Z 29 580-599 192 192 192 333 333 333 F F F
210362024-03-06T21:06:30.781ZMax gen: 195, Max flush: 340
210372024-03-06T21:06:30.784Z
210382024-03-06T21:06:30.785ZBegin loop 98
210392024-03-06T21:06:30.785ZDownstairs to restart: 0
210402024-03-06T21:06:30.804Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.538288242Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926}
210412024-03-06T21:06:30.804ZCrucible runtime is spawned
210422024-03-06T21:06:30.804Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.538689395Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926}
210432024-03-06T21:06:30.804Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.538730222Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926}
210442024-03-06T21:06:30.812Z{"msg":"Crucible 4ef7a08a-655a-4868-a17d-ffcd8eb5f9d6 has session id: 8ef7f4a3-536a-47e4-ad91-d6cb4083e33d","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.538748501Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210452024-03-06T21:06:30.812Z{"msg":"Upstairs opts: Upstairs UUID: 4ef7a08a-655a-4868-a17d-ffcd8eb5f9d6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.538780379Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210462024-03-06T21:06:30.812Z{"msg":"Crucible stats registered with UUID: 4ef7a08a-655a-4868-a17d-ffcd8eb5f9d6","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.538795178Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210472024-03-06T21:06:30.812Z{"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.538807917Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926}
210482024-03-06T21:06:30.812Z{"msg":"4ef7a08a-655a-4868-a17d-ffcd8eb5f9d6 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.538819356Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210492024-03-06T21:06:30.812Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.538831985Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"client":"0","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210502024-03-06T21:06:30.812Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.538846404Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"client":"0","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210512024-03-06T21:06:30.812Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.538860503Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"client":"1","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210522024-03-06T21:06:30.812Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.538880262Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"client":"1","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210532024-03-06T21:06:30.812Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.5389014Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"client":"2","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210542024-03-06T21:06:30.812Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.538927969Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"client":"2","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210552024-03-06T21:06:30.812Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.539020752Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210562024-03-06T21:06:30.812Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.539060469Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210572024-03-06T21:06:30.812Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.539076278Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210582024-03-06T21:06:30.812Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.539261275Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210592024-03-06T21:06:30.813Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.539305722Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210602024-03-06T21:06:30.813Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.53933669Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210612024-03-06T21:06:30.813Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.539961447Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"client":"1","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210622024-03-06T21:06:30.813Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.540011454Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"client":"1","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210632024-03-06T21:06:30.813Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.540045661Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"client":"2","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210642024-03-06T21:06:30.813Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.540085659Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"client":"2","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210652024-03-06T21:06:30.813Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.540656939Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"client":"1","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210662024-03-06T21:06:30.813Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.540713345Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"client":"2","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210672024-03-06T21:06:30.813Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.540850076Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"client":"0","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210682024-03-06T21:06:30.813Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.540895763Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"client":"0","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210692024-03-06T21:06:30.813Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.541292655Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"client":"2","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210702024-03-06T21:06:30.813Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.541331793Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"client":"2","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210712024-03-06T21:06:30.813Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.541356421Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210722024-03-06T21:06:30.813Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.541385689Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"client":"1","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210732024-03-06T21:06:30.813Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.541409447Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"client":"1","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210742024-03-06T21:06:30.813Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.541439715Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210752024-03-06T21:06:30.813Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.541594775Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"client":"0","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210762024-03-06T21:06:30.813Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.542025175Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"client":"0","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210772024-03-06T21:06:30.813Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.542077261Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"client":"0","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210782024-03-06T21:06:30.813Z{"msg":"[0]R flush_numbers[0..12]: [337, 337, 337, 327, 335, 328, 337, 337, 339, 334, 340, 339]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.542113999Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"":"downstairs","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210792024-03-06T21:06:30.813Z{"msg":"[0]R generation[0..12]: [194, 194, 194, 187, 192, 188, 194, 194, 194, 192, 195, 194]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.542146996Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"":"downstairs","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210802024-03-06T21:06:30.813Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.542180364Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"":"downstairs","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210812024-03-06T21:06:30.813Z{"msg":"[1]R flush_numbers[0..12]: [337, 337, 337, 327, 335, 328, 337, 337, 339, 334, 340, 339]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.542213542Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO
210822024-03-06T21:06:30.814Zsession_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210832024-03-06T21:06:30.814Z{"msg":"[1]R generation[0..12]: [194, 194, 194, 187, 192, 188, 194, 194, 194, 192, 195, 194]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.542249869Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"":"downstairs","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210842024-03-06T21:06:30.814Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.542282507Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"":"downstairs","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210852024-03-06T21:06:30.814Z{"msg":"[2]R flush_numbers[0..12]: [337, 337, 337, 327, 335, 328, 337, 337, 339, 334, 340, 339]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.542314975Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"":"downstairs","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210862024-03-06T21:06:30.814Z{"msg":"[2]R generation[0..12]: [194, 194, 194, 187, 192, 188, 194, 194, 194, 192, 195, 194]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.542351142Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"":"downstairs","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210872024-03-06T21:06:30.814Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.54238607Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"":"downstairs","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210882024-03-06T21:06:30.814Z{"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.542419678Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"":"downstairs","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210892024-03-06T21:06:30.814Z{"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.542452886Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"":"downstairs","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210902024-03-06T21:06:30.814Z{"msg":"Next flush: 341","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.542485083Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"":"downstairs","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210912024-03-06T21:06:30.814Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.542519481Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"":"downstairs","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210922024-03-06T21:06:30.814Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.542553148Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210932024-03-06T21:06:30.814Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.542585186Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"client":"0","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210942024-03-06T21:06:30.814Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.542618184Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"client":"1","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210952024-03-06T21:06:30.814Z{"msg":"Transition from WaitQuorum to Active"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
210962024-03-06T21:06:30.814Z,"v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.54267346Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"client":"2","":"downstairs","session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210972024-03-06T21:06:30.814Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.542707008Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210982024-03-06T21:06:30.814Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.542727507Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
210992024-03-06T21:06:30.814Z{"msg":"4ef7a08a-655a-4868-a17d-ffcd8eb5f9d6 is now active with session: 8ef7f4a3-536a-47e4-ad91-d6cb4083e33d","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.542756385Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
211002024-03-06T21:06:30.814Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.542770214Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926,"session_id":"8ef7f4a3-536a-47e4-ad91-d6cb4083e33d"}
211012024-03-06T21:06:30.814Z{"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:29.542800182Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926}
211022024-03-06T21:06:30.818ZLoading write count information from file "/tmp/test_repair_verify.data"
211032024-03-06T21:06:30.818ZRun Repair workload
211042024-03-06T21:06:30.818Z01/30 Write block 360 len 28672 data: 18 22 17 20 18 16 14
211052024-03-06T21:06:30.818Z02/30 Flush
211062024-03-06T21:06:30.818Z03/30 Write block 490 len 32768 data: 14 15 14 17 16 18 18 18
211072024-03-06T21:06:30.818Z04/30 Flush
211082024-03-06T21:06:30.818Z05/30 Write block 357 len 4096 data: 17
211092024-03-06T21:06:30.818Z06/30 Read block 176 len 28672
211102024-03-06T21:06:30.821Z07/30 Read block 243 len 32768
211112024-03-06T21:06:30.821Z08/30 Write block 577 len 36864 data: 13 11 13 12 15 14 16 18 18
211122024-03-06T21:06:30.821Z09/30 Read block 58 len 4096
211132024-03-06T21:06:30.824Z10/30 Read block 208 len 32768
211142024-03-06T21:06:30.824Z11/30 Write block 383 len 4096 data: 21
211152024-03-06T21:06:30.824Z12/30 Read block 231 len 16384
211162024-03-06T21:06:30.824Z13/30 Flush
211172024-03-06T21:06:30.828Z14/30 Write block 499 len 32768 data: 13 12 8 9 9 11 12 14
211182024-03-06T21:06:30.828Z15/30 Read block 84 len 24576
211192024-03-06T21:06:30.831Z16/30 Read block 105 len 16384
211202024-03-06T21:06:30.831Z17/30 Read block 234 len 20480
211212024-03-06T21:06:30.831Z18/30 Write block 254 len 4096 data: 10
211222024-03-06T21:06:30.831Z19/30 Read block 120 len 8192
211232024-03-06T21:06:30.835Z20/30 Read block 465 len 8192
211242024-03-06T21:06:30.835Z21/30 Write block 462 len 28672 data: 16 21 21 20 17 17 16
211252024-03-06T21:06:30.835Z22/30 Write block 262 len 28672 data: 8 11 10 14 13 13 14
211262024-03-06T21:06:30.835Z23/30 Flush
211272024-03-06T21:06:30.838Z24/30 Write block 373 len 12288 data: 17 16 21
211282024-03-06T21:06:30.838Z25/30 Read block 239 len 16384
211292024-03-06T21:06:30.841Z26/30 Write block 209 len 12288 data: 12 15 12
211302024-03-06T21:06:30.841Z27/30 Read block 143 len 36864
211312024-03-06T21:06:30.841Z28/30 Read block 307 len 40960
211322024-03-06T21:06:30.845Z29/30 Read block 423 len 4096
211332024-03-06T21:06:30.845Z30/30 Read block 448 len 24576
211342024-03-06T21:06:30.845Z----------------------------------------------------------------
211352024-03-06T21:06:30.845Z Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:26
211362024-03-06T21:06:30.845ZStates: Active Active Active
211372024-03-06T21:06:30.845ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
211382024-03-06T21:06:30.845Z 5 Acked 1004 Write 1 Sent Done Done false
211392024-03-06T21:06:30.845Z 6 Acked 1005 Read 7 Sent Done Done false
211402024-03-06T21:06:30.845Z 7 Acked 1006 Read 8 Sent Done Done false
211412024-03-06T21:06:30.845Z 8 Acked 1007 Write 9 Sent Done Done false
211422024-03-06T21:06:30.845Z 9 Acked 1008 Read 1 Sent Done Done false
211432024-03-06T21:06:30.845Z 10 Acked 1009 Read 8 Sent Done Done false
211442024-03-06T21:06:30.845Z 11 Acked 1010 Write 1 Sent Done Done false
211452024-03-06T21:06:30.845Z 12 Acked 1011 Read 4 Sent Done Done false
211462024-03-06T21:06:30.845Z 13 Acked 1012 Flush 0 Sent Done Done false
211472024-03-06T21:06:30.845Z 14 Acked 1013 Write 8 Sent Done Done false
211482024-03-06T21:06:30.846Z 15 Acked 1014 Read 6 Sent Done Done false
211492024-03-06T21:06:30.846Z 16 Acked 1015 Read 4 Sent Done Done false
211502024-03-06T21:06:30.846Z 17 Acked 1016 Read 5 Sent Done Done false
211512024-03-06T21:06:30.846Z 18 Acked 1017 Write 1 Sent Done Done false
211522024-03-06T21:06:30.846Z 19 Acked 1018 Read 2 Sent Done Done false
211532024-03-06T21:06:30.846Z 20 Acked 1019 Read 2 Sent Done Done false
211542024-03-06T21:06:30.846Z 21 Acked 1020 Write 7 Sent Done Done false
211552024-03-06T21:06:30.846Z 22 Acked 1021 Write 7 Sent Done Done false
211562024-03-06T21:06:30.846Z 23 Acked 1022 Flush 0 Sent Done Done false
211572024-03-06T21:06:30.846Z 24 Acked 1023 Write 3 Sent Done Done false
211582024-03-06T21:06:30.846Z 25 Acked 1024 Read 4 Sent Done Done false
211592024-03-06T21:06:30.846Z 26 Acked 1025 Write 3 Sent Done Done false
211602024-03-06T21:06:30.846Z 27 Acked 1026 Read 9 Sent Done Done false
211612024-03-06T21:06:30.846Z 28 Acked 1027 Read 10 Sent Done Done false
211622024-03-06T21:06:30.846Z 29 Acked 1028 Read 1 Sent Done Done false
211632024-03-06T21:06:30.846Z 30 Acked 1029 Read 6 Sent Done Done false
211642024-03-06T21:06:30.846Z STATES DS:0 DS:1 DS:2 TOTAL
211652024-03-06T21:06:30.846Z New 0 0 0 0
211662024-03-06T21:06:30.846Z Sent 26 0 0 26
211672024-03-06T21:06:30.846Z Done 0 26 26 52
211682024-03-06T21:06:30.846Z Skipped 0 0 0 0
211692024-03-06T21:06:30.846Z Error 0 0 0 0
211702024-03-06T21:06:30.846ZLast Flush: 1003 1022 1022
211712024-03-06T21:06:30.847ZDownstairs last five completed: 1003 1002 1001 1000
211722024-03-06T21:06:30.847ZUpstairs last five completed: 30 29 28 27 26
211732024-03-06T21:06:30.847Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:06:29.578702096Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1926}
211742024-03-06T21:06:30.847ZWrote out file "/tmp/test_repair_verify.data"
211752024-03-06T21:06:30.850Z
211762024-03-06T21:06:30.853ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
211772024-03-06T21:06:30.869ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
211782024-03-06T21:06:30.869Z 0 000-019 194 194 194 337 337 337 F F F
211792024-03-06T21:06:30.869Z 1 020-039 194 194 194 337 337 337 F F F
211802024-03-06T21:06:30.869Z 2 040-059 194 194 194 337 337 337 F F F
211812024-03-06T21:06:30.869Z 3 060-079 187 187 187 327 327 327 F F F
211822024-03-06T21:06:30.869Z 4 080-099 192 192 192 335 335 335 F F F
211832024-03-06T21:06:30.869Z 5 100-119 188 188 188 328 328 328 F F F
211842024-03-06T21:06:30.869Z 6 120-139 194 194 194 337 337 337 F F F
211852024-03-06T21:06:30.869Z 7 140-159 194 194 194 337 337 337 F F F
211862024-03-06T21:06:30.869Z 8 160-179 194 194 194 339 339 339 F F F
211872024-03-06T21:06:30.869Z 9 180-199 192 192 192 334 334 334 F F F
211882024-03-06T21:06:30.869Z 10 200-219 195 195 195 340 340 340 F T T <---
211892024-03-06T21:06:30.869Z 11 220-239 194 194 194 339 339 339 F F F
211902024-03-06T21:06:30.869Z 12 240-259 192 196 196 332 344 344 F F F <---
211912024-03-06T21:06:30.869Z 13 260-279 194 196 196 339 344 344 F F F <---
211922024-03-06T21:06:30.869Z 14 280-299 191 191 191 331 331 331 F F F
211932024-03-06T21:06:30.869Z 15 300-319 191 191 191 331 331 331 F F F
211942024-03-06T21:06:30.869Z 16 320-339 194 194 194 337 337 337 F F F
211952024-03-06T21:06:30.869Z 17 340-359 194 196 196 337 343 343 F F F <---
211962024-03-06T21:06:30.870Z 18 360-379 196 196 196 341 341 341 F T T <---
211972024-03-06T21:06:30.878Z 19 380-399 194 196 196 337 343 343 F F F <---
211982024-03-06T21:06:30.878Z 20 400-419 194 194 194 337 337 337 F F F
211992024-03-06T21:06:30.878Z 21 420-439 194 194 194 337 337 337 F F F
212002024-03-06T21:06:30.878Z 22 440-459 186 186 186 325 325 325 F F F
212012024-03-06T21:06:30.878Z 23 460-479 195 196 196 340 344 344 F F F <---
212022024-03-06T21:06:30.878Z 24 480-499 196 196 196 342 344 344 F F F <---
212032024-03-06T21:06:30.878Z 25 500-519 194 196 196 339 344 344 F F F <---
212042024-03-06T21:06:30.878Z 26 520-539 190 190 190 329 329 329 F F F
212052024-03-06T21:06:30.878Z 27 540-559 190 190 190 330 330 330 F F F
212062024-03-06T21:06:30.878Z 28 560-579 190 196 196 330 343 343 F F F <---
212072024-03-06T21:06:30.878Z 29 580-599 192 196 196 333 343 343 F F F <---
212082024-03-06T21:06:30.878ZMax gen: 196, Max flush: 344
212092024-03-06T21:06:30.878ZError: Difference in extent metadata found!
212102024-03-06T21:06:30.878ZA Difference in extent metadata is expected here
212112024-03-06T21:06:30.878ZOn loop 98
212122024-03-06T21:06:30.878Z
212132024-03-06T21:06:30.883ZVerifying data now
212142024-03-06T21:06:35.477ZNo change after verify
212152024-03-06T21:06:35.477ZLoop: 98 Downstairs dump after verify (and repair):
212162024-03-06T21:06:35.492ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
212172024-03-06T21:06:35.492Z 0 000-019 194 194 194 337 337 337 F F F
212182024-03-06T21:06:35.492Z 1 020-039 194 194 194 337 337 337 F F F
212192024-03-06T21:06:35.492Z 2 040-059 194 194 194 337 337 337 F F F
212202024-03-06T21:06:35.492Z 3 060-079 187 187 187 327 327 327 F F F
212212024-03-06T21:06:35.492Z 4 080-099 192 192 192 335 335 335 F F F
212222024-03-06T21:06:35.492Z 5 100-119 188 188 188 328 328 328 F F F
212232024-03-06T21:06:35.492Z 6 120-139 194 194 194 337 337 337 F F F
212242024-03-06T21:06:35.492Z 7 140-159 194 194 194 337 337 337 F F F
212252024-03-06T21:06:35.492Z 8 160-179 194 194 194 339 339 339 F F F
212262024-03-06T21:06:35.492Z 9 180-199 192 192 192 334 334 334 F F F
212272024-03-06T21:06:35.492Z 10 200-219 197 197 197 345 345 345 F F F
212282024-03-06T21:06:35.493Z 11 220-239 194 194 194 339 339 339 F F F
212292024-03-06T21:06:35.493Z 12 240-259 196 196 196 344 344 344 F F F
212302024-03-06T21:06:35.493Z 13 260-279 196 196 196 344 344 344 F F F
212312024-03-06T21:06:35.493Z 14 280-299 191 191 191 331 331 331 F F F
212322024-03-06T21:06:35.493Z 15 300-319 191 191 191 331 331 331 F F F
212332024-03-06T21:06:35.497Z 16 320-339 194 194 194 337 337 337 F F F
212342024-03-06T21:06:35.497Z 17 340-359 196 196 196 343 343 343 F F F
212352024-03-06T21:06:35.497Z 18 360-379 197 197 197 345 345 345 F F F
212362024-03-06T21:06:35.497Z 19 380-399 196 196 196 343 343 343 F F F
212372024-03-06T21:06:35.497Z 20 400-419 194 194 194 337 337 337 F F F
212382024-03-06T21:06:35.497Z 21 420-439 194 194 194 337 337 337 F F F
212392024-03-06T21:06:35.497Z 22 440-459 186 186 186 325 325 325 F F F
212402024-03-06T21:06:35.497Z 23 460-479 196 196 196 344 344 344 F F F
212412024-03-06T21:06:35.497Z 24 480-499 196 196 196 344 344 344 F F F
212422024-03-06T21:06:35.497Z 25 500-519 196 196 196 344 344 344 F F F
212432024-03-06T21:06:35.497Z 26 520-539 190 190 190 329 329 329 F F F
212442024-03-06T21:06:35.497Z 27 540-559 190 190 190 330 330 330 F F F
212452024-03-06T21:06:35.497Z 28 560-579 196 196 196 343 343 343 F F F
212462024-03-06T21:06:35.497Z 29 580-599 196 196 196 343 343 343 F F F
212472024-03-06T21:06:35.497ZMax gen: 197, Max flush: 345
212482024-03-06T21:06:35.497Z
212492024-03-06T21:06:35.497ZBegin loop 99
212502024-03-06T21:06:35.497ZDownstairs to restart: 1
212512024-03-06T21:06:35.518Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.251719937Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934}
212522024-03-06T21:06:35.518Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"27c060b02133783737d505ea2610b3d179ebd892\",\n git_commit_timestamp: \"2024-03-06T20:36:21.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.251938112Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934}
212532024-03-06T21:06:35.518Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.25195979Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934}
212542024-03-06T21:06:35.523Z{"msg":"Crucible 0e266327-9ab4-4a99-8b0d-700398cd4b22 has session id: f7a7aa7e-67b6-4a41-9fdf-6e794122cbea","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.251993978Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212552024-03-06T21:06:35.523Z{"msg":"Upstairs opts: Upstairs UUID: 0e266327-9ab4-4a99-8b0d-700398cd4b22, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.252039865Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212562024-03-06T21:06:35.524Z{"msg":"Crucible stats registered with UUID: 0e266327-9ab4-4a99-8b0d-700398cd4b22","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.252071982Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212572024-03-06T21:06:35.524ZCrucible runtime is spawned
212582024-03-06T21:06:35.524Z{"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.252215752Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934}
212592024-03-06T21:06:35.524Z{"msg":"0e266327-9ab4-4a99-8b0d-700398cd4b22 active request set","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.252319436Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212602024-03-06T21:06:35.524Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.252386851Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"client":"0","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212612024-03-06T21:06:35.524Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.252457276Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"client":"0","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212622024-03-06T21:06:35.524Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.252494943Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"client":"1","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212632024-03-06T21:06:35.524Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.25253757Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"client":"1","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212642024-03-06T21:06:35.524Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.252582507Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"client":"2","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212652024-03-06T21:06:35.524Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.252625404Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"client":"2","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212662024-03-06T21:06:35.524Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.252671681Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212672024-03-06T21:06:35.524Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.252716038Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212682024-03-06T21:06:35.524Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.252766655Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212692024-03-06T21:06:35.524Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.252787993Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212702024-03-06T21:06:35.524Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.252815471Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212712024-03-06T21:06:35.524Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.25283378Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212722024-03-06T21:06:35.524Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.253398181Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"client":"2","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212732024-03-06T21:06:35.524Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.253444788Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"client":"2","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212742024-03-06T21:06:35.524Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.253476086Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"client":"0","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212752024-03-06T21:06:35.524Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.253523962Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"client":"0","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212762024-03-06T21:06:35.524Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.254035047Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"client":"2","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212772024-03-06T21:06:35.524Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.254201106Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"client":"0","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212782024-03-06T21:06:35.524Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.254346186Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"client":"1","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212792024-03-06T21:06:35.525Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.254387823Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"client":"1","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212802024-03-06T21:06:35.525Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.254504875Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"client":"2","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212812024-03-06T21:06:35.525Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.254537452Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"client":"2","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212822024-03-06T21:06:35.525Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.254552311Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212832024-03-06T21:06:35.525Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.254696032Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"client":"0","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212842024-03-06T21:06:35.525Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.254732729Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"client":"0","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212852024-03-06T21:06:35.525Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.254749758Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212862024-03-06T21:06:35.525Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.254930096Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"client":"1","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212872024-03-06T21:06:35.525Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.255358086Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"client":"1","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212882024-03-06T21:06:35.525Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.255393473Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"client":"1","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212892024-03-06T21:06:35.525Z{"msg":"[0]R flush_numbers[0..12]: [337, 337, 337, 327, 335, 328, 337, 337, 339, 334, 345, 339]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.255410012Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"":"downstairs","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212902024-03-06T21:06:35.525Z{"msg":"[0]R generation[0..12]: [194, 194, 194, 187, 192, 188, 194, 194, 194, 192, 197, 194]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.255431931Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"":"downstairs","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212912024-03-06T21:06:35.525Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.255448Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"":"downstairs","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212922024-03-06T21:06:35.525Z{"msg":"[1]R flush_numbers[0..12]: [337, 337, 337, 327, 335, 328, 337, 337, 339, 334, 345, 339]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
212932024-03-06T21:06:35.525Z,"time":"2024-03-06T21:06:34.255469028Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"":"downstairs","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212942024-03-06T21:06:35.525Z{"msg":"[1]R generation[0..12]: [194, 194, 194, 187, 192, 188, 194, 194, 194, 192, 197, 194]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.255483557Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"":"downstairs","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212952024-03-06T21:06:35.525Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.255498566Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"":"downstairs","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212962024-03-06T21:06:35.525Z{"msg":"[2]R flush_numbers[0..12]: [337, 337, 337, 327, 335, 328, 337, 337, 339, 334, 345, 339]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.255519375Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"":"downstairs","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212972024-03-06T21:06:35.525Z{"msg":"[2]R generation[0..12]: [194, 194, 194, 187, 192, 188, 194, 194, 194, 192, 197, 194]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.255534124Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"":"downstairs","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212982024-03-06T21:06:35.525Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.255563242Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"":"downstairs","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
212992024-03-06T21:06:35.525Z{"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.25558869Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"":"downstairs","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
213002024-03-06T21:06:35.525Z{"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.255622778Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"":"downstairs","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
213012024-03-06T21:06:35.525Z{"msg":"Next flush: 346","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.255644246Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"":"downstairs","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
213022024-03-06T21:06:35.530Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.255676284Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"":"downstairs","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
213032024-03-06T21:06:35.530Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.255709322Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
213042024-03-06T21:06:35.530Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.25572989Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"client":"0","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
213052024-03-06T21:06:35.530Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.255752009Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"client":"1","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
213062024-03-06T21:06:35.530Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.255774007Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"client":"2","":"downstairs","session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
213072024-03-06T21:06:35.530Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.255804665Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
213082024-03-06T21:06:35.530Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.255824554Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
213092024-03-06T21:06:35.530Zw-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
213102024-03-06T21:06:35.530Z{"msg":"0e266327-9ab4-4a99-8b0d-700398cd4b22 is now active with session: f7a7aa7e-67b6-4a41-9fdf-6e794122cbea","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.255860451Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
213112024-03-06T21:06:35.530Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.255894259Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934,"session_id":"f7a7aa7e-67b6-4a41-9fdf-6e794122cbea"}
213122024-03-06T21:06:35.530Z{"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-03-06T21:06:34.255914438Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934}
213132024-03-06T21:06:35.530ZLoading write count information from file "/tmp/test_repair_verify.data"
213142024-03-06T21:06:35.530ZRun Repair workload
213152024-03-06T21:06:35.530Z01/30 Write block 109 len 12288 data: 11 12 13
213162024-03-06T21:06:35.530Z02/30 Flush
213172024-03-06T21:06:35.530Z03/30 Write block 538 len 20480 data: 11 9 14 15 17
213182024-03-06T21:06:35.530Z04/30 Write block 176 len 40960 data: 17 14 13 19 19 23 21 20 21 20
213192024-03-06T21:06:35.530Z05/30 Write block 514 len 32768 data: 20 22 20 15 14 14 8 8
213202024-03-06T21:06:35.530Z06/30 Read block 402 len 4096
213212024-03-06T21:06:35.534Z07/30 Flush
213222024-03-06T21:06:35.534Z08/30 Write block 560 len 8192 data: 15 14
213232024-03-06T21:06:35.534Z09/30 Write block 200 len 28672 data: 17 14 12 12 12 12 13
213242024-03-06T21:06:35.534Z10/30 Read block 559 len 4096
213252024-03-06T21:06:35.537Z11/30 Read block 321 len 16384
213262024-03-06T21:06:35.537Z12/30 Write block 4 len 36864 data: 16 16 16 17 16 16 17 19 17
213272024-03-06T21:06:35.537Z13/30 Write block 16 len 12288 data: 19 21 19
213282024-03-06T21:06:35.540Z14/30 Read block 139 len 16384
213292024-03-06T21:06:35.540Z15/30 Read block 123 len 4096
213302024-03-06T21:06:35.540Z16/30 Write block 117 len 28672 data: 14 15 16 12 11 12 10
213312024-03-06T21:06:35.543Z17/30 Write block 185 len 40960 data: 21 18 16 16 23 24 17 19 16 14
213322024-03-06T21:06:35.543Z18/30 Flush
213332024-03-06T21:06:35.546Z19/30 Flush
213342024-03-06T21:06:35.546Z20/30 Write block 123 len 36864 data: 11 14 20 19 19 18 19 15 11
213352024-03-06T21:06:35.546Z21/30 Read block 64 len 32768
213362024-03-06T21:06:35.549Z22/30 Read block 309 len 16384
213372024-03-06T21:06:35.549Z23/30 Flush
213382024-03-06T21:06:35.552Z24/30 Write block 265 len 24576 data: 15 14 14 15 14 13
213392024-03-06T21:06:35.552Z25/30 Write block 193 len 8192 data: 17 15
213402024-03-06T21:06:35.552Z26/30 Read block 35 len 24576
213412024-03-06T21:06:35.556Z27/30 Write block 151 len 8192 data: 15 12
213422024-03-06T21:06:35.556Z28/30 Write block 137 len 36864 data: 10 8 9 9 7 7 8 8 8
213432024-03-06T21:06:35.556Z29/30 Read block 447 len 4096
213442024-03-06T21:06:35.556Z30/30 Write block 514 len 8192 data: 21 23
213452024-03-06T21:06:35.556Z----------------------------------------------------------------
213462024-03-06T21:06:35.556Z Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:28
213472024-03-06T21:06:35.556ZStates: Active Active Active
213482024-03-06T21:06:35.556ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
213492024-03-06T21:06:35.556Z 3 Acked 1002 Write 5 Done Sent Done false
213502024-03-06T21:06:35.557Z 4 Acked 1003 Write 10 Done Sent Done false
213512024-03-06T21:06:35.557Z 5 Acked 1004 Write 8 Done Sent Done false
213522024-03-06T21:06:35.557Z 6 Acked 1005 Read 1 Done Sent Done false
213532024-03-06T21:06:35.557Z 7 Acked 1006 Flush 0 Done Sent Done false
213542024-03-06T21:06:35.557Z 8 Acked 1007 Write 2 Done Sent Done false
213552024-03-06T21:06:35.557Z 9 Acked 1008 Write 7 Done Sent Done false
213562024-03-06T21:06:35.557Z 10 Acked 1009 Read 1 Done Sent Done false
213572024-03-06T21:06:35.557Z 11 Acked 1010 Read 4 Done Sent Done false
213582024-03-06T21:06:35.557Z 12 Acked 1011 Write 9 Done Sent Done false
213592024-03-06T21:06:35.557Z 13 Acked 1012 Write 3 Done Sent Done false
213602024-03-06T21:06:35.557Z 14 Acked 1013 Read 4 Done Sent Done false
213612024-03-06T21:06:35.557Z 15 Acked 1014 Read 1 Done Sent Done false
213622024-03-06T21:06:35.557Z 16 Acked 1015 Write 7 Done Sent Done false
213632024-03-06T21:06:35.557Z 17 Acked 1016 Write 10 Done Sent Done false
213642024-03-06T21:06:35.557Z 18 Acked 1017 Flush 0 Done Sent Done false
213652024-03-06T21:06:35.558Z 19 Acked 1018 Flush 0 Done Sent Done false
213662024-03-06T21:06:35.558Z 20 Acked 1019 Write 9 Done Sent Done false
213672024-03-06T21:06:35.558Z 21 Acked 1020 Read 8 Done Sent Done false
213682024-03-06T21:06:35.558Z 22 Acked 1021 Read 4 Done Sent Done false
213692024-03-06T21:06:35.558Z 23 Acked 1022 Flush 0 Done Sent Done false
213702024-03-06T21:06:35.558Z 24 Acked 1023 Write 6 Done Sent Done false
213712024-03-06T21:06:35.558Z 25 Acked 1024 Write 2 Done Sent Done false
213722024-03-06T21:06:35.558Z 26 Acked 1025 Read 6 Done Sent Done false
213732024-03-06T21:06:35.558Z 27 Acked 1026 Write 2 Done Sent Done false
213742024-03-06T21:06:35.558Z 28 Acked 1027 Write 9 Done Sent Done false
213752024-03-06T21:06:35.558Z 29 Acked 1028 Read 1 Done Sent Done false
213762024-03-06T21:06:35.558Z 30 Acked 1029 Write 2 Sent Sent Sent false
213772024-03-06T21:06:35.558Z STATES DS:0 DS:1 DS:2 TOTAL
213782024-03-06T21:06:35.558Z New 0 0 0 0
213792024-03-06T21:06:35.558Z Sent 1 28 1 30
213802024-03-06T21:06:35.558Z Done 27 0 27 54
213812024-03-06T21:06:35.559Z Skipped 0 0 0 0
213822024-03-06T21:06:35.559Z Error 0 0 0 0
213832024-03-06T21:06:35.572ZLast Flush: 1022 1001 1022
213842024-03-06T21:06:35.573ZDownstairs last five completed: 1001 1000
213852024-03-06T21:06:35.573ZUpstairs last five completed: 30 29 28 27 26
213862024-03-06T21:06:35.573Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-03-06T21:06:34.290517203Z","hostname":"w-01HRARQTS0TVE14DKZ437MF5TB","pid":1934}
213872024-03-06T21:06:35.573ZWrote out file "/tmp/test_repair_verify.data"
213882024-03-06T21:06:35.573Z
213892024-03-06T21:06:35.584ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
213902024-03-06T21:06:35.600ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
213912024-03-06T21:06:35.600Z 0 000-019 198 194 198 348 337 348 F F F <---
213922024-03-06T21:06:35.600Z 1 020-039 194 194 194 337 337 337 F F F
213932024-03-06T21:06:35.600Z 2 040-059 194 194 194 337 337 337 F F F
213942024-03-06T21:06:35.601Z 3 060-079 187 187 187 327 327 327 F F F
213952024-03-06T21:06:35.601Z 4 080-099 192 192 192 335 335 335 F F F
213962024-03-06T21:06:35.601Z 5 100-119 198 198 198 348 346 348 F F F <---
213972024-03-06T21:06:35.601Z 6 120-139 198 194 198 350 337 350 T F T <---
213982024-03-06T21:06:35.601Z 7 140-159 194 194 194 337 337 337 T F T <---
213992024-03-06T21:06:35.601Z 8 160-179 198 194 198 347 339 347 F F F <---
214002024-03-06T21:06:35.602Z 9 180-199 198 192 198 348 334 348 T F T <---
214012024-03-06T21:06:35.602Z 10 200-219 198 197 198 348 345 348 F F F <---
214022024-03-06T21:06:35.602Z 11 220-239 194 194 194 339 339 339 F F F
214032024-03-06T21:06:35.602Z 12 240-259 196 196 196 344 344 344 F F F
214042024-03-06T21:06:35.602Z 13 260-279 196 196 196 344 344 344 T F T <---
214052024-03-06T21:06:35.602Z 14 280-299 191 191 191 331 331 331 F F F
214062024-03-06T21:06:35.603Z 15 300-319 191 191 191 331 331 331 F F F
214072024-03-06T21:06:35.603Z 16 320-339 194 194 194 337 337 337 F F F
214082024-03-06T21:06:35.603Z 17 340-359 196 196 196 343 343 343 F F F
214092024-03-06T21:06:35.603Z 18 360-379 197 197 197 345 345 345 F F F
214102024-03-06T21:06:35.603Z 19 380-399 196 196 196 343 343 343 F F F
214112024-03-06T21:06:35.603Z 20 400-419 194 194 194 337 337 337 F F F
214122024-03-06T21:06:35.604Z 21 420-439 194 194 194 337 337 337 F F F
214132024-03-06T21:06:35.604Z 22 440-459 186 186 186 325 325 325 F F F
214142024-03-06T21:06:35.604Z 23 460-479 196 196 196 344 344 344 F F F
214152024-03-06T21:06:35.604Z 24 480-499 196 196 196 344 344 344 F F F
214162024-03-06T21:06:35.604Z 25 500-519 198 196 198 347 344 347 T F T <---
214172024-03-06T21:06:35.604Z 26 520-539 198 190 198 347 329 347 F F F <---
214182024-03-06T21:06:35.604Z 27 540-559 198 190 198 347 330 347 F F F <---
214192024-03-06T21:06:35.605Z 28 560-579 198 196 198 348 343 348 F F F <---
214202024-03-06T21:06:35.605Z 29 580-599 196 196 196 343 343 343 F F F
214212024-03-06T21:06:35.605ZMax gen: 198, Max flush: 350
214222024-03-06T21:06:35.605ZError: Difference in extent metadata found!
214232024-03-06T21:06:35.605ZA Difference in extent metadata is expected here
214242024-03-06T21:06:35.605ZOn loop 99
214252024-03-06T21:06:35.606Z
214262024-03-06T21:06:35.612ZVerifying data now
214272024-03-06T21:06:40.639ZNo change after verify
214282024-03-06T21:06:40.639ZLoop: 99 Downstairs dump after verify (and repair):
214292024-03-06T21:06:40.680ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
214302024-03-06T21:06:40.681Z 0 000-019 198 198 198 348 348 348 F F F
214312024-03-06T21:06:40.681Z 1 020-039 194 194 194 337 337 337 F F F
214322024-03-06T21:06:40.681Z 2 040-059 194 194 194 337 337 337 F F F
214332024-03-06T21:06:40.681Z 3 060-079 187 187 187 327 327 327 F F F
214342024-03-06T21:06:40.681Z 4 080-099 192 192 192 335 335 335 F F F
214352024-03-06T21:06:40.681Z 5 100-119 198 198 198 348 348 348 F F F
214362024-03-06T21:06:40.681Z 6 120-139 199 199 199 351 351 351 F F F
214372024-03-06T21:06:40.681Z 7 140-159 199 199 199 351 351 351 F F F
214382024-03-06T21:06:40.681Z 8 160-179 198 198 198 347 347 347 F F F
214392024-03-06T21:06:40.681Z 9 180-199 199 199 199 351 351 351 F F F
214402024-03-06T21:06:40.681Z 10 200-219 198 198 198 348 348 348 F F F
214412024-03-06T21:06:40.681Z 11 220-239 194 194 194 339 339 339 F F F
214422024-03-06T21:06:40.681Z 12 240-259 196 196 196 344 344 344 F F F
214432024-03-06T21:06:40.681Z 13 260-279 199 199 199 351 351 351 F F F
214442024-03-06T21:06:40.682Z 14 280-299 191 191 191 331 331 331 F F F
214452024-03-06T21:06:40.682Z 15 300-319 191 191 191 331 331 331 F F F
214462024-03-06T21:06:40.682Z 16 320-339 194 194 194 337 337 337 F F F
214472024-03-06T21:06:40.682Z 17 340-359 196 196 196 343 343 343 F F F
214482024-03-06T21:06:40.682Z 18 360-379 197 197 197 345 345 345 F F F
214492024-03-06T21:06:40.682Z 19 380-399 196 196 196 343 343 343 F F F
214502024-03-06T21:06:40.682Z 20 400-419 194 194 194 337 337 337 F F F
214512024-03-06T21:06:40.682Z 21 420-439 194 194 194 337 337 337 F F F
214522024-03-06T21:06:40.682Z 22 440-459 186 186 186 325 325 325 F F F
214532024-03-06T21:06:40.682Z 23 460-479 196 196 196 344 344 344 F F F
214542024-03-06T21:06:40.682Z 24 480-499 196 196 196 344 344 344 F F F
214552024-03-06T21:06:40.682Z 25 500-519 199 199 199 351 351 351 F F F
214562024-03-06T21:06:40.683Z 26 520-539 198 198 198 347 347 347 F F F
214572024-03-06T21:06:40.683Z 27 540-559 198 198 198 347 347 347 F F F
214582024-03-06T21:06:40.683Z 28 560-579 198 198 198 348 348 348 F F F
214592024-03-06T21:06:40.683Z 29 580-599 196 196 196 343 343 343 F F F
214602024-03-06T21:06:40.683ZMax gen: 199, Max flush: 351
214612024-03-06T21:06:40.683Z8:39 Test duration
214622024-03-06T21:06:40.683ZTest completed
214632024-03-06T21:06:40.683Z
214642024-03-06T21:06:40.683Zreal 8:39.967691385
214652024-03-06T21:06:40.684Zuser 1:43.742606678
214662024-03-06T21:06:40.684Zsys 21.095919961
214672024-03-06T21:06:40.684Ztrap 0.115404551
214682024-03-06T21:06:40.684Ztflt 0.055153145
214692024-03-06T21:06:40.684Zdflt 0.057112162
214702024-03-06T21:06:40.684Zkflt 0.003053277
214712024-03-06T21:06:40.684Zlock 10:11:03.494657279
214722024-03-06T21:06:40.684Zslp 41:33.102548508
214732024-03-06T21:06:40.684Zlat 8.543681654
214742024-03-06T21:06:40.684Zstop 0.052872200
214752024-03-06T21:06:40.684Zprocess exited: duration 527309 ms, exit code 0
 
214762024-03-06T21:06:40.694Zfound 4 output files
214772024-03-06T21:06:40.695Zuploading: /tmp/test_repair_ds8810.txt (46425 bytes)
214782024-03-06T21:06:42.150Zuploaded: /tmp/test_repair_ds8810.txt
214792024-03-06T21:06:42.151Zuploading: /tmp/test_repair_ds8820.txt (20699 bytes)
214802024-03-06T21:06:43.179Zuploaded: /tmp/test_repair_ds8820.txt
214812024-03-06T21:06:43.179Zuploading: /tmp/test_repair_ds8830.txt (47131 bytes)
214822024-03-06T21:06:48.323Zuploaded: /tmp/test_repair_ds8830.txt
214832024-03-06T21:06:48.323Zuploading: /tmp/test_repair_out.txt (79133 bytes)
214842024-03-06T21:06:49.356Zuploaded: /tmp/test_repair_out.txt