01HVPFE51RKEV73C41A2ZC6FP0: test-repair

BasicConfig {
    output_rules: [
        "/tmp/*.txt",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HVPFEFMJY1GQS9YZ2DX5G903

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-04-17T17:07:18.376Zjob dependencies complete; ready to run (waiting for 17 m 8 s)
22024-04-17T17:09:25.039Zjob assigned to worker 01HVPGDYTTAYG62ZSA4VY7JVXQ (queued for 2 m 6 s)
32024-04-17T17:09:32.247Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-04-17T17:09:47.802Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-04-17T17:09:47.802Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-04-17T17:09:55.332Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-04-17T17:09:55.332Zdownloading input: /input/build/work/bins/crutest.gz
82024-04-17T17:10:07.023Zdownloaded input: /input/build/work/bins/crutest.gz
92024-04-17T17:10:07.023Zdownloading input: /input/build/work/bins/dsc.gz
102024-04-17T17:10:14.416Zdownloaded input: /input/build/work/bins/dsc.gz
112024-04-17T17:10:14.430Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-04-17T17:10:14.439Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-04-17T17:10:14.440Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-04-17T17:10:14.485Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-04-17T17:10:14.485Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-04-17T17:10:14.508Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-04-17T17:10:14.508Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-04-17T17:10:14.551Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-04-17T17:10:14.552Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-04-17T17:10:14.591Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-04-17T17:10:14.591Zdownloading input: /input/build/work/scripts/test_up.sh
222024-04-17T17:10:14.688Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-04-17T17:10:14.689Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-04-17T17:10:14.715Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-04-17T17:10:14.733Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-04-17T17:10:33.502Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-04-17T17:10:33.502Zstarting task 0: "setup"
282024-04-17T17:10:33.584Z++ uname -s
292024-04-17T17:10:33.590Z+ kern=SunOS
302024-04-17T17:10:33.590Z+ case "$kern" in
312024-04-17T17:10:33.590Z+ groupadd -g 12345 build
322024-04-17T17:10:33.825Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-04-17T17:10:35.845Z+ zfs create -o mountpoint=/work rpool/work
342024-04-17T17:10:36.032Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-04-17T17:10:36.042Z+ home_fs=zfs
362024-04-17T17:10:36.043Z+ [[ zfs == autofs ]]
372024-04-17T17:10:36.043Z+ mkdir -p /home/build
382024-04-17T17:10:36.043Z+ chown build:build /home/build /work
392024-04-17T17:10:38.032Z+ chmod 0700 /home/build /work
402024-04-17T17:10:38.036Zprocess exited: duration 4584 ms, exit code 0
 
412024-04-17T17:10:38.046Zstarting task 1: "authentication"
422024-04-17T17:10:38.064Zprocess exited: duration 20 ms, exit code 0
 
432024-04-17T17:10:38.078Zstarting task 2: "build"
442024-04-17T17:10:38.085Z+ banner cores
452024-04-17T17:10:38.085Z
462024-04-17T17:10:38.085Z #### #### ##### ###### ####
472024-04-17T17:10:38.085Z # # # # # # # #
482024-04-17T17:10:38.085Z # # # # # ##### ####
492024-04-17T17:10:38.085Z # # # ##### # #
502024-04-17T17:10:38.085Z # # # # # # # # #
512024-04-17T17:10:38.085Z #### #### # # ###### ####
522024-04-17T17:10:38.085Z
532024-04-17T17:10:38.085Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-04-17T17:10:38.095Z+ echo 'input bins dir contains:'
552024-04-17T17:10:38.102Zinput bins dir contains:
562024-04-17T17:10:38.102Z+ ls -ltr /input/build/work/bins
572024-04-17T17:10:38.102Ztotal 659847
582024-04-17T17:10:38.102Z-rw-r--r-- 1 root root 98275455 Apr 17 17:09 crucible-downstairs.gz
592024-04-17T17:10:38.102Z-rw-r--r-- 1 root root 79853354 Apr 17 17:09 crucible-hammer.gz
602024-04-17T17:10:38.103Z-rw-r--r-- 1 root root 95763310 Apr 17 17:10 crutest.gz
612024-04-17T17:10:38.103Z-rw-r--r-- 1 root root 63730486 Apr 17 17:10 dsc.gz
622024-04-17T17:10:38.103Z+ echo 'input script dir contains:'
632024-04-17T17:10:38.103Z+ ls -ltr /input/build/work/scripts
642024-04-17T17:10:38.103Zinput script dir contains:
652024-04-17T17:10:38.103Ztotal 40
662024-04-17T17:10:38.103Z-rw-r--r-- 1 root root 3056 Apr 17 17:10 perf-downstairs-tick.d
672024-04-17T17:10:38.103Z-rw-r--r-- 1 root root 2102 Apr 17 17:10 test_ds.sh
682024-04-17T17:10:38.103Z-rw-r--r-- 1 root root 3746 Apr 17 17:10 test_live_repair.sh
692024-04-17T17:10:38.103Z-rw-r--r-- 1 root root 6416 Apr 17 17:10 test_repair.sh
702024-04-17T17:10:38.103Z-rw-r--r-- 1 root root 2969 Apr 17 17:10 test_replay.sh
712024-04-17T17:10:38.103Z-rw-r--r-- 1 root root 9465 Apr 17 17:10 test_up.sh
722024-04-17T17:10:38.103Z-rw-r--r-- 1 root root 2942 Apr 17 17:10 upstairs_info.d
732024-04-17T17:10:38.103Z+ pfexec chmod +x /input/build/work/scripts/perf-downstairs-tick.d /input/build/work/scripts/test_ds.sh /input/build/work/scripts/test_live_repair.sh /input/build/work/scripts/test_repair.sh /input/build/work/scripts/test_replay.sh /input/build/work/scripts/test_up.sh /input/build/work/scripts/upstairs_info.d
742024-04-17T17:10:38.111Z+ echo ' chmod input script dir contains:'
752024-04-17T17:10:38.111Z chmod input script dir contains:
762024-04-17T17:10:38.111Z+ ls -ltr /input/build/work/scripts
772024-04-17T17:10:38.111Ztotal 40
782024-04-17T17:10:38.111Z-rwxr-xr-x 1 root root 3056 Apr 17 17:10 perf-downstairs-tick.d
792024-04-17T17:10:38.111Z-rwxr-xr-x 1 root root 2102 Apr 17 17:10 test_ds.sh
802024-04-17T17:10:38.111Z-rwxr-xr-x 1 root root 3746 Apr 17 17:10 test_live_repair.sh
812024-04-17T17:10:38.111Z-rwxr-xr-x 1 root root 6416 Apr 17 17:10 test_repair.sh
822024-04-17T17:10:38.111Z-rwxr-xr-x 1 root root 2969 Apr 17 17:10 test_replay.sh
832024-04-17T17:10:38.111Z-rwxr-xr-x 1 root root 9465 Apr 17 17:10 test_up.sh
842024-04-17T17:10:38.111Z-rwxr-xr-x 1 root root 2942 Apr 17 17:10 upstairs_info.d
852024-04-17T17:10:38.111Z+ banner unpack
862024-04-17T17:10:38.111Z
872024-04-17T17:10:38.111Z # # # # ##### ## #### # #
882024-04-17T17:10:38.111Z # # ## # # # # # # # # #
892024-04-17T17:10:38.111Z # # # # # # # # # # ####
902024-04-17T17:10:38.111Z # # # # # ##### ###### # # #
912024-04-17T17:10:38.112Z # # # ## # # # # # # #
922024-04-17T17:10:38.112Z #### # # # # # #### # #
932024-04-17T17:10:38.112Z
942024-04-17T17:10:38.112Z+ mkdir -p /var/tmp/bins
952024-04-17T17:10:38.112Z+ for t in "$input/bins/"*.gz
962024-04-17T17:10:38.112Z++ basename /input/build/work/bins/crucible-downstairs.gz
972024-04-17T17:10:38.112Z+ b=crucible-downstairs.gz
982024-04-17T17:10:38.112Z+ b=crucible-downstairs
992024-04-17T17:10:38.112Z+ gunzip
1002024-04-17T17:10:40.165Z+ chmod +x /var/tmp/bins/crucible-downstairs
1012024-04-17T17:10:40.172Z+ for t in "$input/bins/"*.gz
1022024-04-17T17:10:40.172Z++ basename /input/build/work/bins/crucible-hammer.gz
1032024-04-17T17:10:40.172Z+ b=crucible-hammer.gz
1042024-04-17T17:10:40.172Z+ b=crucible-hammer
1052024-04-17T17:10:40.172Z+ gunzip
1062024-04-17T17:10:41.868Z+ chmod +x /var/tmp/bins/crucible-hammer
1072024-04-17T17:10:41.875Z+ for t in "$input/bins/"*.gz
1082024-04-17T17:10:41.875Z++ basename /input/build/work/bins/crutest.gz
1092024-04-17T17:10:41.875Z+ b=crutest.gz
1102024-04-17T17:10:41.875Z+ b=crutest
1112024-04-17T17:10:41.875Z+ gunzip
1122024-04-17T17:10:43.880Z+ chmod +x /var/tmp/bins/crutest
1132024-04-17T17:10:43.887Z+ for t in "$input/bins/"*.gz
1142024-04-17T17:10:43.887Z++ basename /input/build/work/bins/dsc.gz
1152024-04-17T17:10:43.887Z+ b=dsc.gz
1162024-04-17T17:10:43.887Z+ b=dsc
1172024-04-17T17:10:43.887Z+ gunzip
1182024-04-17T17:10:45.287Z+ chmod +x /var/tmp/bins/dsc
1192024-04-17T17:10:45.294Z+ export BINDIR=/var/tmp/bins
1202024-04-17T17:10:45.294Z+ BINDIR=/var/tmp/bins
1212024-04-17T17:10:45.294Z+ banner repair
1222024-04-17T17:10:45.294Z
1232024-04-17T17:10:45.294Z ##### ###### ##### ## # #####
1242024-04-17T17:10:45.294Z # # # # # # # # # #
1252024-04-17T17:10:45.294Z # # ##### # # # # # # #
1262024-04-17T17:10:45.294Z ##### # ##### ###### # #####
1272024-04-17T17:10:45.294Z # # # # # # # # #
1282024-04-17T17:10:45.294Z # # ###### # # # # # #
1292024-04-17T17:10:45.294Z
1302024-04-17T17:10:45.294Z+ ptime -m bash /input/build/work/scripts/test_repair.sh -N
1312024-04-17T17:10:45.561ZTurn off color for downstairs dump
1322024-04-17T17:10:45.579ZCreating region directory at: /var/tmp/test_repair
1332024-04-17T17:10:45.583ZCreating dsc directory at: /tmp/dsc
1342024-04-17T17:10:45.766ZDownstairs region 0 created at /var/tmp/test_repair/8810 in 0.18397015
1352024-04-17T17:10:45.813ZDownstairs region 1 created at /var/tmp/test_repair/8820 in 0.04749287
1362024-04-17T17:10:45.868ZDownstairs region 2 created at /var/tmp/test_repair/8830 in 0.045929052
1372024-04-17T17:10:45.868ZRegion set with 3 regions was created
1382024-04-17T17:10:45.874Z/var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1170 1171 1172
1392024-04-17T17:10:45.874ZDump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
1402024-04-17T17:10:45.901Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.507158065Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174}
1412024-04-17T17:10:45.901Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.507678717Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174}
1422024-04-17T17:10:45.901Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.507698018Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174}
1432024-04-17T17:10:45.905Z{"msg":"Crucible c707303b-dcfd-4e51-9286-762b4101f899 has session id: 3352538b-6c1f-4654-b19b-bdf65391bdbf","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.507712399Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1442024-04-17T17:10:45.906Z{"msg":"Upstairs opts: Upstairs UUID: c707303b-dcfd-4e51-9286-762b4101f899, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.507765502Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1452024-04-17T17:10:45.906Z{"msg":"Crucible stats registered with UUID: c707303b-dcfd-4e51-9286-762b4101f899","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.507782713Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1462024-04-17T17:10:45.906ZCrucible runtime is spawned
1472024-04-17T17:10:45.906Z{"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.50872037Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174}
1482024-04-17T17:10:45.906Z{"msg":"c707303b-dcfd-4e51-9286-762b4101f899 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.508951354Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1492024-04-17T17:10:45.906Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.508988406Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"client":"0","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1502024-04-17T17:10:45.906Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.509066351Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"client":"0","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1512024-04-17T17:10:45.906Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.509111844Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"client":"1","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1522024-04-17T17:10:45.906Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.509154506Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"client":"1","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1532024-04-17T17:10:45.906Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.509185628Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"client":"2","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1542024-04-17T17:10:45.906Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.50921198Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"client":"2","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1552024-04-17T17:10:45.906Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.509237821Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"":"io task","client":"0","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1562024-04-17T17:10:45.906Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.509293125Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"":"io task","client":"1","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1572024-04-17T17:10:45.906Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.509317126Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"":"io task","client":"2","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1582024-04-17T17:10:45.906Z{"msg":"ds_connection connected from Ok(127.0.0.1:36847)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.509429843Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"":"io task","client":"2","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1592024-04-17T17:10:45.906Z{"msg":"ds_connection connected from Ok(127.0.0.1:41430)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.509479176Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"":"io task","client":"0","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1602024-04-17T17:10:45.906Z{"msg":"ds_connection connected from Ok(127.0.0.1:45089)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.509513718Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"":"io task","client":"1","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1612024-04-17T17:10:45.912Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.511575243Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"client":"2","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1622024-04-17T17:10:45.912Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.511615195Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"client":"2","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1632024-04-17T17:10:45.912Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.511641777Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"client":"1","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1642024-04-17T17:10:45.913Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.511710691Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"client":"1","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1652024-04-17T17:10:45.913Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.511748513Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"client":"0","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1662024-04-17T17:10:45.913Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.511782435Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"client":"0","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1672024-04-17T17:10:45.913Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.512544852Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"client":"2","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1682024-04-17T17:10:45.913Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.512594205Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"client":"1","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1692024-04-17T17:10:45.913Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.5126816Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"client":"0","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1702024-04-17T17:10:45.913Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.513202132Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"client":"2","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1712024-04-17T17:10:45.913Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.513239394Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"client":"2","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1722024-04-17T17:10:45.913Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.5133334Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1732024-04-17T17:10:45.913Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.513360541Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"client":"0","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1742024-04-17T17:10:45.913Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.513380222Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"client":"0","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1752024-04-17T17:10:45.913Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.513402614Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1762024-04-17T17:10:45.913Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.513416114Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"client":"1","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1772024-04-17T17:10:45.913Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.513436456Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"client":"1","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1782024-04-17T17:10:45.913Z{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.513451497Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"":"downstairs","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1792024-04-17T17:10:45.913Z{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.513472168Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"":"downstairs","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1802024-04-17T17:10:45.913Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.513486839Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"":"downstairs","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1812024-04-17T17:10:45.913Z{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.51350766Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"":"downstairs","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1822024-04-17T17:10:45.913Z{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.513522401Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"":"downstairs","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1832024-04-17T17:10:45.913Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.513543992Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"":"downstairs","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1842024-04-17T17:10:45.913Z{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.513557803Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"":"downstairs","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1852024-04-17T17:10:45.913Z{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.513577914Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"":"downstairs","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1862024-04-17T17:10:45.913Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.513592255Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"":"downstairs","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1872024-04-17T17:10:45.913Z{"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.513611406Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"":"downstairs","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1882024-04-17T17:10:45.914Z{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.513625447Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"":"downstairs","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1892024-04-17T17:10:45.914Z{"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.513645188Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"":"downstairs","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1902024-04-17T17:10:45.914Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.513659479Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"":"downstairs","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1912024-04-17T17:10:45.914Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.513678981Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1922024-04-17T17:10:45.914Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.513691151Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"client":"0"Wait for a query_work_queue command to finish before sending IO
1932024-04-17T17:10:45.914Z,"":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1942024-04-17T17:10:45.914Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.513718213Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"client":"1","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1952024-04-17T17:10:45.914Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.513731844Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"client":"2","":"downstairs","session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1962024-04-17T17:10:45.914Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.513751765Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1972024-04-17T17:10:45.914Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.513764096Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1982024-04-17T17:10:45.914Z{"msg":"c707303b-dcfd-4e51-9286-762b4101f899 is now active with session: 3352538b-6c1f-4654-b19b-bdf65391bdbf","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.513784157Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
1992024-04-17T17:10:45.914Z{"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.513796408Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174}
2002024-04-17T17:10:45.914Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:44.513813939Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174,"session_id":"3352538b-6c1f-4654-b19b-bdf65391bdbf"}
2012024-04-17T17:10:45.914ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2022024-04-17T17:10:45.914ZFill test
2032024-04-17T17:10:45.963ZRead and Verify all blocks (0..600 range:false)
2042024-04-17T17:10:45.990ZWrote out file "/tmp/test_repair_verify.data"
2052024-04-17T17:10:45.990ZCLIENT: Tests done. All submitted work has been ACK'd
2062024-04-17T17:10:45.994Z----------------------------------------------------------------
2072024-04-17T17:10:45.994Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3
2082024-04-17T17:10:45.994ZStates: Active Active Active
2092024-04-17T17:10:45.994ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
2102024-04-17T17:10:45.994Z 5 Acked 1004 Read 256 Done Done Done false
2112024-04-17T17:10:45.994Z 6 Acked 1005 Read 88 Done Done Done false
2122024-04-17T17:10:45.994Z 7 Acked 1006 Read 256 Done Done Done false
2132024-04-17T17:10:45.994Z STATES DS:0 DS:1 DS:2 TOTAL
2142024-04-17T17:10:45.994Z New 0 0 0 0
2152024-04-17T17:10:45.994Z Sent 0 0 0 0
2162024-04-17T17:10:45.994Z Done 3 3 3 9
2172024-04-17T17:10:45.994Z Skipped 0 0 0 0
2182024-04-17T17:10:45.994Z Error 0 0 0 0
2192024-04-17T17:10:45.994ZLast Flush: 1003 1003 1003
2202024-04-17T17:10:45.994ZDownstairs last five completed: 1003 1002 1001 1000
2212024-04-17T17:10:45.994ZUpstairs last five completed: 7 6 5 4 3
2222024-04-17T17:10:45.994ZCLIENT: Up:0 ds:3 act:3
2232024-04-17T17:10:49.992Z----------------------------------------------------------------
2242024-04-17T17:10:49.992Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0
2252024-04-17T17:10:49.993ZDownstairs last five completed: 1007 1006 1005 1004 1003
2262024-04-17T17:10:49.993ZUpstairs last five completed: 8 7 6 5 4
2272024-04-17T17:10:49.993ZCLIENT: Up:0 ds:0 act:3
2282024-04-17T17:10:49.993ZCLIENT: All crucible jobs finished, exiting program
2292024-04-17T17:10:49.999Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:10:48.598456088Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1174}
2302024-04-17T17:10:50.006Z
2312024-04-17T17:10:50.006ZBegin loop 1
2322024-04-17T17:10:50.006ZDownstairs to restart: 1
2332024-04-17T17:10:50.018Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.623612203Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176}
2342024-04-17T17:10:50.018Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.623916621Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176}
2352024-04-17T17:10:50.026Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.623956694Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176}
2362024-04-17T17:10:50.026Z{"msg":"Crucible eb7f20b2-5dcd-4fea-9d6e-cd999d2204c9 has session id: f0bb5966-84f6-4834-8d20-5d8a3ce28117","v":0Crucible runtime is spawned
2372024-04-17T17:10:50.026Z,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.62405762Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2382024-04-17T17:10:50.026Z{"msg":"Upstairs opts: Upstairs UUID: eb7f20b2-5dcd-4fea-9d6e-cd999d2204c9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.624136195Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2392024-04-17T17:10:50.026Z{"msg":"Crucible stats registered with UUID: eb7f20b2-5dcd-4fea-9d6e-cd999d2204c9","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.624174277Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2402024-04-17T17:10:50.026Z{"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.624195328Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176}
2412024-04-17T17:10:50.026Z{"msg":"eb7f20b2-5dcd-4fea-9d6e-cd999d2204c9 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.62423203Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2422024-04-17T17:10:50.026Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.624260742Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"client":"0","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2432024-04-17T17:10:50.026Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.624299895Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"client":"0","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2442024-04-17T17:10:50.027Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.624369209Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"client":"1","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2452024-04-17T17:10:50.027Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.624414381Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"client":"1","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2462024-04-17T17:10:50.027Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.624431833Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"client":"2","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2472024-04-17T17:10:50.027Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.624468105Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"client":"2","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2482024-04-17T17:10:50.027Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.624493156Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"":"io task","client":"2","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2492024-04-17T17:10:50.027Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.624534739Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"":"io task","client":"0","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2502024-04-17T17:10:50.027Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.62456169Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"":"io task","client":"1","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2512024-04-17T17:10:50.027Z{"msg":"ds_connection connected from Ok(127.0.0.1:58618)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.624598273Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"":"io task","client":"0","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2522024-04-17T17:10:50.027Z{"msg":"ds_connection connected from Ok(127.0.0.1:40495)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.624637395Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"":"io task","client":"2","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2532024-04-17T17:10:50.027Z{"msg":"ds_connection connected from Ok(127.0.0.1:35351)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.624665037Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"":"io task","client":"1","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2542024-04-17T17:10:50.027Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.625409462Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"client":"2","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2552024-04-17T17:10:50.027Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.625456885Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"client":"2","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2562024-04-17T17:10:50.027Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.625503258Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"client":"0","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2572024-04-17T17:10:50.027Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.625524739Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"client":"0","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2582024-04-17T17:10:50.027Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.625559531Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"client":"1","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2592024-04-17T17:10:50.027Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.625598283Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"client":"1","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2602024-04-17T17:10:50.027Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.626061001Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"client":"2","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2612024-04-17T17:10:50.027Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.626120075Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"client":"1","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2622024-04-17T17:10:50.027Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.626298506Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"client":"0","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2632024-04-17T17:10:50.027Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.626677349Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"client":"2","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2642024-04-17T17:10:50.027Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.626719851Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"client":"2","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2652024-04-17T17:10:50.027Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.626779135Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2662024-04-17T17:10:50.027Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.626801006Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"client":"1","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2672024-04-17T17:10:50.028Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.626828998Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"client":"1","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2682024-04-17T17:10:50.028Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.626843599Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2692024-04-17T17:10:50.028Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.62686259Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"client":"0","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2702024-04-17T17:10:50.028Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.626876081Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"client":"0","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2712024-04-17T17:10:50.028Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.626889991Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"":"downstairs","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2722024-04-17T17:10:50.028Z{"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.626910943Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"":"downstairs","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2732024-04-17T17:10:50.028Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.626925364Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"":"downstairs","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2742024-04-17T17:10:50.028Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.626945365Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"":"downstairs","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2752024-04-17T17:10:50.028Z{"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.626959636Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"":"downstairs","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2762024-04-17T17:10:50.028Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
2772024-04-17T17:10:50.028Z2024-04-17T17:10:48.626996018Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"":"downstairs","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2782024-04-17T17:10:50.028Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.62702254Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"":"downstairs","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2792024-04-17T17:10:50.028Z{"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.627051481Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"":"downstairs","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2802024-04-17T17:10:50.028Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.627076493Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"":"downstairs","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2812024-04-17T17:10:50.028Z{"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.627102964Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"":"downstairs","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2822024-04-17T17:10:50.028Z{"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.627126076Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"":"downstairs","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2832024-04-17T17:10:50.028Z{"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.627152538Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"":"downstairs","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2842024-04-17T17:10:50.028Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.627174839Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"":"downstairs","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2852024-04-17T17:10:50.028Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.62720209Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2862024-04-17T17:10:50.028Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.627223002Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"client":"0","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2872024-04-17T17:10:50.028Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.627250844Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"client":"1","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2882024-04-17T17:10:50.028ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2892024-04-17T17:10:50.028Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.627278125Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"client":"2","":"downstairs","session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2902024-04-17T17:10:50.028Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.627307637Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2912024-04-17T17:10:50.029Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.627329228Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2922024-04-17T17:10:50.029Z{"msg":"eb7f20b2-5dcd-4fea-9d6e-cd999d2204c9 is now active with session: f0bb5966-84f6-4834-8d20-5d8a3ce28117","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.62735669Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2932024-04-17T17:10:50.029Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.627378121Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176,"session_id":"f0bb5966-84f6-4834-8d20-5d8a3ce28117"}
2942024-04-17T17:10:50.029Z{"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:48.627407923Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176}
2952024-04-17T17:10:50.029ZLoading write count information from file "/tmp/test_repair_verify.data"
2962024-04-17T17:10:50.029ZRun Repair workload
2972024-04-17T17:10:50.034Z01/30 Write block 347 len 24576 data: 2 2 2 2 2 2
2982024-04-17T17:10:50.034Z02/30 Flush
2992024-04-17T17:10:50.034Z03/30 Write block 500 len 12288 data: 2 2 2
3002024-04-17T17:10:50.034Z04/30 Read block 396 len 32768
3012024-04-17T17:10:50.034Z05/30 Write block 10 len 4096 data: 2
3022024-04-17T17:10:50.034Z06/30 Read block 408 len 20480
3032024-04-17T17:10:50.034Z07/30 Read block 179 len 32768
3042024-04-17T17:10:50.034Z08/30 Read block 500 len 24576
3052024-04-17T17:10:50.034Z09/30 Write block 143 len 16384 data: 2 2 2 2
3062024-04-17T17:10:50.034Z10/30 Read block 447 len 32768
3072024-04-17T17:10:50.034Z11/30 Write block 301 len 4096 data: 2
3082024-04-17T17:10:50.034Z12/30 Write block 319 len 8192 data: 2 2
3092024-04-17T17:10:50.039Z13/30 Write block 574 len 32768 data: 2 2 2 2 2 2 2 2
3102024-04-17T17:10:50.039Z14/30 Write block 335 len 4096 data: 2
3112024-04-17T17:10:50.039Z15/30 Read block 275 len 28672
3122024-04-17T17:10:50.039Z16/30 Write block 132 len 20480 data: 2 2 2 2 2
3132024-04-17T17:10:50.039Z17/30 Read block 328 len 16384
3142024-04-17T17:10:50.039Z18/30 Write block 252 len 20480 data: 2 2 2 2 2
3152024-04-17T17:10:50.045Z19/30 Read block 494 len 8192
3162024-04-17T17:10:50.045Z20/30 Write block 378 len 20480 data: 2 2 2 2 2
3172024-04-17T17:10:50.045Z21/30 Flush
3182024-04-17T17:10:50.049Z22/30 Write block 479 len 32768 data: 2 2 2 2 2 2 2 2
3192024-04-17T17:10:50.049Z23/30 Flush
3202024-04-17T17:10:50.049Z24/30 Flush
3212024-04-17T17:10:50.070Z25/30 Write block 536 len 32768 data: 2 2 2 2 2 2 2 2
3222024-04-17T17:10:50.070Z26/30 Read block 490 len 12288
3232024-04-17T17:10:50.070Z27/30 Read block 467 len 16384
3242024-04-17T17:10:50.070Z28/30 Read block 548 len 20480
3252024-04-17T17:10:50.071Z29/30 Read block 516 len 32768
3262024-04-17T17:10:50.071Z30/30 Read block 89 len 36864
3272024-04-17T17:10:50.071Z----------------------------------------------------------------
3282024-04-17T17:10:50.071Z Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:28
3292024-04-17T17:10:50.071ZStates: Active Active Active
3302024-04-17T17:10:50.071ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
3312024-04-17T17:10:50.071Z 3 Acked 1002 Write 3 Done Done Done false
3322024-04-17T17:10:50.071Z 4 Acked 1003 Read 8 Done Done Done false
3332024-04-17T17:10:50.071Z 5 Acked 1004 Write 1 Done Done Done false
3342024-04-17T17:10:50.071Z 6 Acked 1005 Read 5 Done Sent Done false
3352024-04-17T17:10:50.072Z 7 Acked 1006 Read 8 Done Sent Done false
3362024-04-17T17:10:50.072Z 8 Acked 1007 Read 6 Done Sent Done false
3372024-04-17T17:10:50.072Z 9 Acked 1008 Write 4 Done Sent Done false
3382024-04-17T17:10:50.072Z 10 Acked 1009 Read 8 Done Sent Done false
3392024-04-17T17:10:50.072Z 11 Acked 1010 Write 1 Done Sent Done false
3402024-04-17T17:10:50.072Z 12 Acked 1011 Write 2 Done Sent Done false
3412024-04-17T17:10:50.072Z 13 Acked 1012 Write 8 Done Sent Done false
3422024-04-17T17:10:50.072Z 14 Acked 1013 Write 1 Done Sent Done false
3432024-04-17T17:10:50.073Z 15 Acked 1014 Read 7 Done Sent Done false
3442024-04-17T17:10:50.073Z 16 Acked 1015 Write 5 Done Sent Done false
3452024-04-17T17:10:50.073Z 17 Acked 1016 Read 4 Done Sent Done false
3462024-04-17T17:10:50.073Z 18 Acked 1017 Write 5 Done Sent Done false
3472024-04-17T17:10:50.073Z 19 Acked 1018 Read 2 Done Sent Done false
3482024-04-17T17:10:50.073Z 20 Acked 1019 Write 5 Done Sent Done false
3492024-04-17T17:10:50.073Z 21 Acked 1020 Flush 0 Done Sent Done false
3502024-04-17T17:10:50.073Z 22 Acked 1021 Write 8 Done Sent Done false
3512024-04-17T17:10:50.073Z 23 Acked 1022 Flush 0 Done Sent Done false
3522024-04-17T17:10:50.073Z 24 Acked 1023 Flush 0 Done Sent Done false
3532024-04-17T17:10:50.073Z 25 Acked 1024 Write 8 Done Sent Done false
3542024-04-17T17:10:50.073Z 26 Acked 1025 Read 3 Done Sent Done false
3552024-04-17T17:10:50.073Z 27 Acked 1026 Read 4 Done Sent Done false
3562024-04-17T17:10:50.073Z 28 Acked 1027 Read 5 Done Sent Done false
3572024-04-17T17:10:50.073Z 29 Acked 1028 Read 8 Done Sent Done false
3582024-04-17T17:10:50.073Z 30 Acked 1029 Read 9 Sent Sent Done false
3592024-04-17T17:10:50.074Z STATES DS:0 DS:1 DS:2 TOTAL
3602024-04-17T17:10:50.074Z New 0 0 0 0
3612024-04-17T17:10:50.074Z Sent 1 25 0 26
3622024-04-17T17:10:50.074Z Done 27 3 28 58
3632024-04-17T17:10:50.074Z Skipped 0 0 0 0
3642024-04-17T17:10:50.074Z Error 0 0 0 0
3652024-04-17T17:10:50.074ZLast Flush: 1023 1001 1023
3662024-04-17T17:10:50.074ZDownstairs last five completed: 1001 1000
3672024-04-17T17:10:50.074ZUpstairs last five completed: 30 29 28 27 26
3682024-04-17T17:10:50.074Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:10:48.660081373Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1176}
3692024-04-17T17:10:50.074ZWrote out file "/tmp/test_repair_verify.data"
3702024-04-17T17:10:50.078Z
3712024-04-17T17:10:50.078ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
3722024-04-17T17:10:50.078ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3732024-04-17T17:10:50.079Z 0 000-019 2 1 2 3 1 3 F T F <---
3742024-04-17T17:10:50.079Z 1 020-039 1 1 1 1 1 1 F F F
3752024-04-17T17:10:50.079Z 2 040-059 1 1 1 1 1 1 F F F
3762024-04-17T17:10:50.079Z 3 060-079 1 1 1 1 1 1 F F F
3772024-04-17T17:10:50.079Z 4 080-099 1 1 1 1 1 1 F F F
3782024-04-17T17:10:50.079Z 5 100-119 1 1 1 1 1 1 F F F
3792024-04-17T17:10:50.079Z 6 120-139 2 1 2 3 1 3 F F F <---
3802024-04-17T17:10:50.079Z 7 140-159 2 1 2 3 1 3 F F F <---
3812024-04-17T17:10:50.079Z 8 160-179 1 1 1 1 1 1 F F F
3822024-04-17T17:10:50.079Z 9 180-199 1 1 1 1 1 1 F F F
3832024-04-17T17:10:50.079Z 10 200-219 1 1 1 1 1 1 F F F
3842024-04-17T17:10:50.079Z 11 220-239 1 1 1 1 1 1 F F F
3852024-04-17T17:10:50.079Z 12 240-259 2 1 2 3 1 3 F F F <---
3862024-04-17T17:10:50.079Z 13 260-279 1 1 1 1 1 1 F F F
3872024-04-17T17:10:50.079Z 14 280-299 1 1 1 1 1 1 F F F
3882024-04-17T17:10:50.079Z 15 300-319 2 1 2 3 1 3 F F F <---
3892024-04-17T17:10:50.079Z 16 320-339 2 1 2 3 1 3 F F F <---
3902024-04-17T17:10:50.079Z 17 340-359 2 2 2 2 2 2 F F F
3912024-04-17T17:10:50.079Z 18 360-379 2 1 2 3 1 3 F F F <---
3922024-04-17T17:10:50.079Z 19 380-399 2 1 2 3 1 3 F F F <---
3932024-04-17T17:10:50.079Z 20 400-419 1 1 1 1 1 1 F F F
3942024-04-17T17:10:50.079Z 21 420-439 1 1 1 1 1 1 F F F
3952024-04-17T17:10:50.079Z 22 440-459 1 1 1 1 1 1 F F F
3962024-04-17T17:10:50.080Z 23 460-479 2 1 2 4 1 4 F F F <---
3972024-04-17T17:10:50.080Z 24 480-499 2 1 2 4 1 4 F F F <---
3982024-04-17T17:10:50.080Z 25 500-519 2 1 2 3 1 3 F T F <---
3992024-04-17T17:10:50.080Z 26 520-539 1 1 1 1 1 1 T F T <---
4002024-04-17T17:10:50.080Z 27 540-559 1 1 1 1 1 1 T F T <---
4012024-04-17T17:10:50.080Z 28 560-579 2 1 2 3 1 3 F F F <---
4022024-04-17T17:10:50.080Z 29 580-599 2 1 2 3 1 3 F F F <---
4032024-04-17T17:10:50.080ZMax gen: 2, Max flush: 4
4042024-04-17T17:10:50.080ZError: Difference in extent metadata found!
4052024-04-17T17:10:50.084ZA Difference in extent metadata is expected here
4062024-04-17T17:10:50.084ZOn loop 1
4072024-04-17T17:10:50.084Z
4082024-04-17T17:10:50.084ZVerifying data now
4092024-04-17T17:10:54.893ZNo change after verify
4102024-04-17T17:10:54.893ZLoop: 1 Downstairs dump after verify (and repair):
4112024-04-17T17:10:54.907ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4122024-04-17T17:10:54.907Z 0 000-019 2 2 2 3 3 3 F F F
4132024-04-17T17:10:54.907Z 1 020-039 1 1 1 1 1 1 F F F
4142024-04-17T17:10:54.907Z 2 040-059 1 1 1 1 1 1 F F F
4152024-04-17T17:10:54.907Z 3 060-079 1 1 1 1 1 1 F F F
4162024-04-17T17:10:54.907Z 4 080-099 1 1 1 1 1 1 F F F
4172024-04-17T17:10:54.907Z 5 100-119 1 1 1 1 1 1 F F F
4182024-04-17T17:10:54.907Z 6 120-139 2 2 2 3 3 3 F F F
4192024-04-17T17:10:54.907Z 7 140-159 2 2 2 3 3 3 F F F
4202024-04-17T17:10:54.907Z 8 160-179 1 1 1 1 1 1 F F F
4212024-04-17T17:10:54.912Z 9 180-199 1 1 1 1 1 1 F F F
4222024-04-17T17:10:54.912Z 10 200-219 1 1 1 1 1 1 F F F
4232024-04-17T17:10:54.912Z 11 220-239 1 1 1 1 1 1 F F F
4242024-04-17T17:10:54.912Z 12 240-259 2 2 2 3 3 3 F F F
4252024-04-17T17:10:54.912Z 13 260-279 1 1 1 1 1 1 F F F
4262024-04-17T17:10:54.912Z 14 280-299 1 1 1 1 1 1 F F F
4272024-04-17T17:10:54.912Z 15 300-319 2 2 2 3 3 3 F F F
4282024-04-17T17:10:54.912Z 16 320-339 2 2 2 3 3 3 F F F
4292024-04-17T17:10:54.912Z 17 340-359 2 2 2 2 2 2 F F F
4302024-04-17T17:10:54.912Z 18 360-379 2 2 2 3 3 3 F F F
4312024-04-17T17:10:54.912Z 19 380-399 2 2 2 3 3 3 F F F
4322024-04-17T17:10:54.912Z 20 400-419 1 1 1 1 1 1 F F F
4332024-04-17T17:10:54.912Z 21 420-439 1 1 1 1 1 1 F F F
4342024-04-17T17:10:54.912Z 22 440-459 1 1 1 1 1 1 F F F
4352024-04-17T17:10:54.912Z 23 460-479 2 2 2 4 4 4 F F F
4362024-04-17T17:10:54.912Z 24 480-499 2 2 2 4 4 4 F F F
4372024-04-17T17:10:54.912Z 25 500-519 2 2 2 3 3 3 F F F
4382024-04-17T17:10:54.912Z 26 520-539 3 3 3 5 5 5 F F F
4392024-04-17T17:10:54.912Z 27 540-559 3 3 3 5 5 5 F F F
4402024-04-17T17:10:54.912Z 28 560-579 2 2 2 3 3 3 F F F
4412024-04-17T17:10:54.912Z 29 580-599 2 2 2 3 3 3 F F F
4422024-04-17T17:10:54.913ZMax gen: 3, Max flush: 5
4432024-04-17T17:10:54.913Z
4442024-04-17T17:10:54.913ZBegin loop 2
4452024-04-17T17:10:54.913ZDownstairs to restart: 2
4462024-04-17T17:10:54.929Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.535551152Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184}
4472024-04-17T17:10:54.929Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.535817258Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184}
4482024-04-17T17:10:54.936Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.535842999Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184}
4492024-04-17T17:10:54.936Z{"msg":"Crucible eefa0851-8060-42d6-9d43-96e679fe92a2 has session id: 7b2e5b54-ea92-4843-ad71-f78fe4126358","v":0,"name":"crucible","level":30,Crucible runtime is spawned
4502024-04-17T17:10:54.936Z"time":"2024-04-17T17:10:53.535885772Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4512024-04-17T17:10:54.936Z{"msg":"Upstairs opts: Upstairs UUID: eefa0851-8060-42d6-9d43-96e679fe92a2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.535963186Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4522024-04-17T17:10:54.936Z{"msg":"Crucible stats registered with UUID: eefa0851-8060-42d6-9d43-96e679fe92a2","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.535998779Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4532024-04-17T17:10:54.936Z{"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.5360234Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184}
4542024-04-17T17:10:54.936Z{"msg":"eefa0851-8060-42d6-9d43-96e679fe92a2 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.536051662Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4552024-04-17T17:10:54.936Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.536074103Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"client":"0","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4562024-04-17T17:10:54.936Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.536107055Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"client":"0","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4572024-04-17T17:10:54.936Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.536126496Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"client":"1","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4582024-04-17T17:10:54.937Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.536148608Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"client":"1","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4592024-04-17T17:10:54.937Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.53617998Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"client":"2","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4602024-04-17T17:10:54.937Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.536211382Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"client":"2","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4612024-04-17T17:10:54.937Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.536235533Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"":"io task","client":"0","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4622024-04-17T17:10:54.937Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.536266925Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"":"io task","client":"2","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4632024-04-17T17:10:54.937Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.536293827Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"":"io task","client":"1","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4642024-04-17T17:10:54.937Z{"msg":"ds_connection connected from Ok(127.0.0.1:56123)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.536463497Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"":"io task","client":"2","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4652024-04-17T17:10:54.937Z{"msg":"ds_connection connected from Ok(127.0.0.1:55600)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.53652378Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"":"io task","client":"1","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4662024-04-17T17:10:54.937Z{"msg":"ds_connection connected from Ok(127.0.0.1:42036)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.536563813Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"":"io task","client":"0","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4672024-04-17T17:10:54.937Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.537237204Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"client":"0","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4682024-04-17T17:10:54.937Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.537273026Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"client":"0","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4692024-04-17T17:10:54.937Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.537289137Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"client":"2","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4702024-04-17T17:10:54.937Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.537302878Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"client":"2","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4712024-04-17T17:10:54.937Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.537327259Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"client":"1","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4722024-04-17T17:10:54.937Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.537351241Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"client":"1","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4732024-04-17T17:10:54.937Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.537946727Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"client":"1","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4742024-04-17T17:10:54.937Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.538032432Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"client":"2","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4752024-04-17T17:10:54.937Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.538087135Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"client":"0","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4762024-04-17T17:10:54.937Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.53849633Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"client":"1","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4772024-04-17T17:10:54.937Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.538541063Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"client":"1","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4782024-04-17T17:10:54.937Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.538568035Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4792024-04-17T17:10:54.937Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.538623678Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"client":"2","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4802024-04-17T17:10:54.937Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.53866189Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"client":"2","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4812024-04-17T17:10:54.938Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.538678431Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4822024-04-17T17:10:54.938Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.538700632Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"client":"0","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4832024-04-17T17:10:54.938Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.538714963Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"client":"0","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4842024-04-17T17:10:54.938Z{"msg":"[0]R flush_numbers[0..12]: [3, 1, 1, 1, 1, 1, 3, 3, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.538735745Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"":"downstairs","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4852024-04-17T17:10:54.938Z{"msg":"[0]R generation[0..12]: [2, 1, 1, 1, 1, 1, 2, 2, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.538749786Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"":"downstairs","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4862024-04-17T17:10:54.938Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.538771037Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"":"downstairs","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4872024-04-17T17:10:54.938Z{"msg":"[1]R flush_numbers[0..12]: [3, 1, 1, 1, 1, 1, 3, 3, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.538785508Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"":"downstairs","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4882024-04-17T17:10:54.938Z{"msg":"[1]R generation[0..12]: [2, 1, 1, 1, 1, 1, 2, 2, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.538799238Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"":"downstairs","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4892024-04-17T17:10:54.938Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.538813959Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"":"downstairs","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4902024-04-17T17:10:54.938Z{"msg":"[2]R flush_numbers[0..12]: [3, 1, 1, 1, 1, 1, 3, 3, 1, 1, 1, 1]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
4912024-04-17T17:10:54.938Z,"time":"2024-04-17T17:10:53.53882747Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"":"downstairs","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4922024-04-17T17:10:54.938Z{"msg":"[2]R generation[0..12]: [2, 1, 1, 1, 1, 1, 2, 2, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.538842381Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"":"downstairs","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4932024-04-17T17:10:54.938Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.538863382Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"":"downstairs","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4942024-04-17T17:10:54.938Z{"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.538876353Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"":"downstairs","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4952024-04-17T17:10:54.938Z{"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.538896944Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"":"downstairs","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4962024-04-17T17:10:54.938Z{"msg":"Next flush: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.538910585Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"":"downstairs","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4972024-04-17T17:10:54.938Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.538930466Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"":"downstairs","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4982024-04-17T17:10:54.938Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.538958038Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
4992024-04-17T17:10:54.938Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.538976309Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"client":"0","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
5002024-04-17T17:10:54.938Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.53899713Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"client":"1","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
5012024-04-17T17:10:54.938Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.539010571Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"client":"2","":"downstairs","session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
5022024-04-17T17:10:54.938Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.539024172Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
5032024-04-17T17:10:54.938Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.539042723Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
5042024-04-17T17:10:54.938Z{"msg":"eefa0851-8060-42d6-9d43-96e679fe92a2 is now active with session: 7b2e5b54-ea92-4843-ad71-f78fe4126358","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.539058664Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
5052024-04-17T17:10:54.939Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.539090806Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184,"session_id":"7b2e5b54-ea92-4843-ad71-f78fe4126358"}
5062024-04-17T17:10:54.939Z{"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:10:53.539124788Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184}
5072024-04-17T17:10:54.939ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
5082024-04-17T17:10:54.939ZLoading write count information from file "/tmp/test_repair_verify.data"
5092024-04-17T17:10:54.939ZRun Repair workload
5102024-04-17T17:10:54.943Z01/30 Write block 392 len 12288 data: 2 2 2
5112024-04-17T17:10:54.943Z02/30 Write block 251 len 16384 data: 2 3 3 3
5122024-04-17T17:10:54.943Z03/30 Write block 284 len 36864 data: 2 2 2 2 2 2 2 2 2
5132024-04-17T17:10:54.943Z04/30 Write block 40 len 4096 data: 2
5142024-04-17T17:10:54.943Z05/30 Write block 411 len 40960 data: 2 2 2 2 2 2 2 2 2 2
5152024-04-17T17:10:54.943Z06/30 Read block 448 len 20480
5162024-04-17T17:10:54.943Z07/30 Flush
5172024-04-17T17:10:54.947Z08/30 Write block 579 len 32768 data: 3 3 3 2 2 2 2 2
5182024-04-17T17:10:54.947Z09/30 Read block 260 len 36864
5192024-04-17T17:10:54.947Z10/30 Read block 179 len 24576
5202024-04-17T17:10:54.947Z11/30 Flush
5212024-04-17T17:10:54.951Z12/30 Write block 473 len 32768 data: 2 2 2 2 2 2 3 3
5222024-04-17T17:10:54.951Z13/30 Write block 45 len 12288 data: 2 2 2
5232024-04-17T17:10:54.951Z14/30 Read block 62 len 40960
5242024-04-17T17:10:54.951Z15/30 Flush
5252024-04-17T17:10:54.955Z16/30 Write block 175 len 8192 data: 2 2
5262024-04-17T17:10:54.955Z17/30 Write block 305 len 4096 data: 2
5272024-04-17T17:10:54.955Z18/30 Write block 227 len 12288 data: 2 2 2
5282024-04-17T17:10:54.958Z19/30 Read block 367 len 24576
5292024-04-17T17:10:54.959Z20/30 Read block 378 len 8192
5302024-04-17T17:10:54.959Z21/30 Write block 203 len 20480 data: 2 2 2 2 2
5312024-04-17T17:10:54.959Z22/30 Read block 215 len 4096
5322024-04-17T17:10:54.959Z23/30 Read block 548 len 20480
5332024-04-17T17:10:54.962Z24/30 Read block 587 len 24576
5342024-04-17T17:10:54.962Z25/30 Write block 257 len 12288 data: 2 2 2
5352024-04-17T17:10:54.962Z26/30 Write block 246 len 28672 data: 2 2 2 2 2 3 4
5362024-04-17T17:10:54.962Z27/30 Read block 473 len 24576
5372024-04-17T17:10:54.962Z28/30 Read block 37 len 12288
5382024-04-17T17:10:54.967Z29/30 Read block 444 len 16384
5392024-04-17T17:10:54.967Z30/30 Read block 239 len 4096
5402024-04-17T17:10:54.967Z----------------------------------------------------------------
5412024-04-17T17:10:54.967Z Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:30
5422024-04-17T17:10:54.967ZStates: Active Active Active
5432024-04-17T17:10:54.967ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
5442024-04-17T17:10:54.967Z 1 Acked 1000 Write 3 Done Done Done false
5452024-04-17T17:10:54.967Z 2 Acked 1001 Write 4 Done Done Done false
5462024-04-17T17:10:54.967Z 3 Acked 1002 Write 9 Done Done Done false
5472024-04-17T17:10:54.967Z 4 Acked 1003 Write 1 Done Done Done false
5482024-04-17T17:10:54.967Z 5 Acked 1004 Write 10 Done Done Sent false
5492024-04-17T17:10:54.967Z 6 Acked 1005 Read 5 Done Done Sent false
5502024-04-17T17:10:54.967Z 7 Acked 1006 Flush 0 Done Done Sent false
5512024-04-17T17:10:54.967Z 8 Acked 1007 Write 8 Done Done Sent false
5522024-04-17T17:10:54.967Z 9 Acked 1008 Read 9 Done Done Sent false
5532024-04-17T17:10:54.967Z 10 Acked 1009 Read 6 Done Done Sent false
5542024-04-17T17:10:54.967Z 11 Acked 1010 Flush 0 Done Done Sent false
5552024-04-17T17:10:54.967Z 12 Acked 1011 Write 8 Done Done Sent false
5562024-04-17T17:10:54.967Z 13 Acked 1012 Write 3 Done Done Sent false
5572024-04-17T17:10:54.967Z 14 Acked 1013 Read 10 Done Done Sent false
5582024-04-17T17:10:54.967Z 15 Acked 1014 Flush 0 Done Done Sent false
5592024-04-17T17:10:54.967Z 16 Acked 1015 Write 2 Done Done Sent false
5602024-04-17T17:10:54.968Z 17 Acked 1016 Write 1 Done Done Sent false
5612024-04-17T17:10:54.968Z 18 Acked 1017 Write 3 Done Done Sent false
5622024-04-17T17:10:54.968Z 19 Acked 1018 Read 6 Done Done Sent false
5632024-04-17T17:10:54.968Z 20 Acked 1019 Read 2 Done Done Sent false
5642024-04-17T17:10:54.968Z 21 Acked 1020 Write 5 Done Done Sent false
5652024-04-17T17:10:54.968Z 22 Acked 1021 Read 1 Done Done Sent false
5662024-04-17T17:10:54.968Z 23 Acked 1022 Read 5 Done Done Sent false
5672024-04-17T17:10:54.968Z 24 Acked 1023 Read 6 Done Done Sent false
5682024-04-17T17:10:54.968Z 25 Acked 1024 Write 3 Done Done Sent false
5692024-04-17T17:10:54.968Z 26 Acked 1025 Write 7 Done Done Sent false
5702024-04-17T17:10:54.968Z 27 Acked 1026 Read 6 Done Done Sent false
5712024-04-17T17:10:54.968Z 28 Acked 1027 Read 3 Done Done Sent false
5722024-04-17T17:10:54.968Z 29 Acked 1028 Read 4 Done Done Sent false
5732024-04-17T17:10:54.968Z 30 Acked 1029 Read 1 Done Done Sent false
5742024-04-17T17:10:54.968Z STATES DS:0 DS:1 DS:2 TOTAL
5752024-04-17T17:10:54.968Z New 0 0 0 0
5762024-04-17T17:10:54.968Z Sent 0 0 26 26
5772024-04-17T17:10:54.968Z Done 30 30 4 64
5782024-04-17T17:10:54.968Z Skipped 0 0 0 0
5792024-04-17T17:10:54.968Z Error 0 0 0 0
5802024-04-17T17:10:54.968ZLast Flush: 1014 1014 0
5812024-04-17T17:10:54.968ZDownstairs last five completed:
5822024-04-17T17:10:54.968ZUpstairs last five completed: 30 29 28 27 26
5832024-04-17T17:10:54.968Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:10:53.571034662Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1184}
5842024-04-17T17:10:54.969ZWrote out file "/tmp/test_repair_verify.data"
5852024-04-17T17:10:54.972Z
5862024-04-17T17:10:54.973ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
5872024-04-17T17:10:54.988ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5882024-04-17T17:10:54.988Z 0 000-019 2 2 2 3 3 3 F F F
5892024-04-17T17:10:54.993Z 1 020-039 1 1 1 1 1 1 F F F
5902024-04-17T17:10:54.993Z 2 040-059 4 4 1 8 8 1 F F T <---
5912024-04-17T17:10:54.993Z 3 060-079 1 1 1 1 1 1 F F F
5922024-04-17T17:10:54.993Z 4 080-099 1 1 1 1 1 1 F F F
5932024-04-17T17:10:54.993Z 5 100-119 1 1 1 1 1 1 F F F
5942024-04-17T17:10:54.993Z 6 120-139 2 2 2 3 3 3 F F F
5952024-04-17T17:10:54.993Z 7 140-159 2 2 2 3 3 3 F F F
5962024-04-17T17:10:54.993Z 8 160-179 1 1 1 1 1 1 T T F <---
5972024-04-17T17:10:54.993Z 9 180-199 1 1 1 1 1 1 F F F
5982024-04-17T17:10:54.993Z 10 200-219 1 1 1 1 1 1 T T F <---
5992024-04-17T17:10:54.993Z 11 220-239 1 1 1 1 1 1 T T F <---
6002024-04-17T17:10:54.993Z 12 240-259 4 4 2 6 6 3 T T T <---
6012024-04-17T17:10:54.993Z 13 260-279 1 1 1 1 1 1 F F F
6022024-04-17T17:10:54.993Z 14 280-299 4 4 1 6 6 1 F F T <---
6032024-04-17T17:10:54.993Z 15 300-319 2 2 2 3 3 3 T T F <---
6042024-04-17T17:10:54.994Z 16 320-339 2 2 2 3 3 3 F F F
6052024-04-17T17:10:54.994Z 17 340-359 2 2 2 2 2 2 F F F
6062024-04-17T17:10:54.994Z 18 360-379 2 2 2 3 3 3 F F F
6072024-04-17T17:10:54.994Z 19 380-399 4 4 2 6 6 3 F F T <---
6082024-04-17T17:10:54.994Z 20 400-419 4 4 1 6 6 1 F F F <---
6092024-04-17T17:10:54.994Z 21 420-439 4 4 1 6 6 1 F F F <---
6102024-04-17T17:10:54.994Z 22 440-459 1 1 1 1 1 1 F F F
6112024-04-17T17:10:54.994Z 23 460-479 4 4 2 8 8 4 F F F <---
6122024-04-17T17:10:54.994Z 24 480-499 4 4 2 8 8 4 F F F <---
6132024-04-17T17:10:54.994Z 25 500-519 2 2 2 3 3 3 F F F
6142024-04-17T17:10:54.994Z 26 520-539 3 3 3 5 5 5 F F F
6152024-04-17T17:10:54.994Z 27 540-559 3 3 3 5 5 5 F F F
6162024-04-17T17:10:54.994Z 28 560-579 4 4 2 7 7 3 F F F <---
6172024-04-17T17:10:54.994Z 29 580-599 4 4 2 7 7 3 F F F <---
6182024-04-17T17:10:54.994ZMax gen: 4, Max flush: 8
6192024-04-17T17:10:54.994ZError: Difference in extent metadata found!
6202024-04-17T17:10:54.994ZA Difference in extent metadata is expected here
6212024-04-17T17:10:54.994ZOn loop 2
6222024-04-17T17:10:54.994Z
6232024-04-17T17:10:54.998ZVerifying data now
6242024-04-17T17:11:09.641ZNo change after verify
6252024-04-17T17:11:09.641ZLoop: 2 Downstairs dump after verify (and repair):
6262024-04-17T17:11:09.654ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6272024-04-17T17:11:09.654Z 0 000-019 2 2 2 3 3 3 F F F
6282024-04-17T17:11:09.659Z 1 020-039 1 1 1 1 1 1 F F F
6292024-04-17T17:11:09.659Z 2 040-059 4 4 4 8 8 8 F F F
6302024-04-17T17:11:09.659Z 3 060-079 1 1 1 1 1 1 F F F
6312024-04-17T17:11:09.659Z 4 080-099 1 1 1 1 1 1 F F F
6322024-04-17T17:11:09.659Z 5 100-119 1 1 1 1 1 1 F F F
6332024-04-17T17:11:09.659Z 6 120-139 2 2 2 3 3 3 F F F
6342024-04-17T17:11:09.659Z 7 140-159 2 2 2 3 3 3 F F F
6352024-04-17T17:11:09.659Z 8 160-179 5 5 5 9 9 9 F F F
6362024-04-17T17:11:09.659Z 9 180-199 1 1 1 1 1 1 F F F
6372024-04-17T17:11:09.659Z 10 200-219 5 5 5 9 9 9 F F F
6382024-04-17T17:11:09.659Z 11 220-239 5 5 5 9 9 9 F F F
6392024-04-17T17:11:09.659Z 12 240-259 5 5 5 9 9 9 F F F
6402024-04-17T17:11:09.659Z 13 260-279 1 1 1 1 1 1 F F F
6412024-04-17T17:11:09.659Z 14 280-299 4 4 4 6 6 6 F F F
6422024-04-17T17:11:09.659Z 15 300-319 5 5 5 9 9 9 F F F
6432024-04-17T17:11:09.659Z 16 320-339 2 2 2 3 3 3 F F F
6442024-04-17T17:11:09.659Z 17 340-359 2 2 2 2 2 2 F F F
6452024-04-17T17:11:09.659Z 18 360-379 2 2 2 3 3 3 F F F
6462024-04-17T17:11:09.659Z 19 380-399 4 4 4 6 6 6 F F F
6472024-04-17T17:11:09.659Z 20 400-419 4 4 4 6 6 6 F F F
6482024-04-17T17:11:09.660Z 21 420-439 4 4 4 6 6 6 F F F
6492024-04-17T17:11:09.660Z 22 440-459 1 1 1 1 1 1 F F F
6502024-04-17T17:11:09.660Z 23 460-479 4 4 4 8 8 8 F F F
6512024-04-17T17:11:09.660Z 24 480-499 4 4 4 8 8 8 F F F
6522024-04-17T17:11:09.660Z 25 500-519 2 2 2 3 3 3 F F F
6532024-04-17T17:11:09.660Z 26 520-539 3 3 3 5 5 5 F F F
6542024-04-17T17:11:09.660Z 27 540-559 3 3 3 5 5 5 F F F
6552024-04-17T17:11:09.660Z 28 560-579 4 4 4 7 7 7 F F F
6562024-04-17T17:11:09.660Z 29 580-599 4 4 4 7 7 7 F F F
6572024-04-17T17:11:09.660ZMax gen: 5, Max flush: 9
6582024-04-17T17:11:09.660Z
6592024-04-17T17:11:09.660ZBegin loop 3
6602024-04-17T17:11:09.660ZDownstairs to restart: 0
6612024-04-17T17:11:09.676Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.282855194Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192}
6622024-04-17T17:11:09.676Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.283099589Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192}
6632024-04-17T17:11:09.682Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.2831268Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192}
6642024-04-17T17:11:09.682Z{"msg":"Crucible e46e4704-9366-4744-947e-1bbdafeecf0a has session id: e89d3b3c-38c4-41e8-b1ea-22eea926c2eb","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.283158102Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6652024-04-17T17:11:09.682Z{"msg":"Upstairs opts: Upstairs UUID: e46e4704-9366-4744-947e-1bbdafeecf0a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.283205765Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6662024-04-17T17:11:09.682ZCrucible runtime is spawned
6672024-04-17T17:11:09.682Z{"msg":"Crucible stats registered with UUID: e46e4704-9366-4744-947e-1bbdafeecf0a","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.283257048Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6682024-04-17T17:11:09.682Z{"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.28329231Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192}
6692024-04-17T17:11:09.682Z{"msg":"e46e4704-9366-4744-947e-1bbdafeecf0a active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.283395936Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6702024-04-17T17:11:09.682Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.283421648Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"client":"0","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6712024-04-17T17:11:09.683Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.283436629Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"client":"0","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6722024-04-17T17:11:09.683Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.283474221Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"client":"1","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6732024-04-17T17:11:09.683Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.283498333Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"client":"1","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6742024-04-17T17:11:09.683Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.283529695Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"client":"2","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6752024-04-17T17:11:09.683Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.283554696Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"client":"2","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6762024-04-17T17:11:09.683Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.283577407Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"":"io task","client":"2","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6772024-04-17T17:11:09.683Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.28362188Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"":"io task","client":"0","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6782024-04-17T17:11:09.683Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.283652322Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"":"io task","client":"1","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6792024-04-17T17:11:09.683Z{"msg":"ds_connection connected from Ok(127.0.0.1:45191)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.283699135Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"":"io task","client":"1","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6802024-04-17T17:11:09.683Z{"msg":"ds_connection connected from Ok(127.0.0.1:61930)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.283742487Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"":"io task","client":"0","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6812024-04-17T17:11:09.683Z{"msg":"ds_connection connected from Ok(127.0.0.1:57994)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.283823462Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"":"io task","client":"2","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6822024-04-17T17:11:09.683Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.284487253Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"client":"2","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6832024-04-17T17:11:09.683Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.284545676Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"client":"2","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6842024-04-17T17:11:09.683Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.284605Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"client":"0","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6852024-04-17T17:11:09.683Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.284668693Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"client":"0","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6862024-04-17T17:11:09.683Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.284717887Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"client":"1","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6872024-04-17T17:11:09.683Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.284741308Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"client":"1","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6882024-04-17T17:11:09.683Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.285142722Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"client":"2","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6892024-04-17T17:11:09.683Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.285187615Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"client":"0","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6902024-04-17T17:11:09.683Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.285224817Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"client":"1","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6912024-04-17T17:11:09.683Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.285752209Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"client":"0","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6922024-04-17T17:11:09.683Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.285787781Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"client":"0","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6932024-04-17T17:11:09.683Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.285821773Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6942024-04-17T17:11:09.684Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.285843645Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"client":"2","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6952024-04-17T17:11:09.684Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.285861066Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"client":"2","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6962024-04-17T17:11:09.684Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.285884617Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6972024-04-17T17:11:09.684Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.285898548Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"client":"1","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6982024-04-17T17:11:09.684Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.285922649Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"client":"1","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
6992024-04-17T17:11:09.684Z{"msg":"[0]R flush_numbers[0..12]: [3, 1, 8, 1, 1, 1, 3, 3, 9, 1, 9, 9]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.285940631Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"":"downstairs","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
7002024-04-17T17:11:09.684ZWait for a query_work_queue command to finish before sending IO
7012024-04-17T17:11:09.684Z{"msg":"[0]R generation[0..12]: [2, 1, 4, 1, 1, 1, 2, 2, 5, 1, 5, 5]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.285966192Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"":"downstairs","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
7022024-04-17T17:11:09.684Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.285985713Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"":"downstairs","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
7032024-04-17T17:11:09.684Z{"msg":"[1]R flush_numbers[0..12]: [3, 1, 8, 1, 1, 1, 3, 3, 9, 1, 9, 9]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.286015145Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"":"downstairs","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
7042024-04-17T17:11:09.684Z{"msg":"[1]R generation[0..12]: [2, 1, 4, 1, 1, 1, 2, 2, 5, 1, 5, 5]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.286038916Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"":"downstairs","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
7052024-04-17T17:11:09.684Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.286053457Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"":"downstairs","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
7062024-04-17T17:11:09.684Z{"msg":"[2]R flush_numbers[0..12]: [3, 1, 8, 1, 1, 1, 3, 3, 9, 1, 9, 9]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.286076809Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"":"downstairs","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
7072024-04-17T17:11:09.684Z{"msg":"[2]R generation[0..12]: [2, 1, 4, 1, 1, 1, 2, 2, 5, 1, 5, 5]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.28610193Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"":"downstairs","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
7082024-04-17T17:11:09.684Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.286125952Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"":"downstairs","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
7092024-04-17T17:11:09.684Z{"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.286140303Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"":"downstairs","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
7102024-04-17T17:11:09.684Z{"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.286168454Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"":"downstairs","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
7112024-04-17T17:11:09.684Z{"msg":"Next flush: 10","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.286204317Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"":"downstairs","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
7122024-04-17T17:11:09.684Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.286231048Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"":"downstairs","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
7132024-04-17T17:11:09.684Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.286247329Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
7142024-04-17T17:11:09.684Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.286279551Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"client":"0","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
7152024-04-17T17:11:09.684Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.286313133Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"client":"1","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
7162024-04-17T17:11:09.684Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.286329564Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"client":"2","":"downstairs","session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
7172024-04-17T17:11:09.685Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.286356506Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
7182024-04-17T17:11:09.685Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.286390828Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
7192024-04-17T17:11:09.685Z{"msg":"e46e4704-9366-4744-947e-1bbdafeecf0a is now active with session: e89d3b3c-38c4-41e8-b1ea-22eea926c2eb","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.286416459Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
7202024-04-17T17:11:09.685Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.28643105Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192,"session_id":"e89d3b3c-38c4-41e8-b1ea-22eea926c2eb"}
7212024-04-17T17:11:09.685ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
7222024-04-17T17:11:09.685Z{"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:08.286463752Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192}
7232024-04-17T17:11:09.689ZLoading write count information from file "/tmp/test_repair_verify.data"
7242024-04-17T17:11:09.689ZRun Repair workload
7252024-04-17T17:11:09.689Z01/30 Write block 29 len 28672 data: 2 2 2 2 2 2 2
7262024-04-17T17:11:09.689Z02/30 Write block 468 len 20480 data: 2 2 2 2 2
7272024-04-17T17:11:09.689Z03/30 Write block 575 len 20480 data: 3 3 3 3 4
7282024-04-17T17:11:09.689Z04/30 Write block 239 len 36864 data: 2 2 2 2 2 2 2 3 3
7292024-04-17T17:11:09.689Z05/30 Read block 28 len 28672
7302024-04-17T17:11:09.689Z06/30 Read block 133 len 32768
7312024-04-17T17:11:09.689Z07/30 Write block 328 len 12288 data: 2 2 2
7322024-04-17T17:11:09.689Z08/30 Flush
7332024-04-17T17:11:09.693Z09/30 Write block 10 len 36864 data: 3 2 2 2 2 2 2 2 2
7342024-04-17T17:11:09.693Z10/30 Write block 514 len 8192 data: 2 2
7352024-04-17T17:11:09.696Z11/30 Read block 259 len 32768
7362024-04-17T17:11:09.696Z12/30 Flush
7372024-04-17T17:11:09.697Z13/30 Write block 250 len 8192 data: 3 4
7382024-04-17T17:11:09.697Z14/30 Read block 584 len 40960
7392024-04-17T17:11:09.700Z15/30 Read block 244 len 36864
7402024-04-17T17:11:09.700Z16/30 Flush
7412024-04-17T17:11:09.704Z17/30 Write block 87 len 28672 data: 2 2 2 2 2 2 2
7422024-04-17T17:11:09.704Z18/30 Write block 273 len 16384 data: 2 2 2 2
7432024-04-17T17:11:09.704Z19/30 Read block 47 len 28672
7442024-04-17T17:11:09.704Z20/30 Read block 138 len 40960
7452024-04-17T17:11:09.704Z21/30 Read block 166 len 20480
7462024-04-17T17:11:09.708Z22/30 Read block 347 len 24576
7472024-04-17T17:11:09.708Z23/30 Read block 198 len 12288
7482024-04-17T17:11:09.708Z24/30 Read block 146 len 40960
7492024-04-17T17:11:09.708Z25/30 Write block 496 len 36864 data: 2 2 2 2 3 3 3 2 2
7502024-04-17T17:11:09.712Z26/30 Read block 363 len 16384
7512024-04-17T17:11:09.712Z27/30 Write block 0 len 28672 data: 2 2 2 2 2 2 2
7522024-04-17T17:11:09.712Z28/30 Write block 106 len 40960 data: 2 2 2 2 2 2 2 2 2 2
7532024-04-17T17:11:09.712Z29/30 Read block 284 len 32768
7542024-04-17T17:11:09.716Z30/30 Write block 350 len 20480 data: 3 3 3 2 2
7552024-04-17T17:11:09.716Z----------------------------------------------------------------
7562024-04-17T17:11:09.716Z Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:30
7572024-04-17T17:11:09.716ZStates: Active Active Active
7582024-04-17T17:11:09.716ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
7592024-04-17T17:11:09.716Z 1 Acked 1000 Write 7 Done Done Done false
7602024-04-17T17:11:09.716Z 2 Acked 1001 Write 5 Sent Done Done false
7612024-04-17T17:11:09.716Z 3 Acked 1002 Write 5 Sent Done Done false
7622024-04-17T17:11:09.716Z 4 Acked 1003 Write 9 Sent Done Done false
7632024-04-17T17:11:09.716Z 5 Acked 1004 Read 7 Sent Done Done false
7642024-04-17T17:11:09.716Z 6 Acked 1005 Read 8 Sent Done Done false
7652024-04-17T17:11:09.716Z 7 Acked 1006 Write 3 Sent Done Done false
7662024-04-17T17:11:09.717Z 8 Acked 1007 Flush 0 Sent Done Done false
7672024-04-17T17:11:09.717Z 9 Acked 1008 Write 9 Sent Done Done false
7682024-04-17T17:11:09.717Z 10 Acked 1009 Write 2 Sent Done Done false
7692024-04-17T17:11:09.717Z 11 Acked 1010 Read 8 Sent Done Done false
7702024-04-17T17:11:09.717Z 12 Acked 1011 Flush 0 Sent Done Done false
7712024-04-17T17:11:09.717Z 13 Acked 1012 Write 2 Sent Done Done false
7722024-04-17T17:11:09.717Z 14 Acked 1013 Read 10 Sent Done Done false
7732024-04-17T17:11:09.717Z 15 Acked 1014 Read 9 Sent Done Done false
7742024-04-17T17:11:09.717Z 16 Acked 1015 Flush 0 Sent Done Done false
7752024-04-17T17:11:09.717Z 17 Acked 1016 Write 7 Sent Done Done false
7762024-04-17T17:11:09.717Z 18 Acked 1017 Write 4 Sent Done Done false
7772024-04-17T17:11:09.717Z 19 Acked 1018 Read 7 Sent Done Done false
7782024-04-17T17:11:09.717Z 20 Acked 1019 Read 10 Sent Done Done false
7792024-04-17T17:11:09.717Z 21 Acked 1020 Read 5 Sent Done Done false
7802024-04-17T17:11:09.717Z 22 Acked 1021 Read 6 Sent Done Done false
7812024-04-17T17:11:09.717Z 23 Acked 1022 Read 3 Sent Done Done false
7822024-04-17T17:11:09.717Z 24 Acked 1023 Read 10 Sent Done Done false
7832024-04-17T17:11:09.717Z 25 Acked 1024 Write 9 Sent Done Done false
7842024-04-17T17:11:09.717Z 26 Acked 1025 Read 4 Sent Done Done false
7852024-04-17T17:11:09.717Z 27 Acked 1026 Write 7 Sent Done Done false
7862024-04-17T17:11:09.717Z 28 Acked 1027 Write 10 Sent Done Done false
7872024-04-17T17:11:09.717Z 29 Acked 1028 Read 8 Sent Done Done false
7882024-04-17T17:11:09.717Z 30 Acked 1029 Write 5 Sent Sent Sent false
7892024-04-17T17:11:09.717Z STATES DS:0 DS:1 DS:2 TOTAL
7902024-04-17T17:11:09.718Z New 0 0 0 0
7912024-04-17T17:11:09.718Z Sent 29 1 1 31
7922024-04-17T17:11:09.718Z Done 1 29 29 59
7932024-04-17T17:11:09.718Z Skipped 0 0 0 0
7942024-04-17T17:11:09.718Z Error 0 0 0 0
7952024-04-17T17:11:09.718ZLast Flush: 0 1015 1015
7962024-04-17T17:11:09.718ZDownstairs last five completed:
7972024-04-17T17:11:09.718ZUpstairs last five completed: 30 29 28 27 26
7982024-04-17T17:11:09.718Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:11:08.320224939Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1192}
7992024-04-17T17:11:09.718ZWrote out file "/tmp/test_repair_verify.data"
8002024-04-17T17:11:09.722Z
8012024-04-17T17:11:09.722ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
8022024-04-17T17:11:09.737ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8032024-04-17T17:11:09.738Z 0 000-019 2 6 6 3 11 11 F T T <---
8042024-04-17T17:11:09.738Z 1 020-039 1 6 6 1 10 10 T F F <---
8052024-04-17T17:11:09.738Z 2 040-059 4 4 4 8 8 8 F F F
8062024-04-17T17:11:09.738Z 3 060-079 1 1 1 1 1 1 F F F
8072024-04-17T17:11:09.738Z 4 080-099 1 1 1 1 1 1 F T T <---
8082024-04-17T17:11:09.754Z 5 100-119 1 1 1 1 1 1 F T T <---
8092024-04-17T17:11:09.754Z 6 120-139 2 2 2 3 3 3 F F F
8102024-04-17T17:11:09.754Z 7 140-159 2 2 2 3 3 3 F F F
8112024-04-17T17:11:09.754Z 8 160-179 5 5 5 9 9 9 F F F
8122024-04-17T17:11:09.754Z 9 180-199 1 1 1 1 1 1 F F F
8132024-04-17T17:11:09.754Z 10 200-219 5 5 5 9 9 9 F F F
8142024-04-17T17:11:09.754Z 11 220-239 5 6 6 9 10 10 F F F <---
8152024-04-17T17:11:09.754Z 12 240-259 5 6 6 9 12 12 F F F <---
8162024-04-17T17:11:09.754Z 13 260-279 1 1 1 1 1 1 F T T <---
8172024-04-17T17:11:09.755Z 14 280-299 4 4 4 6 6 6 F F F
8182024-04-17T17:11:09.755Z 15 300-319 5 5 5 9 9 9 F F F
8192024-04-17T17:11:09.755Z 16 320-339 2 6 6 3 10 10 F F F <---
8202024-04-17T17:11:09.755Z 17 340-359 2 2 2 2 2 2 F T T <---
8212024-04-17T17:11:09.755Z 18 360-379 2 2 2 3 3 3 F F F
8222024-04-17T17:11:09.755Z 19 380-399 4 4 4 6 6 6 F F F
8232024-04-17T17:11:09.755Z 20 400-419 4 4 4 6 6 6 F F F
8242024-04-17T17:11:09.755Z 21 420-439 4 4 4 6 6 6 F F F
8252024-04-17T17:11:09.755Z 22 440-459 1 1 1 1 1 1 F F F
8262024-04-17T17:11:09.755Z 23 460-479 4 6 6 8 10 10 F F F <---
8272024-04-17T17:11:09.755Z 24 480-499 4 4 4 8 8 8 F T T <---
8282024-04-17T17:11:09.755Z 25 500-519 2 6 6 3 11 11 F T T <---
8292024-04-17T17:11:09.756Z 26 520-539 3 3 3 5 5 5 F F F
8302024-04-17T17:11:09.756Z 27 540-559 3 3 3 5 5 5 F F F
8312024-04-17T17:11:09.756Z 28 560-579 4 6 6 7 10 10 F F F <---
8322024-04-17T17:11:09.756Z 29 580-599 4 4 4 7 7 7 F F F
8332024-04-17T17:11:09.756ZMax gen: 6, Max flush: 12
8342024-04-17T17:11:09.756ZError: Difference in extent metadata found!
8352024-04-17T17:11:09.756ZA Difference in extent metadata is expected here
8362024-04-17T17:11:09.756ZOn loop 3
8372024-04-17T17:11:09.756Z
8382024-04-17T17:11:09.761ZVerifying data now
8392024-04-17T17:11:14.347ZNo change after verify
8402024-04-17T17:11:14.347ZLoop: 3 Downstairs dump after verify (and repair):
8412024-04-17T17:11:14.362ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8422024-04-17T17:11:14.362Z 0 000-019 7 7 7 13 13 13 F F F
8432024-04-17T17:11:14.362Z 1 020-039 6 6 6 10 10 10 F F F
8442024-04-17T17:11:14.362Z 2 040-059 4 4 4 8 8 8 F F F
8452024-04-17T17:11:14.363Z 3 060-079 1 1 1 1 1 1 F F F
8462024-04-17T17:11:14.363Z 4 080-099 7 7 7 13 13 13 F F F
8472024-04-17T17:11:14.363Z 5 100-119 7 7 7 13 13 13 F F F
8482024-04-17T17:11:14.363Z 6 120-139 2 2 2 3 3 3 F F F
8492024-04-17T17:11:14.363Z 7 140-159 2 2 2 3 3 3 F F F
8502024-04-17T17:11:14.363Z 8 160-179 5 5 5 9 9 9 F F F
8512024-04-17T17:11:14.371Z 9 180-199 1 1 1 1 1 1 F F F
8522024-04-17T17:11:14.371Z 10 200-219 5 5 5 9 9 9 F F F
8532024-04-17T17:11:14.371Z 11 220-239 6 6 6 10 10 10 F F F
8542024-04-17T17:11:14.371Z 12 240-259 6 6 6 12 12 12 F F F
8552024-04-17T17:11:14.371Z 13 260-279 7 7 7 13 13 13 F F F
8562024-04-17T17:11:14.371Z 14 280-299 4 4 4 6 6 6 F F F
8572024-04-17T17:11:14.371Z 15 300-319 5 5 5 9 9 9 F F F
8582024-04-17T17:11:14.371Z 16 320-339 6 6 6 10 10 10 F F F
8592024-04-17T17:11:14.371Z 17 340-359 7 7 7 13 13 13 F F F
8602024-04-17T17:11:14.371Z 18 360-379 2 2 2 3 3 3 F F F
8612024-04-17T17:11:14.371Z 19 380-399 4 4 4 6 6 6 F F F
8622024-04-17T17:11:14.371Z 20 400-419 4 4 4 6 6 6 F F F
8632024-04-17T17:11:14.371Z 21 420-439 4 4 4 6 6 6 F F F
8642024-04-17T17:11:14.371Z 22 440-459 1 1 1 1 1 1 F F F
8652024-04-17T17:11:14.371Z 23 460-479 6 6 6 10 10 10 F F F
8662024-04-17T17:11:14.372Z 24 480-499 7 7 7 13 13 13 F F F
8672024-04-17T17:11:14.372Z 25 500-519 7 7 7 13 13 13 F F F
8682024-04-17T17:11:14.372Z 26 520-539 3 3 3 5 5 5 F F F
8692024-04-17T17:11:14.372Z 27 540-559 3 3 3 5 5 5 F F F
8702024-04-17T17:11:14.372Z 28 560-579 6 6 6 10 10 10 F F F
8712024-04-17T17:11:14.372Z 29 580-599 4 4 4 7 7 7 F F F
8722024-04-17T17:11:14.372ZMax gen: 7, Max flush: 13
8732024-04-17T17:11:14.372Z
8742024-04-17T17:11:14.372ZBegin loop 4
8752024-04-17T17:11:14.372ZDownstairs to restart: 0
8762024-04-17T17:11:14.385Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.99079231Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200}
8772024-04-17T17:11:14.385Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.991131261Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200}
8782024-04-17T17:11:14.392Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0Crucible runtime is spawned
8792024-04-17T17:11:14.392Z,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.991196054Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200}
8802024-04-17T17:11:14.392Z{"msg":"Crucible 1d0177b4-a5e4-4b73-ba43-b9bb0a33b52e has session id: 2f525993-b564-44d8-a2b2-de5fc6be6dcd","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.991229836Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
8812024-04-17T17:11:14.392Z{"msg":"Upstairs opts: Upstairs UUID: 1d0177b4-a5e4-4b73-ba43-b9bb0a33b52e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.99129294Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
8822024-04-17T17:11:14.392Z{"msg":"Crucible stats registered with UUID: 1d0177b4-a5e4-4b73-ba43-b9bb0a33b52e","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.991335483Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
8832024-04-17T17:11:14.392Z{"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.991358084Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200}
8842024-04-17T17:11:14.392Z{"msg":"1d0177b4-a5e4-4b73-ba43-b9bb0a33b52e active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.991375625Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
8852024-04-17T17:11:14.392Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.991403187Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"client":"0","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
8862024-04-17T17:11:14.392Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.991425108Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"client":"0","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
8872024-04-17T17:11:14.392Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.99144718Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"client":"1","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
8882024-04-17T17:11:14.392Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.991468861Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"client":"1","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
8892024-04-17T17:11:14.392Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.991489162Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"client":"2","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
8902024-04-17T17:11:14.392Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.991512404Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"client":"2","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
8912024-04-17T17:11:14.392Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.991536475Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
8922024-04-17T17:11:14.392Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.991569767Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"":"io task","client":"0","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
8932024-04-17T17:11:14.392Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.991595499Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"":"io task","client":"1","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
8942024-04-17T17:11:14.392Z{"msg":"ds_connection connected from Ok(127.0.0.1:44856)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.99178005Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
8952024-04-17T17:11:14.393Z{"msg":"ds_connection connected from Ok(127.0.0.1:60068)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.991962961Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"":"io task","client":"0","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
8962024-04-17T17:11:14.393Z{"msg":"ds_connection connected from Ok(127.0.0.1:44372)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.992013574Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"":"io task","client":"1","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
8972024-04-17T17:11:14.393Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.992453361Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"client":"1","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
8982024-04-17T17:11:14.393Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.992488993Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"client":"1","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
8992024-04-17T17:11:14.393Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.992504974Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"client":"0","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9002024-04-17T17:11:14.393Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.992521425Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"client":"0","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9012024-04-17T17:11:14.393Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.992585769Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"client":"2","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9022024-04-17T17:11:14.393Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.992673074Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"client":"2","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9032024-04-17T17:11:14.393Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.993044446Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"client":"2","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9042024-04-17T17:11:14.393Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.993084809Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"client":"1","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9052024-04-17T17:11:14.393Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.993144803Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"client":"0","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9062024-04-17T17:11:14.394Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.993587799Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"client":"2","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9072024-04-17T17:11:14.394Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.993641323Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"client":"2","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9082024-04-17T17:11:14.394Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.993713257Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9092024-04-17T17:11:14.394Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.993749029Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"client":"1","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9102024-04-17T17:11:14.394Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.993774601Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"client":"1","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9112024-04-17T17:11:14.394Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.993806683Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9122024-04-17T17:11:14.394Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.993828774Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"client":"0","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9132024-04-17T17:11:14.394Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.993856686Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"client":"0","":"Wait for a query_work_queue command to finish before sending IO
9142024-04-17T17:11:14.394Zdownstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9152024-04-17T17:11:14.394Z{"msg":"[0]R flush_numbers[0..12]: [13, 10, 8, 1, 13, 13, 3, 3, 9, 1, 9, 10]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.993883407Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"":"downstairs","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9162024-04-17T17:11:14.394Z{"msg":"[0]R generation[0..12]: [7, 6, 4, 1, 7, 7, 2, 2, 5, 1, 5, 6]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.993910729Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"":"downstairs","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9172024-04-17T17:11:14.394Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.993936231Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"":"downstairs","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9182024-04-17T17:11:14.394Z{"msg":"[1]R flush_numbers[0..12]: [13, 10, 8, 1, 13, 13, 3, 3, 9, 1, 9, 10]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.993960172Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"":"downstairs","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9192024-04-17T17:11:14.394Z{"msg":"[1]R generation[0..12]: [7, 6, 4, 1, 7, 7, 2, 2, 5, 1, 5, 6]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.993989824Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"":"downstairs","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9202024-04-17T17:11:14.394Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.994014705Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"":"downstairs","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9212024-04-17T17:11:14.395Z{"msg":"[2]R flush_numbers[0..12]: [13, 10, 8, 1, 13, 13, 3, 3, 9, 1, 9, 10]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.994042197Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"":"downstairs","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9222024-04-17T17:11:14.395Z{"msg":"[2]R generation[0..12]: [7, 6, 4, 1, 7, 7, 2, 2, 5, 1, 5, 6]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.994065758Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"":"downstairs","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9232024-04-17T17:11:14.395Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.99409433Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"":"downstairs","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9242024-04-17T17:11:14.395Z{"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.994124052Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"":"downstairs","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9252024-04-17T17:11:14.395Z{"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.994144483Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"":"downstairs","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9262024-04-17T17:11:14.395Z{"msg":"Next flush: 14","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.994174885Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"":"downstairs","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9272024-04-17T17:11:14.395Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.994198006Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"":"downstairs"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
9282024-04-17T17:11:14.395Z,"":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9292024-04-17T17:11:14.395Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.994231008Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9302024-04-17T17:11:14.395Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.99425275Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"client":"0","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9312024-04-17T17:11:14.395Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.994283661Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"client":"1","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9322024-04-17T17:11:14.395Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.994307183Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"client":"2","":"downstairs","session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9332024-04-17T17:11:14.395Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.994333735Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9342024-04-17T17:11:14.395Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.994355126Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9352024-04-17T17:11:14.395Z{"msg":"1d0177b4-a5e4-4b73-ba43-b9bb0a33b52e is now active with session: 2f525993-b564-44d8-a2b2-de5fc6be6dcd","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.994382458Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9362024-04-17T17:11:14.396Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.994403639Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200,"session_id":"2f525993-b564-44d8-a2b2-de5fc6be6dcd"}
9372024-04-17T17:11:14.396Z{"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:12.9944296Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200}
9382024-04-17T17:11:14.396ZLoading write count information from file "/tmp/test_repair_verify.data"
9392024-04-17T17:11:14.396ZRun Repair workload
9402024-04-17T17:11:14.401Z01/30 Write block 398 len 36864 data: 2 2 2 2 2 2 2 2 2
9412024-04-17T17:11:14.401Z02/30 Read block 252 len 20480
9422024-04-17T17:11:14.401Z03/30 Read block 236 len 12288
9432024-04-17T17:11:14.401Z04/30 Read block 236 len 28672
9442024-04-17T17:11:14.401Z05/30 Flush
9452024-04-17T17:11:14.401Z06/30 Write block 568 len 24576 data: 2 2 2 2 2 2
9462024-04-17T17:11:14.401Z07/30 Read block 575 len 12288
9472024-04-17T17:11:14.402Z08/30 Read block 120 len 8192
9482024-04-17T17:11:14.402Z09/30 Read block 592 len 8192
9492024-04-17T17:11:14.402Z10/30 Write block 126 len 16384 data: 2 2 2 2
9502024-04-17T17:11:14.402Z11/30 Write block 383 len 32768 data: 2 2 2 2 2 2 2 2
9512024-04-17T17:11:14.402Z12/30 Read block 431 len 36864
9522024-04-17T17:11:14.406Z13/30 Write block 119 len 4096 data: 2
9532024-04-17T17:11:14.406Z14/30 Flush
9542024-04-17T17:11:14.406Z15/30 Flush
9552024-04-17T17:11:14.406Z16/30 Write block 407 len 32768 data: 2 2 2 2 3 3 3 3
9562024-04-17T17:11:14.406Z17/30 Write block 461 len 24576 data: 2 2 2 2 2 2
9572024-04-17T17:11:14.410Z18/30 Write block 244 len 40960 data: 3 3 4 4 3 3 4 5 5 4
9582024-04-17T17:11:14.411Z19/30 Read block 333 len 40960
9592024-04-17T17:11:14.411Z20/30 Write block 175 len 20480 data: 3 3 2 2 2
9602024-04-17T17:11:14.411Z21/30 Write block 433 len 4096 data: 2
9612024-04-17T17:11:14.415Z22/30 Write block 303 len 40960 data: 2 2 3 2 2 2 2 2 2 2
9622024-04-17T17:11:14.415Z23/30 Write block 25 len 8192 data: 2 2
9632024-04-17T17:11:14.415Z24/30 Read block 401 len 8192
9642024-04-17T17:11:14.415Z25/30 Read block 367 len 4096
9652024-04-17T17:11:14.415Z26/30 Write block 256 len 36864 data: 3 3 3 3 2 2 2 2 2
9662024-04-17T17:11:14.420Z27/30 Write block 54 len 32768 data: 2 2 2 2 2 2 2 2
9672024-04-17T17:11:14.420Z28/30 Write block 420 len 12288 data: 3 2 2
9682024-04-17T17:11:14.420Z29/30 Write block 29 len 28672 data: 3 3 3 3 3 3 3
9692024-04-17T17:11:14.420Z30/30 Read block 268 len 40960
9702024-04-17T17:11:14.420Z----------------------------------------------------------------
9712024-04-17T17:11:14.420Z Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:30
9722024-04-17T17:11:14.420ZStates: Active Active Active
9732024-04-17T17:11:14.420ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
9742024-04-17T17:11:14.420Z 1 Acked 1000 Write 9 Done Done Done false
9752024-04-17T17:11:14.420Z 2 Acked 1001 Read 5 Done Done Done false
9762024-04-17T17:11:14.420Z 3 Acked 1002 Read 3 Done Done Done false
9772024-04-17T17:11:14.421Z 4 Acked 1003 Read 7 Sent Done Done false
9782024-04-17T17:11:14.421Z 5 Acked 1004 Flush 0 Sent Done Done false
9792024-04-17T17:11:14.421Z 6 Acked 1005 Write 6 Sent Done Done false
9802024-04-17T17:11:14.421Z 7 Acked 1006 Read 3 Sent Done Done false
9812024-04-17T17:11:14.421Z 8 Acked 1007 Read 2 Sent Done Done false
9822024-04-17T17:11:14.421Z 9 Acked 1008 Read 2 Sent Done Done false
9832024-04-17T17:11:14.421Z 10 Acked 1009 Write 4 Sent Done Done false
9842024-04-17T17:11:14.421Z 11 Acked 1010 Write 8 Sent Done Done false
9852024-04-17T17:11:14.421Z 12 Acked 1011 Read 9 Sent Done Done false
9862024-04-17T17:11:14.421Z 13 Acked 1012 Write 1 Sent Done Done false
9872024-04-17T17:11:14.421Z 14 Acked 1013 Flush 0 Sent Done Done false
9882024-04-17T17:11:14.421Z 15 Acked 1014 Flush 0 Sent Done Done false
9892024-04-17T17:11:14.421Z 16 Acked 1015 Write 8 Sent Done Done false
9902024-04-17T17:11:14.421Z 17 Acked 1016 Write 6 Sent Done Done false
9912024-04-17T17:11:14.421Z 18 Acked 1017 Write 10 Sent Done Done false
9922024-04-17T17:11:14.421Z 19 Acked 1018 Read 10 Sent Done Done false
9932024-04-17T17:11:14.421Z 20 Acked 1019 Write 5 Sent Done Done false
9942024-04-17T17:11:14.421Z 21 Acked 1020 Write 1 Sent Done Done false
9952024-04-17T17:11:14.422Z 22 Acked 1021 Write 10 Sent Done Done false
9962024-04-17T17:11:14.422Z 23 Acked 1022 Write 2 Sent Done Done false
9972024-04-17T17:11:14.422Z 24 Acked 1023 Read 2 Sent Done Done false
9982024-04-17T17:11:14.422Z 25 Acked 1024 Read 1 Sent Done Done false
9992024-04-17T17:11:14.422Z 26 Acked 1025 Write 9 Sent Done Done false
10002024-04-17T17:11:14.422Z 27 Acked 1026 Write 8 Sent Done Done false
10012024-04-17T17:11:14.422Z 28 Acked 1027 Write 3 Sent Done Done false
10022024-04-17T17:11:14.422Z 29 Acked 1028 Write 7 Sent Done Done false
10032024-04-17T17:11:14.422Z 30 Acked 1029 Read 10 Sent Sent Done false
10042024-04-17T17:11:14.422Z STATES DS:0 DS:1 DS:2 TOTAL
10052024-04-17T17:11:14.422Z New 0 0 0 0
10062024-04-17T17:11:14.422Z Sent 27 1 0 28
10072024-04-17T17:11:14.422Z Done 3 29 30 62
10082024-04-17T17:11:14.422Z Skipped 0 0 0 0
10092024-04-17T17:11:14.422Z Error 0 0 0 0
10102024-04-17T17:11:14.422ZLast Flush: 0 1014 1014
10112024-04-17T17:11:14.422ZDownstairs last five completed:
10122024-04-17T17:11:14.422ZUpstairs last five completed: 30 29 28 27 26
10132024-04-17T17:11:14.422Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:11:13.025758739Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1200}
10142024-04-17T17:11:14.427ZWrote out file "/tmp/test_repair_verify.data"
10152024-04-17T17:11:14.427Z
10162024-04-17T17:11:14.427ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
10172024-04-17T17:11:14.444ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10182024-04-17T17:11:14.445Z 0 000-019 7 7 7 13 13 13 F F F
10192024-04-17T17:11:14.451Z 1 020-039 6 6 6 10 10 10 F T T <---
10202024-04-17T17:11:14.451Z 2 040-059 4 4 4 8 8 8 F T T <---
10212024-04-17T17:11:14.451Z 3 060-079 1 1 1 1 1 1 F T T <---
10222024-04-17T17:11:14.451Z 4 080-099 7 7 7 13 13 13 F F F
10232024-04-17T17:11:14.451Z 5 100-119 7 8 8 13 15 15 F F F <---
10242024-04-17T17:11:14.452Z 6 120-139 2 8 8 3 15 15 F F F <---
10252024-04-17T17:11:14.452Z 7 140-159 2 2 2 3 3 3 F F F
10262024-04-17T17:11:14.452Z 8 160-179 5 5 5 9 9 9 F T T <---
10272024-04-17T17:11:14.452Z 9 180-199 1 1 1 1 1 1 F F F
10282024-04-17T17:11:14.452Z 10 200-219 5 5 5 9 9 9 F F F
10292024-04-17T17:11:14.452Z 11 220-239 6 6 6 10 10 10 F F F
10302024-04-17T17:11:14.452Z 12 240-259 6 6 6 12 12 12 F T T <---
10312024-04-17T17:11:14.452Z 13 260-279 7 7 7 13 13 13 F T T <---
10322024-04-17T17:11:14.452Z 14 280-299 4 4 4 6 6 6 F F F
10332024-04-17T17:11:14.452Z 15 300-319 5 5 5 9 9 9 F T T <---
10342024-04-17T17:11:14.452Z 16 320-339 6 6 6 10 10 10 F F F
10352024-04-17T17:11:14.452Z 17 340-359 7 7 7 13 13 13 F F F
10362024-04-17T17:11:14.452Z 18 360-379 2 2 2 3 3 3 F F F
10372024-04-17T17:11:14.452Z 19 380-399 4 8 8 6 15 15 T F F <---
10382024-04-17T17:11:14.452Z 20 400-419 4 8 8 6 14 14 T T T <---
10392024-04-17T17:11:14.452Z 21 420-439 4 4 4 6 6 6 F T T <---
10402024-04-17T17:11:14.453Z 22 440-459 1 1 1 1 1 1 F F F
10412024-04-17T17:11:14.453Z 23 460-479 6 6 6 10 10 10 F T T <---
10422024-04-17T17:11:14.453Z 24 480-499 7 7 7 13 13 13 F F F
10432024-04-17T17:11:14.453Z 25 500-519 7 7 7 13 13 13 F F F
10442024-04-17T17:11:14.453Z 26 520-539 3 3 3 5 5 5 F F F
10452024-04-17T17:11:14.453Z 27 540-559 3 3 3 5 5 5 F F F
10462024-04-17T17:11:14.453Z 28 560-579 6 8 8 10 15 15 F F F <---
10472024-04-17T17:11:14.453Z 29 580-599 4 4 4 7 7 7 F F F
10482024-04-17T17:11:14.453ZMax gen: 8, Max flush: 15
10492024-04-17T17:11:14.453ZError: Difference in extent metadata found!
10502024-04-17T17:11:14.453ZA Difference in extent metadata is expected here
10512024-04-17T17:11:14.453ZOn loop 4
10522024-04-17T17:11:14.453Z
10532024-04-17T17:11:14.477ZVerifying data now
10542024-04-17T17:11:19.103ZNo change after verify
10552024-04-17T17:11:19.104ZLoop: 4 Downstairs dump after verify (and repair):
10562024-04-17T17:11:19.114ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10572024-04-17T17:11:19.114Z 0 000-019 7 7 7 13 13 13 F F F
10582024-04-17T17:11:19.120Z 1 020-039 9 9 9 16 16 16 F F F
10592024-04-17T17:11:19.120Z 2 040-059 9 9 9 16 16 16 F F F
10602024-04-17T17:11:19.120Z 3 060-079 9 9 9 16 16 16 F F F
10612024-04-17T17:11:19.120Z 4 080-099 7 7 7 13 13 13 F F F
10622024-04-17T17:11:19.120Z 5 100-119 8 8 8 15 15 15 F F F
10632024-04-17T17:11:19.121Z 6 120-139 8 8 8 15 15 15 F F F
10642024-04-17T17:11:19.121Z 7 140-159 2 2 2 3 3 3 F F F
10652024-04-17T17:11:19.121Z 8 160-179 9 9 9 16 16 16 F F F
10662024-04-17T17:11:19.121Z 9 180-199 1 1 1 1 1 1 F F F
10672024-04-17T17:11:19.122Z 10 200-219 5 5 5 9 9 9 F F F
10682024-04-17T17:11:19.122Z 11 220-239 6 6 6 10 10 10 F F F
10692024-04-17T17:11:19.122Z 12 240-259 9 9 9 16 16 16 F F F
10702024-04-17T17:11:19.122Z 13 260-279 9 9 9 16 16 16 F F F
10712024-04-17T17:11:19.122Z 14 280-299 4 4 4 6 6 6 F F F
10722024-04-17T17:11:19.123Z 15 300-319 9 9 9 16 16 16 F F F
10732024-04-17T17:11:19.123Z 16 320-339 6 6 6 10 10 10 F F F
10742024-04-17T17:11:19.123Z 17 340-359 7 7 7 13 13 13 F F F
10752024-04-17T17:11:19.123Z 18 360-379 2 2 2 3 3 3 F F F
10762024-04-17T17:11:19.123Z 19 380-399 8 8 8 15 15 15 F F F
10772024-04-17T17:11:19.123Z 20 400-419 9 9 9 16 16 16 F F F
10782024-04-17T17:11:19.124Z 21 420-439 9 9 9 16 16 16 F F F
10792024-04-17T17:11:19.124Z 22 440-459 1 1 1 1 1 1 F F F
10802024-04-17T17:11:19.124Z 23 460-479 9 9 9 16 16 16 F F F
10812024-04-17T17:11:19.124Z 24 480-499 7 7 7 13 13 13 F F F
10822024-04-17T17:11:19.124Z 25 500-519 7 7 7 13 13 13 F F F
10832024-04-17T17:11:19.124Z 26 520-539 3 3 3 5 5 5 F F F
10842024-04-17T17:11:19.124Z 27 540-559 3 3 3 5 5 5 F F F
10852024-04-17T17:11:19.125Z 28 560-579 8 8 8 15 15 15 F F F
10862024-04-17T17:11:19.125Z 29 580-599 4 4 4 7 7 7 F F F
10872024-04-17T17:11:19.125ZMax gen: 9, Max flush: 16
10882024-04-17T17:11:19.125Z
10892024-04-17T17:11:19.125ZBegin loop 5
10902024-04-17T17:11:19.125ZDownstairs to restart: 2
10912024-04-17T17:11:19.137Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.741487527Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208}
10922024-04-17T17:11:19.137Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.741810196Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208}
10932024-04-17T17:11:19.146Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.741840218Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208}
10942024-04-17T17:11:19.147ZCrucible runtime is spawned
10952024-04-17T17:11:19.147Z{"msg":"Crucible a22c6be4-5b38-4526-877e-268669cd998e has session id: 7732b01d-94d0-4921-8380-92998b4e73e8","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.741891491Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
10962024-04-17T17:11:19.147Z{"msg":"Upstairs opts: Upstairs UUID: a22c6be4-5b38-4526-877e-268669cd998e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.741958045Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
10972024-04-17T17:11:19.147Z{"msg":"Crucible stats registered with UUID: a22c6be4-5b38-4526-877e-268669cd998e","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.742007038Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
10982024-04-17T17:11:19.147Z{"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.742027289Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208}
10992024-04-17T17:11:19.147Z{"msg":"a22c6be4-5b38-4526-877e-268669cd998e active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.742070032Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11002024-04-17T17:11:19.147Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.742104254Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"client":"0","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11012024-04-17T17:11:19.147Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.742132626Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"client":"0","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11022024-04-17T17:11:19.147Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.742150277Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"client":"1","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11032024-04-17T17:11:19.147Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.742168368Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"client":"1","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11042024-04-17T17:11:19.147Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.742185359Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"client":"2","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11052024-04-17T17:11:19.147Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.74220426Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"client":"2","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11062024-04-17T17:11:19.148Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.742235232Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11072024-04-17T17:11:19.148Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.742267204Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11082024-04-17T17:11:19.148Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.742296596Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11092024-04-17T17:11:19.148Z{"msg":"ds_connection connected from Ok(127.0.0.1:42429)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.742429904Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11102024-04-17T17:11:19.148Z{"msg":"ds_connection connected from Ok(127.0.0.1:52765)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.742482517Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11112024-04-17T17:11:19.148Z{"msg":"ds_connection connected from Ok(127.0.0.1:56850)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.742512509Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11122024-04-17T17:11:19.148Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.743291896Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"client":"0","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11132024-04-17T17:11:19.148Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.743328318Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"client":"0","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11142024-04-17T17:11:19.148Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.743344419Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"client":"1","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11152024-04-17T17:11:19.148Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.743380191Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"client":"1","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11162024-04-17T17:11:19.148Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.743771485Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"client":"2","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11172024-04-17T17:11:19.148Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.743812498Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"client":"2","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11182024-04-17T17:11:19.148Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.743966917Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"client":"1","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11192024-04-17T17:11:19.148Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.744004559Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"client":"0","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11202024-04-17T17:11:19.149Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.744370181Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"client":"2","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11212024-04-17T17:11:19.149Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.744588625Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"client":"1","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11222024-04-17T17:11:19.149Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.744623557Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"client":"1","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11232024-04-17T17:11:19.149Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.744639308Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11242024-04-17T17:11:19.149Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.744654589Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"client":"0","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11252024-04-17T17:11:19.149Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.744686111Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"client":"0","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11262024-04-17T17:11:19.149Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.744704472Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11272024-04-17T17:11:19.149Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.744863691Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"client":"2","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11282024-04-17T17:11:19.149Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.744919065Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"client":"2","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11292024-04-17T17:11:19.149Z{"msg":"[0]R flush_numbers[0..12]: [13, 16, 16, 16, 13, 15, 15, 3, 16, 1, 9, 10]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.744935146Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"":"downstairs","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11302024-04-17T17:11:19.149Z{"msg":"[0]R generation[0..12]: [7, 9, 9, 9, 7, 8, 8, 2, 9, 1, 5, 6]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.744949447Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"":"downstairs","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11312024-04-17T17:11:19.149Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.744971338Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"":"downstairs","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11322024-04-17T17:11:19.150Z{"msg":"[1]R flush_numbers[0..12]: [13, 16, 16, 16, 13, 15, 15, 3, 16, 1, 9, 10]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.744985349Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"":"downstairs","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11332024-04-17T17:11:19.150Z{"msg":"[1]R generation[0..12]: [7, 9, 9, 9, 7, 8, 8, 2, 9, 1, 5, 6]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.74500624Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"":"downstairs","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11342024-04-17T17:11:19.150Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.745021911Z","Wait for a query_work_queue command to finish before sending IO
11352024-04-17T17:11:19.150Zhostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"":"downstairs","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11362024-04-17T17:11:19.150Z{"msg":"[2]R flush_numbers[0..12]: [13, 16, 16, 16, 13, 15, 15, 3, 16, 1, 9, 10]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.745039722Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"":"downstairs","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11372024-04-17T17:11:19.150Z{"msg":"[2]R generation[0..12]: [7, 9, 9, 9, 7, 8, 8, 2, 9, 1, 5, 6]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.745053333Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"":"downstairs","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11382024-04-17T17:11:19.150Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.745074134Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"":"downstairs","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11392024-04-17T17:11:19.150Z{"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.745087255Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"":"downstairs","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11402024-04-17T17:11:19.150Z{"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.745101056Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"":"downstairs","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11412024-04-17T17:11:19.150Z{"msg":"Next flush: 17","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.745113706Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"":"downstairs","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11422024-04-17T17:11:19.150Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.745130727Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"":"downstairs","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11432024-04-17T17:11:19.150Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.745160829Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11442024-04-17T17:11:19.151Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.745179581Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"client":"0","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11452024-04-17T17:11:19.151Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.745196311Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"client":"1","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11462024-04-17T17:11:19.151Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.745219473Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"client":"2","":"downstairs","session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11472024-04-17T17:11:19.151Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.745233704Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11482024-04-17T17:11:19.151Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.745245925Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11492024-04-17T17:11:19.151Z{"msg":"a22c6be4-5b38-4526-877e-268669cd998e is now active with session: 7732b01d-94d0-4921-8380-92998b4e73e8","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.745265976Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11502024-04-17T17:11:19.151Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:17.745300108Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208,"session_id":"7732b01d-94d0-4921-8380-92998b4e73e8"}
11512024-04-17T17:11:19.151Z{"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
11522024-04-17T17:11:19.151Z,"time":"2024-04-17T17:11:17.745325119Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208}
11532024-04-17T17:11:19.151ZLoading write count information from file "/tmp/test_repair_verify.data"
11542024-04-17T17:11:19.151ZRun Repair workload
11552024-04-17T17:11:19.151Z01/30 Write block 356 len 36864 data: 2 2 2 2 2 2 2 2 2
11562024-04-17T17:11:19.151Z02/30 Read block 403 len 28672
11572024-04-17T17:11:19.161Z03/30 Read block 509 len 4096
11582024-04-17T17:11:19.161Z04/30 Write block 31 len 28672 data: 4 4 4 4 4 2 2
11592024-04-17T17:11:19.161Z05/30 Read block 25 len 16384
11602024-04-17T17:11:19.161Z06/30 Flush
11612024-04-17T17:11:19.161Z07/30 Write block 552 len 32768 data: 2 2 2 2 2 2 2 2
11622024-04-17T17:11:19.161Z08/30 Read block 557 len 36864
11632024-04-17T17:11:19.161Z09/30 Read block 487 len 4096
11642024-04-17T17:11:19.162Z10/30 Write block 403 len 20480 data: 3 3 3 3 3
11652024-04-17T17:11:19.162Z11/30 Read block 232 len 28672
11662024-04-17T17:11:19.162Z12/30 Write block 590 len 12288 data: 2 2 2
11672024-04-17T17:11:19.162Z13/30 Read block 201 len 24576
11682024-04-17T17:11:19.162Z14/30 Flush
11692024-04-17T17:11:19.162Z15/30 Write block 539 len 28672 data: 3 3 3 3 3 2 2
11702024-04-17T17:11:19.162Z16/30 Read block 395 len 36864
11712024-04-17T17:11:19.167Z17/30 Write block 558 len 20480 data: 3 3 2 2 2
11722024-04-17T17:11:19.168Z18/30 Read block 415 len 40960
11732024-04-17T17:11:19.168Z19/30 Read block 505 len 4096
11742024-04-17T17:11:19.168Z20/30 Write block 218 len 4096 data: 2
11752024-04-17T17:11:19.168Z21/30 Write block 407 len 12288 data: 4 3 3
11762024-04-17T17:11:19.168Z22/30 Read block 308 len 40960
11772024-04-17T17:11:19.168Z23/30 Read block 284 len 40960
11782024-04-17T17:11:19.168Z24/30 Flush
11792024-04-17T17:11:19.184Z25/30 Flush
11802024-04-17T17:11:19.184Z26/30 Write block 36 len 40960 data: 3 3 2 2 3 2 2 2 2 3
11812024-04-17T17:11:19.184Z27/30 Read block 570 len 36864
11822024-04-17T17:11:19.184Z28/30 Read block 592 len 24576
11832024-04-17T17:11:19.184Z29/30 Read block 196 len 12288
11842024-04-17T17:11:19.184Z30/30 Read block 16 len 16384
11852024-04-17T17:11:19.184Z----------------------------------------------------------------
11862024-04-17T17:11:19.185Z Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:30
11872024-04-17T17:11:19.185ZStates: Active Active Active
11882024-04-17T17:11:19.185ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
11892024-04-17T17:11:19.185Z 1 Acked 1000 Write 9 Done Done Done false
11902024-04-17T17:11:19.185Z 2 Acked 1001 Read 7 Done Done Done false
11912024-04-17T17:11:19.185Z 3 Acked 1002 Read 1 Done Done Done false
11922024-04-17T17:11:19.185Z 4 Acked 1003 Write 7 Done Done Done false
11932024-04-17T17:11:19.185Z 5 Acked 1004 Read 4 Done Done Done false
11942024-04-17T17:11:19.185Z 6 Acked 1005 Flush 0 Done Done Sent false
11952024-04-17T17:11:19.186Z 7 Acked 1006 Write 8 Done Done Sent false
11962024-04-17T17:11:19.186Z 8 Acked 1007 Read 9 Done Done Sent false
11972024-04-17T17:11:19.186Z 9 Acked 1008 Read 1 Done Done Sent false
11982024-04-17T17:11:19.186Z 10 Acked 1009 Write 5 Done Done Sent false
11992024-04-17T17:11:19.186Z 11 Acked 1010 Read 7 Done Done Sent false
12002024-04-17T17:11:19.186Z 12 Acked 1011 Write 3 Done Done Sent false
12012024-04-17T17:11:19.186Z 13 Acked 1012 Read 6 Done Done Sent false
12022024-04-17T17:11:19.187Z 14 Acked 1013 Flush 0 Done Done Sent false
12032024-04-17T17:11:19.187Z 15 Acked 1014 Write 7 Done Done Sent false
12042024-04-17T17:11:19.187Z 16 Acked 1015 Read 9 Done Done Sent false
12052024-04-17T17:11:19.187Z 17 Acked 1016 Write 5 Done Done Sent false
12062024-04-17T17:11:19.187Z 18 Acked 1017 Read 10 Done Done Sent false
12072024-04-17T17:11:19.187Z 19 Acked 1018 Read 1 Done Done Sent false
12082024-04-17T17:11:19.187Z 20 Acked 1019 Write 1 Done Done Sent false
12092024-04-17T17:11:19.188Z 21 Acked 1020 Write 3 Done Done Sent false
12102024-04-17T17:11:19.188Z 22 Acked 1021 Read 10 Done Done Sent false
12112024-04-17T17:11:19.188Z 23 Acked 1022 Read 10 Done Done Sent false
12122024-04-17T17:11:19.188Z 24 Acked 1023 Flush 0 Done Done Sent false
12132024-04-17T17:11:19.188Z 25 Acked 1024 Flush 0 Done Done Sent false
12142024-04-17T17:11:19.188Z 26 Acked 1025 Write 10 Done Done Sent false
12152024-04-17T17:11:19.189Z 27 Acked 1026 Read 9 Done Done Sent false
12162024-04-17T17:11:19.189Z 28 Acked 1027 Read 6 Done Done Sent false
12172024-04-17T17:11:19.189Z 29 Acked 1028 Read 3 Done Done Sent false
12182024-04-17T17:11:19.189Z 30 Acked 1029 Read 4 Done Done Sent false
12192024-04-17T17:11:19.189Z STATES DS:0 DS:1 DS:2 TOTAL
12202024-04-17T17:11:19.189Z New 0 0 0 0
12212024-04-17T17:11:19.189Z Sent 0 0 25 25
12222024-04-17T17:11:19.190Z Done 30 30 5 65
12232024-04-17T17:11:19.190Z Skipped 0 0 0 0
12242024-04-17T17:11:19.190Z Error 0 0 0 0
12252024-04-17T17:11:19.190ZLast Flush: 1024 1024 0
12262024-04-17T17:11:19.190ZDownstairs last five completed:
12272024-04-17T17:11:19.190ZUpstairs last five completed: 30 29 28 27 26
12282024-04-17T17:11:19.190Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:11:17.779251045Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1208}
12292024-04-17T17:11:19.191ZWrote out file "/tmp/test_repair_verify.data"
12302024-04-17T17:11:19.198Z
12312024-04-17T17:11:19.198ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
12322024-04-17T17:11:19.207ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12332024-04-17T17:11:19.207Z 0 000-019 7 7 7 13 13 13 F F F
12342024-04-17T17:11:19.208Z 1 020-039 10 10 9 17 17 16 T T T <---
12352024-04-17T17:11:19.208Z 2 040-059 9 9 9 16 16 16 T T F <---
12362024-04-17T17:11:19.208Z 3 060-079 9 9 9 16 16 16 F F F
12372024-04-17T17:11:19.208Z 4 080-099 7 7 7 13 13 13 F F F
12382024-04-17T17:11:19.208Z 5 100-119 8 8 8 15 15 15 F F F
12392024-04-17T17:11:19.208Z 6 120-139 8 8 8 15 15 15 F F F
12402024-04-17T17:11:19.209Z 7 140-159 2 2 2 3 3 3 F F F
12412024-04-17T17:11:19.209Z 8 160-179 9 9 9 16 16 16 F F F
12422024-04-17T17:11:19.209Z 9 180-199 1 1 1 1 1 1 F F F
12432024-04-17T17:11:19.209Z 10 200-219 10 10 5 19 19 9 F F F <---
12442024-04-17T17:11:19.209Z 11 220-239 6 6 6 10 10 10 F F F
12452024-04-17T17:11:19.209Z 12 240-259 9 9 9 16 16 16 F F F
12462024-04-17T17:11:19.210Z 13 260-279 9 9 9 16 16 16 F F F
12472024-04-17T17:11:19.210Z 14 280-299 4 4 4 6 6 6 F F F
12482024-04-17T17:11:19.210Z 15 300-319 9 9 9 16 16 16 F F F
12492024-04-17T17:11:19.210Z 16 320-339 6 6 6 10 10 10 F F F
12502024-04-17T17:11:19.210Z 17 340-359 10 10 7 17 17 13 F F T <---
12512024-04-17T17:11:19.210Z 18 360-379 10 10 2 17 17 3 F F T <---
12522024-04-17T17:11:19.211Z 19 380-399 8 8 8 15 15 15 F F F
12532024-04-17T17:11:19.211Z 20 400-419 10 10 9 19 19 16 F F F <---
12542024-04-17T17:11:19.211Z 21 420-439 9 9 9 16 16 16 F F F
12552024-04-17T17:11:19.211Z 22 440-459 1 1 1 1 1 1 F F F
12562024-04-17T17:11:19.211Z 23 460-479 9 9 9 16 16 16 F F F
12572024-04-17T17:11:19.211Z 24 480-499 7 7 7 13 13 13 F F F
12582024-04-17T17:11:19.212Z 25 500-519 7 7 7 13 13 13 F F F
12592024-04-17T17:11:19.212Z 26 520-539 10 10 3 19 19 5 F F F <---
12602024-04-17T17:11:19.212Z 27 540-559 10 10 3 19 19 5 F F F <---
12612024-04-17T17:11:19.212Z 28 560-579 10 10 8 19 19 15 F F F <---
12622024-04-17T17:11:19.212Z 29 580-599 10 10 4 18 18 7 F F F <---
12632024-04-17T17:11:19.213ZMax gen: 10, Max flush: 19
12642024-04-17T17:11:19.213ZError: Difference in extent metadata found!
12652024-04-17T17:11:19.213ZA Difference in extent metadata is expected here
12662024-04-17T17:11:19.213ZOn loop 5
12672024-04-17T17:11:19.213Z
12682024-04-17T17:11:19.213ZVerifying data now
12692024-04-17T17:11:33.675ZNo change after verify
12702024-04-17T17:11:33.675ZLoop: 5 Downstairs dump after verify (and repair):
12712024-04-17T17:11:33.688ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12722024-04-17T17:11:33.688Z 0 000-019 7 7 7 13 13 13 F F F
12732024-04-17T17:11:33.692Z 1 020-039 11 11 11 20 20 20 F F F
12742024-04-17T17:11:33.693Z 2 040-059 11 11 11 20 20 20 F F F
12752024-04-17T17:11:33.693Z 3 060-079 9 9 9 16 16 16 F F F
12762024-04-17T17:11:33.693Z 4 080-099 7 7 7 13 13 13 F F F
12772024-04-17T17:11:33.693Z 5 100-119 8 8 8 15 15 15 F F F
12782024-04-17T17:11:33.693Z 6 120-139 8 8 8 15 15 15 F F F
12792024-04-17T17:11:33.693Z 7 140-159 2 2 2 3 3 3 F F F
12802024-04-17T17:11:33.693Z 8 160-179 9 9 9 16 16 16 F F F
12812024-04-17T17:11:33.693Z 9 180-199 1 1 1 1 1 1 F F F
12822024-04-17T17:11:33.693Z 10 200-219 10 10 10 19 19 19 F F F
12832024-04-17T17:11:33.693Z 11 220-239 6 6 6 10 10 10 F F F
12842024-04-17T17:11:33.693Z 12 240-259 9 9 9 16 16 16 F F F
12852024-04-17T17:11:33.693Z 13 260-279 9 9 9 16 16 16 F F F
12862024-04-17T17:11:33.693Z 14 280-299 4 4 4 6 6 6 F F F
12872024-04-17T17:11:33.693Z 15 300-319 9 9 9 16 16 16 F F F
12882024-04-17T17:11:33.693Z 16 320-339 6 6 6 10 10 10 F F F
12892024-04-17T17:11:33.693Z 17 340-359 10 10 10 17 17 17 F F F
12902024-04-17T17:11:33.693Z 18 360-379 10 10 10 17 17 17 F F F
12912024-04-17T17:11:33.693Z 19 380-399 8 8 8 15 15 15 F F F
12922024-04-17T17:11:33.693Z 20 400-419 10 10 10 19 19 19 F F F
12932024-04-17T17:11:33.693Z 21 420-439 9 9 9 16 16 16 F F F
12942024-04-17T17:11:33.693Z 22 440-459 1 1 1 1 1 1 F F F
12952024-04-17T17:11:33.693Z 23 460-479 9 9 9 16 16 16 F F F
12962024-04-17T17:11:33.693Z 24 480-499 7 7 7 13 13 13 F F F
12972024-04-17T17:11:33.694Z 25 500-519 7 7 7 13 13 13 F F F
12982024-04-17T17:11:33.694Z 26 520-539 10 10 10 19 19 19 F F F
12992024-04-17T17:11:33.694Z 27 540-559 10 10 10 19 19 19 F F F
13002024-04-17T17:11:33.694Z 28 560-579 10 10 10 19 19 19 F F F
13012024-04-17T17:11:33.694Z 29 580-599 10 10 10 18 18 18 F F F
13022024-04-17T17:11:33.694ZMax gen: 11, Max flush: 20
13032024-04-17T17:11:33.694Z
13042024-04-17T17:11:33.694ZBegin loop 6
13052024-04-17T17:11:33.694ZDownstairs to restart: 0
13062024-04-17T17:11:33.711Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.316564833Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216}
13072024-04-17T17:11:33.711Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.316832229Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216}
13082024-04-17T17:11:33.711Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.31685001Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216}
13092024-04-17T17:11:33.717Z{"msg":"Crucible 81080fa5-b5a8-48f7-8fa9-35c5da2eb2d7 has session id: 98ba381d-e561-45a4-9617-f92ce32ad1f7","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.316880332Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13102024-04-17T17:11:33.717Z{"msg":"Crucible runtime is spawned
13112024-04-17T17:11:33.717ZUpstairs opts: Upstairs UUID: 81080fa5-b5a8-48f7-8fa9-35c5da2eb2d7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.316945096Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13122024-04-17T17:11:33.717Z{"msg":"Crucible stats registered with UUID: 81080fa5-b5a8-48f7-8fa9-35c5da2eb2d7","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.316982708Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13132024-04-17T17:11:33.718Z{"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.317104325Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216}
13142024-04-17T17:11:33.718Z{"msg":"81080fa5-b5a8-48f7-8fa9-35c5da2eb2d7 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.317152748Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13152024-04-17T17:11:33.718Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.31717886Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"client":"0","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13162024-04-17T17:11:33.718Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.317215272Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"client":"0","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13172024-04-17T17:11:33.718Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.317249974Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"client":"1","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13182024-04-17T17:11:33.718Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.317277326Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"client":"1","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13192024-04-17T17:11:33.718Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.317321018Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"client":"2","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13202024-04-17T17:11:33.718Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.31734678Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"client":"2","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13212024-04-17T17:11:33.718Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.317388242Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13222024-04-17T17:11:33.718Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.317431315Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13232024-04-17T17:11:33.718Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.317459307Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13242024-04-17T17:11:33.718Z{"msg":"ds_connection connected from Ok(127.0.0.1:33058)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.317563093Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13252024-04-17T17:11:33.718Z{"msg":"ds_connection connected from Ok(127.0.0.1:60357)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.317618587Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13262024-04-17T17:11:33.718Z{"msg":"ds_connection connected from Ok(127.0.0.1:59666)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.317648418Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13272024-04-17T17:11:33.718Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.31833944Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"client":"2","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13282024-04-17T17:11:33.718Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.318374142Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"client":"2","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13292024-04-17T17:11:33.718Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.318487729Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"client":"0","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13302024-04-17T17:11:33.718Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.318531862Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"client":"0","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13312024-04-17T17:11:33.718Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.318612357Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"client":"1","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13322024-04-17T17:11:33.718Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.318650489Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"client":"1","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13332024-04-17T17:11:33.718Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.319075545Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"client":"2","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13342024-04-17T17:11:33.718Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.319124868Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"client":"0","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13352024-04-17T17:11:33.718Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.319334231Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"client":"1","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13362024-04-17T17:11:33.719Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.319640359Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"client":"2","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13372024-04-17T17:11:33.719Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.319668561Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"client":"2","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13382024-04-17T17:11:33.719Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.319683332Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13392024-04-17T17:11:33.719Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.319695602Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"client":"0","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13402024-04-17T17:11:33.719Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.319727904Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"client":"0","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13412024-04-17T17:11:33.719Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.319752356Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13422024-04-17T17:11:33.719Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.3198211Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"client":"1","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13432024-04-17T17:11:33.719Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.319857282Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"client":"1","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13442024-04-17T17:11:33.719Z{"msg":"[0]R flush_numbers[0..12]: [13, 20, 20, 16, 13, 15, 15, 3, 16, 1, 19, 10]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.319889294Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"":"downstairs","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13452024-04-17T17:11:33.719Z{"msg":"[0]R generation[0..12]: [7, 11, 11, 9, 7, 8, 8, 2, 9, 1, 10, 6]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.319922816Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"":"downstairs","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13462024-04-17T17:11:33.719Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.319956098Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"":"downstairs","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13472024-04-17T17:11:33.719ZWait for a query_work_queue command to finish before sending IO
13482024-04-17T17:11:33.719Z{"msg":"[1]R flush_numbers[0..12]: [13, 20, 20, 16, 13, 15, 15, 3, 16, 1, 19, 10]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.31999217Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"":"downstairs","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13492024-04-17T17:11:33.719Z{"msg":"[1]R generation[0..12]: [7, 11, 11, 9, 7, 8, 8, 2, 9, 1, 10, 6]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.320024722Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"":"downstairs","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13502024-04-17T17:11:33.719Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.320051944Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"":"downstairs","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13512024-04-17T17:11:33.719Z{"msg":"[2]R flush_numbers[0..12]: [13, 20, 20, 16, 13, 15, 15, 3, 16, 1, 19, 10]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.320066615Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"":"downstairs","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13522024-04-17T17:11:33.719Z{"msg":"[2]R generation[0..12]: [7, 11, 11, 9, 7, 8, 8, 2, 9, 1, 10, 6]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.320086876Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"":"downstairs","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13532024-04-17T17:11:33.719Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.320101357Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"":"downstairs","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13542024-04-17T17:11:33.719Z{"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.320115628Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"":"downstairs","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13552024-04-17T17:11:33.719Z{"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.320128929Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"":"downstairs","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13562024-04-17T17:11:33.719Z{"msg":"Next flush: 21","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.32014858Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"":"downstairs","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13572024-04-17T17:11:33.719Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.320161711Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"":"downstairs","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13582024-04-17T17:11:33.719Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.320181282Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13592024-04-17T17:11:33.720Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.320194052Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"client":"0","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13602024-04-17T17:11:33.720Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.320215134Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"client":"1","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13612024-04-17T17:11:33.720Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.320228555Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"client":"2","":"downstairs","session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13622024-04-17T17:11:33.720Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.320248126Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13632024-04-17T17:11:33.720Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.320260456Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13642024-04-17T17:11:33.720Z{"msg":"81080fa5-b5a8-48f7-8fa9-35c5da2eb2d7 is now active with session: 98ba381d-e561-45a4-9617-f92ce32ad1f7","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.320279568Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13652024-04-17T17:11:33.720Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.320291569Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216,"session_id":"98ba381d-e561-45a4-9617-f92ce32ad1f7"}
13662024-04-17T17:11:33.720Z{"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:32.3203101Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216}
13672024-04-17T17:11:33.720ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
13682024-04-17T17:11:33.720ZLoading write count information from file "/tmp/test_repair_verify.data"
13692024-04-17T17:11:33.720ZRun Repair workload
13702024-04-17T17:11:33.720Z01/30 Write block 235 len 4096 data: 2
13712024-04-17T17:11:33.724Z02/30 Read block 191 len 40960
13722024-04-17T17:11:33.724Z03/30 Read block 440 len 16384
13732024-04-17T17:11:33.724Z04/30 Read block 38 len 16384
13742024-04-17T17:11:33.724Z05/30 Read block 185 len 12288
13752024-04-17T17:11:33.725Z06/30 Write block 518 len 4096 data: 2
13762024-04-17T17:11:33.725Z07/30 Write block 564 len 16384 data: 2 2 2 2
13772024-04-17T17:11:33.725Z08/30 Write block 469 len 28672 data: 3 3 3 3 3 3 3
13782024-04-17T17:11:33.725Z09/30 Read block 547 len 12288
13792024-04-17T17:11:33.725Z10/30 Write block 25 len 8192 data: 3 3
13802024-04-17T17:11:33.725Z11/30 Read block 280 len 12288
13812024-04-17T17:11:33.725Z12/30 Read block 212 len 20480
13822024-04-17T17:11:33.725Z13/30 Read block 284 len 4096
13832024-04-17T17:11:33.725Z14/30 Read block 152 len 40960
13842024-04-17T17:11:33.729Z15/30 Read block 164 len 12288
13852024-04-17T17:11:33.729Z16/30 Read block 541 len 4096
13862024-04-17T17:11:33.729Z17/30 Write block 275 len 16384 data: 3 3 2 2
13872024-04-17T17:11:33.729Z18/30 Flush
13882024-04-17T17:11:33.732Z19/30 Flush
13892024-04-17T17:11:33.732Z20/30 Write block 458 len 28672 data: 2 2 2 3 3 3 3
13902024-04-17T17:11:33.732Z21/30 Read block 458 len 20480
13912024-04-17T17:11:33.732Z22/30 Read block 78 len 20480
13922024-04-17T17:11:33.737Z23/30 Write block 171 len 12288 data: 2 2 2
13932024-04-17T17:11:33.737Z24/30 Write block 17 len 32768 data: 3 3 2 2 2 2 2 2
13942024-04-17T17:11:33.737Z25/30 Read block 399 len 36864
13952024-04-17T17:11:33.737Z26/30 Flush
13962024-04-17T17:11:33.741Z27/30 Write block 428 len 8192 data: 2 2
13972024-04-17T17:11:33.745Z28/30 Write block 484 len 36864 data: 3 3 3 2 2 2 2 2 2
13982024-04-17T17:11:33.745Z29/30 Write block 478 len 24576 data: 3 4 4 3 3 3
13992024-04-17T17:11:33.745Z30/30 Write block 119 len 28672 data: 3 2 2 2 2 2 2
14002024-04-17T17:11:33.745Z----------------------------------------------------------------
14012024-04-17T17:11:33.745Z Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30
14022024-04-17T17:11:33.745ZStates: Active Active Active
14032024-04-17T17:11:33.745ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
14042024-04-17T17:11:33.745Z 1 Acked 1000 Write 1 Done Done Done false
14052024-04-17T17:11:33.745Z 2 Acked 1001 Read 10 Done Done Done false
14062024-04-17T17:11:33.745Z 3 Acked 1002 Read 4 Done Done Done false
14072024-04-17T17:11:33.745Z 4 Acked 1003 Read 4 Done Done Done false
14082024-04-17T17:11:33.745Z 5 Acked 1004 Read 3 Done Done Done false
14092024-04-17T17:11:33.745Z 6 Acked 1005 Write 1 Sent Done Done false
14102024-04-17T17:11:33.745Z 7 Acked 1006 Write 4 Sent Done Done false
14112024-04-17T17:11:33.745Z 8 Acked 1007 Write 7 Sent Done Done false
14122024-04-17T17:11:33.745Z 9 Acked 1008 Read 3 Sent Done Done false
14132024-04-17T17:11:33.745Z 10 Acked 1009 Write 2 Sent Done Done false
14142024-04-17T17:11:33.745Z 11 Acked 1010 Read 3 Sent Done Done false
14152024-04-17T17:11:33.745Z 12 Acked 1011 Read 5 Sent Done Done false
14162024-04-17T17:11:33.746Z 13 Acked 1012 Read 1 Sent Done Done false
14172024-04-17T17:11:33.746Z 14 Acked 1013 Read 10 Sent Done Done false
14182024-04-17T17:11:33.746Z 15 Acked 1014 Read 3 Sent Done Done false
14192024-04-17T17:11:33.746Z 16 Acked 1015 Read 1 Sent Done Done false
14202024-04-17T17:11:33.746Z 17 Acked 1016 Write 4 Sent Done Done false
14212024-04-17T17:11:33.746Z 18 Acked 1017 Flush 0 Sent Done Done false
14222024-04-17T17:11:33.746Z 19 Acked 1018 Flush 0 Sent Done Done false
14232024-04-17T17:11:33.746Z 20 Acked 1019 Write 7 Sent Done Done false
14242024-04-17T17:11:33.746Z 21 Acked 1020 Read 5 Sent Done Done false
14252024-04-17T17:11:33.746Z 22 Acked 1021 Read 5 Sent Done Done false
14262024-04-17T17:11:33.746Z 23 Acked 1022 Write 3 Sent Done Done false
14272024-04-17T17:11:33.746Z 24 Acked 1023 Write 8 Sent Done Done false
14282024-04-17T17:11:33.746Z 25 Acked 1024 Read 9 Sent Done Done false
14292024-04-17T17:11:33.746Z 26 Acked 1025 Flush 0 Sent Done Done false
14302024-04-17T17:11:33.746Z 27 Acked 1026 Write 2 Sent Done Done false
14312024-04-17T17:11:33.746Z 28 Acked 1027 Write 9 Sent Done Done false
14322024-04-17T17:11:33.746Z 29 Acked 1028 Write 6 Sent Done Done false
14332024-04-17T17:11:33.746Z 30 Acked 1029 Write 7 Sent Sent Sent false
14342024-04-17T17:11:33.746Z STATES DS:0 DS:1 DS:2 TOTAL
14352024-04-17T17:11:33.746Z New 0 0 0 0
14362024-04-17T17:11:33.746Z Sent 25 1 1 27
14372024-04-17T17:11:33.746Z Done 5 29 29 63
14382024-04-17T17:11:33.746Z Skipped 0 0 0 0
14392024-04-17T17:11:33.746Z Error 0 0 0 0
14402024-04-17T17:11:33.747ZLast Flush: 0 1025 1025
14412024-04-17T17:11:33.747ZDownstairs last five completed:
14422024-04-17T17:11:33.747ZUpstairs last five completed: 30 29 28 27 26
14432024-04-17T17:11:33.747Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:11:32.350734374Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1216}
14442024-04-17T17:11:33.747ZWrote out file "/tmp/test_repair_verify.data"
14452024-04-17T17:11:33.750Z
14462024-04-17T17:11:33.754ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
14472024-04-17T17:11:33.767ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14482024-04-17T17:11:33.767Z 0 000-019 7 12 12 13 23 23 F F F <---
14492024-04-17T17:11:33.768Z 1 020-039 11 12 12 20 23 23 F F F <---
14502024-04-17T17:11:33.768Z 2 040-059 11 11 11 20 20 20 F F F
14512024-04-17T17:11:33.768Z 3 060-079 9 9 9 16 16 16 F F F
14522024-04-17T17:11:33.768Z 4 080-099 7 7 7 13 13 13 F F F
14532024-04-17T17:11:33.768Z 5 100-119 8 8 8 15 15 15 F T T <---
14542024-04-17T17:11:33.768Z 6 120-139 8 8 8 15 15 15 F T T <---
14552024-04-17T17:11:33.772Z 7 140-159 2 2 2 3 3 3 F F F
14562024-04-17T17:11:33.772Z 8 160-179 9 12 12 16 23 23 F F F <---
14572024-04-17T17:11:33.772Z 9 180-199 1 1 1 1 1 1 F F F
14582024-04-17T17:11:33.772Z 10 200-219 10 10 10 19 19 19 F F F
14592024-04-17T17:11:33.772Z 11 220-239 6 12 12 10 21 21 T F F <---
14602024-04-17T17:11:33.772Z 12 240-259 9 9 9 16 16 16 F F F
14612024-04-17T17:11:33.772Z 13 260-279 9 12 12 16 21 21 F F F <---
14622024-04-17T17:11:33.772Z 14 280-299 4 4 4 6 6 6 F F F
14632024-04-17T17:11:33.772Z 15 300-319 9 9 9 16 16 16 F F F
14642024-04-17T17:11:33.772Z 16 320-339 6 6 6 10 10 10 F F F
14652024-04-17T17:11:33.772Z 17 340-359 10 10 10 17 17 17 F F F
14662024-04-17T17:11:33.772Z 18 360-379 10 10 10 17 17 17 F F F
14672024-04-17T17:11:33.772Z 19 380-399 8 8 8 15 15 15 F F F
14682024-04-17T17:11:33.772Z 20 400-419 10 10 10 19 19 19 F F F
14692024-04-17T17:11:33.772Z 21 420-439 9 9 9 16 16 16 F T T <---
14702024-04-17T17:11:33.772Z 22 440-459 1 12 12 1 23 23 F F F <---
14712024-04-17T17:11:33.772Z 23 460-479 9 12 12 16 23 23 F T T <---
14722024-04-17T17:11:33.772Z 24 480-499 7 7 7 13 13 13 F T T <---
14732024-04-17T17:11:33.772Z 25 500-519 7 12 12 13 21 21 F F F <---
14742024-04-17T17:11:33.772Z 26 520-539 10 10 10 19 19 19 F F F
14752024-04-17T17:11:33.772Z 27 540-559 10 10 10 19 19 19 F F F
14762024-04-17T17:11:33.773Z 28 560-579 10 12 12 19 21 21 F F F <---
14772024-04-17T17:11:33.773Z 29 580-599 10 10 10 18 18 18 F F F
14782024-04-17T17:11:33.773ZMax gen: 12, Max flush: 23
14792024-04-17T17:11:33.773ZError: Difference in extent metadata found!
14802024-04-17T17:11:33.773ZA Difference in extent metadata is expected here
14812024-04-17T17:11:33.773ZOn loop 6
14822024-04-17T17:11:33.773Z
14832024-04-17T17:11:33.777ZVerifying data now
14842024-04-17T17:11:38.382ZNo change after verify
14852024-04-17T17:11:38.382ZLoop: 6 Downstairs dump after verify (and repair):
14862024-04-17T17:11:38.406ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14872024-04-17T17:11:38.407Z 0 000-019 12 12 12 23 23 23 F F F
14882024-04-17T17:11:38.407Z 1 020-039 12 12 12 23 23 23 F F F
14892024-04-17T17:11:38.407Z 2 040-059 11 11 11 20 20 20 F F F
14902024-04-17T17:11:38.407Z 3 060-079 9 9 9 16 16 16 F F F
14912024-04-17T17:11:38.408Z 4 080-099 7 7 7 13 13 13 F F F
14922024-04-17T17:11:38.408Z 5 100-119 13 13 13 24 24 24 F F F
14932024-04-17T17:11:38.408Z 6 120-139 13 13 13 24 24 24 F F F
14942024-04-17T17:11:38.408Z 7 140-159 2 2 2 3 3 3 F F F
14952024-04-17T17:11:38.408Z 8 160-179 12 12 12 23 23 23 F F F
14962024-04-17T17:11:38.408Z 9 180-199 1 1 1 1 1 1 F F F
14972024-04-17T17:11:38.408Z 10 200-219 10 10 10 19 19 19 F F F
14982024-04-17T17:11:38.408Z 11 220-239 12 12 12 21 21 21 F F F
14992024-04-17T17:11:38.408Z 12 240-259 9 9 9 16 16 16 F F F
15002024-04-17T17:11:38.408Z 13 260-279 12 12 12 21 21 21 F F F
15012024-04-17T17:11:38.409Z 14 280-299 4 4 4 6 6 6 F F F
15022024-04-17T17:11:38.409Z 15 300-319 9 9 9 16 16 16 F F F
15032024-04-17T17:11:38.409Z 16 320-339 6 6 6 10 10 10 F F F
15042024-04-17T17:11:38.409Z 17 340-359 10 10 10 17 17 17 F F F
15052024-04-17T17:11:38.409Z 18 360-379 10 10 10 17 17 17 F F F
15062024-04-17T17:11:38.409Z 19 380-399 8 8 8 15 15 15 F F F
15072024-04-17T17:11:38.409Z 20 400-419 10 10 10 19 19 19 F F F
15082024-04-17T17:11:38.410Z 21 420-439 13 13 13 24 24 24 F F F
15092024-04-17T17:11:38.410Z 22 440-459 12 12 12 23 23 23 F F F
15102024-04-17T17:11:38.410Z 23 460-479 13 13 13 24 24 24 F F F
15112024-04-17T17:11:38.410Z 24 480-499 13 13 13 24 24 24 F F F
15122024-04-17T17:11:38.410Z 25 500-519 12 12 12 21 21 21 F F F
15132024-04-17T17:11:38.410Z 26 520-539 10 10 10 19 19 19 F F F
15142024-04-17T17:11:38.410Z 27 540-559 10 10 10 19 19 19 F F F
15152024-04-17T17:11:38.410Z 28 560-579 12 12 12 21 21 21 F F F
15162024-04-17T17:11:38.410Z 29 580-599 10 10 10 18 18 18 F F F
15172024-04-17T17:11:38.411ZMax gen: 13, Max flush: 24
15182024-04-17T17:11:39.054Z
15192024-04-17T17:11:39.054ZBegin loop 7
15202024-04-17T17:11:39.054ZDownstairs to restart: 0
15212024-04-17T17:11:39.054Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.019157584Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224}
15222024-04-17T17:11:39.054Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.019447282Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224}
15232024-04-17T17:11:39.054Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30Crucible runtime is spawned
15242024-04-17T17:11:39.054Z,"time":"2024-04-17T17:11:37.019498175Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224}
15252024-04-17T17:11:39.054Z{"msg":"Crucible 1bceb14f-e30c-4d3d-8aba-ea167cf4e0e7 has session id: 7785b7a9-e7df-4cf5-8310-158cc3db42d7","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.019533957Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15262024-04-17T17:11:39.054Z{"msg":"Upstairs opts: Upstairs UUID: 1bceb14f-e30c-4d3d-8aba-ea167cf4e0e7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.019602291Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15272024-04-17T17:11:39.054Z{"msg":"Crucible stats registered with UUID: 1bceb14f-e30c-4d3d-8aba-ea167cf4e0e7","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.019688446Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15282024-04-17T17:11:39.054Z{"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.019720788Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224}
15292024-04-17T17:11:39.054Z{"msg":"1bceb14f-e30c-4d3d-8aba-ea167cf4e0e7 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.01974156Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15302024-04-17T17:11:39.054Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.019764221Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"client":"0","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15312024-04-17T17:11:39.054Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.019791003Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"client":"0","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15322024-04-17T17:11:39.054Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.019829745Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"client":"1","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15332024-04-17T17:11:39.054Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.019855776Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"client":"1","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15342024-04-17T17:11:39.054Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.019895399Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"client":"2","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15352024-04-17T17:11:39.054Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.01992038Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"client":"2","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15362024-04-17T17:11:39.055Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.019951192Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15372024-04-17T17:11:39.055Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.019981384Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15382024-04-17T17:11:39.055Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.020014416Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15392024-04-17T17:11:39.055Z{"msg":"ds_connection connected from Ok(127.0.0.1:58479)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.020135004Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15402024-04-17T17:11:39.055Z{"msg":"ds_connection connected from Ok(127.0.0.1:55584)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.020188907Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15412024-04-17T17:11:39.055Z{"msg":"ds_connection connected from Ok(127.0.0.1:51458)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.020222919Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15422024-04-17T17:11:39.055Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.020819065Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"client":"1","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15432024-04-17T17:11:39.055Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.020861657Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"client":"1","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15442024-04-17T17:11:39.055Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.020884469Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"client":"2","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15452024-04-17T17:11:39.055Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.02090618Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"client":"2","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15462024-04-17T17:11:39.055Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.021036848Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"client":"0","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15472024-04-17T17:11:39.055Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.021082331Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"client":"0","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15482024-04-17T17:11:39.055Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.02139935Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"client":"1","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15492024-04-17T17:11:39.055Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.021436992Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"client":"2","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15502024-04-17T17:11:39.055Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.0217301Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"client":"0","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15512024-04-17T17:11:39.055Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.022030398Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"client":"1","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15522024-04-17T17:11:39.055Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.022074421Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"client":"1","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15532024-04-17T17:11:39.055Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.022113863Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15542024-04-17T17:11:39.055Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.022136815Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"client":"2","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15552024-04-17T17:11:39.055Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.022156396Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"client":"2","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15562024-04-17T17:11:39.055Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.022178237Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15572024-04-17T17:11:39.055Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.022201209Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"client":"0","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15582024-04-17T17:11:39.061Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.02222585Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"client":"0","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15592024-04-17T17:11:39.061Z{"msg":"[0]R flush_numbers[0..12]: [23, 23, 20, 16, 13, 24, 24, 3, 23, 1, 19, 21]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.022250412Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"":"downstairs","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15602024-04-17T17:11:39.061Z{"msg":"[0]R generation[0..12]: [12, 12, 11, 9, 7, 13, 13, 2, 12, 1, 10, 12]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.022264782Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"":"downstairs","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15612024-04-17T17:11:39.061Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.022279563Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"":"downstairs","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15622024-04-17T17:11:39.062Z{"msg":"[1]R flush_numbers[0..12]: [23, 23, 20, 16, 13, 24, 24, 3, 23, 1, 19, 21]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.022293444Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"":"downstairs","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15632024-04-17T17:11:39.062Z{"msg":"[1]R generation[0..12]: [12, 12, 11, 9, 7, 13, 13, 2, 12, 1, 10, 12]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.022307405Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"":"downstairs","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15642024-04-17T17:11:39.062Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.022328456Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"":"downstairs","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15652024-04-17T17:11:39.062Z{"msg":"[2]R flush_numbers[0..12]: [23, 23, 20, 16, 13, 24, 24, 3, 23, 1, 19, 21]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
15662024-04-17T17:11:39.062Z,"time":"2024-04-17T17:11:37.022342827Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"":"downstairs","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15672024-04-17T17:11:39.062Z{"msg":"[2]R generation[0..12]: [12, 12, 11, 9, 7, 13, 13, 2, 12, 1, 10, 12]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.022357898Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"":"downstairs","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15682024-04-17T17:11:39.062Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.02237874Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"":"downstairs","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15692024-04-17T17:11:39.062Z{"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.02239183Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"":"downstairs","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15702024-04-17T17:11:39.062Z{"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.022404851Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"":"downstairs","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15712024-04-17T17:11:39.062Z{"msg":"Next flush: 25","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.022426582Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"":"downstairs","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15722024-04-17T17:11:39.062Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.022439503Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"":"downstairs","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15732024-04-17T17:11:39.062Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.022452294Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15742024-04-17T17:11:39.062Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.022482606Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"client":"0","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15752024-04-17T17:11:39.062Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.022504297Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"client":"1","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15762024-04-17T17:11:39.062Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.022532959Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"client":"2","":"downstairs","session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15772024-04-17T17:11:39.062Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.02254737Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15782024-04-17T17:11:39.062Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.022564151Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15792024-04-17T17:11:39.062Z{"msg":"1bceb14f-e30c-4d3d-8aba-ea167cf4e0e7 is now active with session: 7785b7a9-e7df-4cf5-8310-158cc3db42d7","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.022591832Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15802024-04-17T17:11:39.062Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:37.022605083Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224,"session_id":"7785b7a9-e7df-4cf5-8310-158cc3db42d7"}
15812024-04-17T17:11:39.062Z{"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
15822024-04-17T17:11:39.062Z,"time":"2024-04-17T17:11:37.022624574Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224}
15832024-04-17T17:11:39.062ZLoading write count information from file "/tmp/test_repair_verify.data"
15842024-04-17T17:11:39.062ZRun Repair workload
15852024-04-17T17:11:39.063Z01/30 Write block 579 len 36864 data: 5 4 4 3 3 3 3 3 2
15862024-04-17T17:11:39.063Z02/30 Read block 225 len 28672
15872024-04-17T17:11:39.063Z03/30 Flush
15882024-04-17T17:11:39.063Z04/30 Write block 198 len 32768 data: 2 2 2 2 2 3 3 3
15892024-04-17T17:11:39.063Z05/30 Read block 160 len 16384
15902024-04-17T17:11:39.063Z06/30 Read block 79 len 32768
15912024-04-17T17:11:39.063Z07/30 Write block 592 len 32768 data: 3 2 2 2 2 2 2 2
15922024-04-17T17:11:39.063Z08/30 Write block 89 len 28672 data: 3 3 3 3 3 2 2
15932024-04-17T17:11:39.063Z09/30 Read block 458 len 4096
15942024-04-17T17:11:39.063Z10/30 Read block 367 len 8192
15952024-04-17T17:11:39.063Z11/30 Read block 511 len 36864
15962024-04-17T17:11:39.063Z12/30 Write block 341 len 36864 data: 2 2 2 2 2 2 3 3 3
15972024-04-17T17:11:39.063Z13/30 Read block 378 len 4096
15982024-04-17T17:11:39.063Z14/30 Write block 274 len 12288 data: 3 4 4
15992024-04-17T17:11:39.063Z15/30 Flush
16002024-04-17T17:11:39.063Z16/30 Flush
16012024-04-17T17:11:39.063Z17/30 Flush
16022024-04-17T17:11:39.063Z18/30 Write block 11 len 20480 data: 3 3 3 3 3
16032024-04-17T17:11:39.063Z19/30 Write block 439 len 32768 data: 2 2 2 2 2 2 2 2
16042024-04-17T17:11:39.063Z20/30 Read block 78 len 32768
16052024-04-17T17:11:39.063Z21/30 Read block 196 len 36864
16062024-04-17T17:11:39.063Z22/30 Read block 352 len 28672
16072024-04-17T17:11:39.063Z23/30 Read block 510 len 20480
16082024-04-17T17:11:39.063Z24/30 Read block 559 len 40960
16092024-04-17T17:11:39.063Z25/30 Read block 28 len 12288
16102024-04-17T17:11:39.064Z26/30 Write block 113 len 32768 data: 3 3 3 2 2 2 4 3
16112024-04-17T17:11:39.064Z27/30 Write block 310 len 40960 data: 3 3 3 2 2 2 2 2 2 3
16122024-04-17T17:11:39.064Z28/30 Read block 130 len 16384
16132024-04-17T17:11:39.064Z29/30 Write block 256 len 36864 data: 4 4 4 4 3 3 3 3 3
16142024-04-17T17:11:39.064Z30/30 Write block 161 len 8192 data: 2 2
16152024-04-17T17:11:39.064Z----------------------------------------------------------------
16162024-04-17T17:11:39.064Z Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:30
16172024-04-17T17:11:39.064ZStates: Active Active Active
16182024-04-17T17:11:39.064ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
16192024-04-17T17:11:39.064Z 1 Acked 1000 Write 9 Done Done Done false
16202024-04-17T17:11:39.064Z 2 Acked 1001 Read 7 Sent Done Done false
16212024-04-17T17:11:39.064Z 3 Acked 1002 Flush 0 Sent Done Done false
16222024-04-17T17:11:39.064Z 4 Acked 1003 Write 8 Sent Done Done false
16232024-04-17T17:11:39.064Z 5 Acked 1004 Read 4 Sent Done Done false
16242024-04-17T17:11:39.064Z 6 Acked 1005 Read 8 Sent Done Done false
16252024-04-17T17:11:39.064Z 7 Acked 1006 Write 8 Sent Done Done false
16262024-04-17T17:11:39.064Z 8 Acked 1007 Write 7 Sent Done Done false
16272024-04-17T17:11:39.064Z 9 Acked 1008 Read 1 Sent Done Done false
16282024-04-17T17:11:39.064Z 10 Acked 1009 Read 2 Sent Done Done false
16292024-04-17T17:11:39.064Z 11 Acked 1010 Read 9 Sent Done Done false
16302024-04-17T17:11:39.064Z 12 Acked 1011 Write 9 Sent Done Done false
16312024-04-17T17:11:39.064Z 13 Acked 1012 Read 1 Sent Done Done false
16322024-04-17T17:11:39.065Z 14 Acked 1013 Write 3 Sent Done Done false
16332024-04-17T17:11:39.065Z 15 Acked 1014 Flush 0 Sent Done Done false
16342024-04-17T17:11:39.065Z 16 Acked 1015 Flush 0 Sent Done Done false
16352024-04-17T17:11:39.065Z 17 Acked 1016 Flush 0 Sent Done Done false
16362024-04-17T17:11:39.065Z 18 Acked 1017 Write 5 Sent Done Done false
16372024-04-17T17:11:39.065Z 19 Acked 1018 Write 8 Sent Done Done false
16382024-04-17T17:11:39.065Z 20 Acked 1019 Read 8 Sent Done Done false
16392024-04-17T17:11:39.065Z 21 Acked 1020 Read 9 Sent Done Done false
16402024-04-17T17:11:39.065Z 22 Acked 1021 Read 7 Sent Done Done false
16412024-04-17T17:11:39.065Z 23 Acked 1022 Read 5 Sent Done Done false
16422024-04-17T17:11:39.065Z 24 Acked 1023 Read 10 Sent Done Done false
16432024-04-17T17:11:39.065Z 25 Acked 1024 Read 3 Sent Done Done false
16442024-04-17T17:11:39.065Z 26 Acked 1025 Write 8 Sent Done Done false
16452024-04-17T17:11:39.065Z 27 Acked 1026 Write 10 Sent Done Done false
16462024-04-17T17:11:39.065Z 28 Acked 1027 Read 4 Sent Done Done false
16472024-04-17T17:11:39.065Z 29 Acked 1028 Write 9 Sent Done Done false
16482024-04-17T17:11:39.065Z 30 Acked 1029 Write 2 Sent Sent Sent false
16492024-04-17T17:11:39.065Z STATES DS:0 DS:1 DS:2 TOTAL
16502024-04-17T17:11:39.065Z New 0 0 0 0
16512024-04-17T17:11:39.065Z Sent 29 1 1 31
16522024-04-17T17:11:39.065Z Done 1 29 29 59
16532024-04-17T17:11:39.065Z Skipped 0 0 0 0
16542024-04-17T17:11:39.065Z Error 0 0 0 0
16552024-04-17T17:11:39.066ZLast Flush: 0 1016 1016
16562024-04-17T17:11:39.066ZDownstairs last five completed:
16572024-04-17T17:11:39.066ZUpstairs last five completed: 30 29 28 27 26
16582024-04-17T17:11:39.070Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:11:37.055508917Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1224}
16592024-04-17T17:11:39.070ZWrote out file "/tmp/test_repair_verify.data"
16602024-04-17T17:11:39.070Z
16612024-04-17T17:11:39.070ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
16622024-04-17T17:11:39.070ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
16632024-04-17T17:11:39.070Z 0 000-019 12 12 12 23 23 23 F T T <---
16642024-04-17T17:11:39.070Z 1 020-039 12 12 12 23 23 23 F F F
16652024-04-17T17:11:39.070Z 2 040-059 11 11 11 20 20 20 F F F
16662024-04-17T17:11:39.070Z 3 060-079 9 9 9 16 16 16 F F F
16672024-04-17T17:11:39.070Z 4 080-099 7 14 14 13 26 26 F F F <---
16682024-04-17T17:11:39.070Z 5 100-119 13 13 13 24 24 24 F T T <---
16692024-04-17T17:11:39.070Z 6 120-139 13 13 13 24 24 24 F T T <---
16702024-04-17T17:11:39.070Z 7 140-159 2 2 2 3 3 3 F F F
16712024-04-17T17:11:39.071Z 8 160-179 12 12 12 23 23 23 F T T <---
16722024-04-17T17:11:39.071Z 9 180-199 1 14 14 1 26 26 F F F <---
16732024-04-17T17:11:39.071Z 10 200-219 10 14 14 19 26 26 F F F <---
16742024-04-17T17:11:39.071Z 11 220-239 12 12 12 21 21 21 F F F
16752024-04-17T17:11:39.071Z 12 240-259 9 9 9 16 16 16 F T T <---
16762024-04-17T17:11:39.071Z 13 260-279 12 14 14 21 26 26 F T T <---
16772024-04-17T17:11:39.071Z 14 280-299 4 4 4 6 6 6 F F F
16782024-04-17T17:11:39.071Z 15 300-319 9 9 9 16 16 16 F T T <---
16792024-04-17T17:11:39.071Z 16 320-339 6 6 6 10 10 10 F F F
16802024-04-17T17:11:39.071Z 17 340-359 10 14 14 17 26 26 F F F <---
16812024-04-17T17:11:39.071Z 18 360-379 10 10 10 17 17 17 F F F
16822024-04-17T17:11:39.071Z 19 380-399 8 8 8 15 15 15 F F F
16832024-04-17T17:11:39.071Z 20 400-419 10 10 10 19 19 19 F F F
16842024-04-17T17:11:39.071Z 21 420-439 13 13 13 24 24 24 F T T <---
16852024-04-17T17:11:39.071Z 22 440-459 12 12 12 23 23 23 F T T <---
16862024-04-17T17:11:39.071Z 23 460-479 13 13 13 24 24 24 F F F
16872024-04-17T17:11:39.071Z 24 480-499 13 13 13 24 24 24 F F F
16882024-04-17T17:11:39.071Z 25 500-519 12 12 12 21 21 21 F F F
16892024-04-17T17:11:39.071Z 26 520-539 10 10 10 19 19 19 F F F
16902024-04-17T17:11:39.071Z 27 540-559 10 10 10 19 19 19 F F F
16912024-04-17T17:11:39.071Z 28 560-579 12 14 14 21 25 25 T F F <---
16922024-04-17T17:11:39.071Z 29 580-599 10 14 14 18 26 26 T F F <---
16932024-04-17T17:11:39.071ZMax gen: 14, Max flush: 26
16942024-04-17T17:11:39.071ZError: Difference in extent metadata found!
16952024-04-17T17:11:39.072ZA Difference in extent metadata is expected here
16962024-04-17T17:11:39.072ZOn loop 7
16972024-04-17T17:11:39.072Z
16982024-04-17T17:11:39.072ZVerifying data now
16992024-04-17T17:11:43.182ZNo change after verify
17002024-04-17T17:11:43.182ZLoop: 7 Downstairs dump after verify (and repair):
17012024-04-17T17:11:43.196ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
17022024-04-17T17:11:43.196Z 0 000-019 15 15 15 27 27 27 F F F
17032024-04-17T17:11:43.200Z 1 020-039 12 12 12 23 23 23 F F F
17042024-04-17T17:11:43.200Z 2 040-059 11 11 11 20 20 20 F F F
17052024-04-17T17:11:43.200Z 3 060-079 9 9 9 16 16 16 F F F
17062024-04-17T17:11:43.200Z 4 080-099 14 14 14 26 26 26 F F F
17072024-04-17T17:11:43.200Z 5 100-119 15 15 15 27 27 27 F F F
17082024-04-17T17:11:43.201Z 6 120-139 15 15 15 27 27 27 F F F
17092024-04-17T17:11:43.201Z 7 140-159 2 2 2 3 3 3 F F F
17102024-04-17T17:11:43.201Z 8 160-179 15 15 15 27 27 27 F F F
17112024-04-17T17:11:43.201Z 9 180-199 14 14 14 26 26 26 F F F
17122024-04-17T17:11:43.201Z 10 200-219 14 14 14 26 26 26 F F F
17132024-04-17T17:11:43.201Z 11 220-239 12 12 12 21 21 21 F F F
17142024-04-17T17:11:43.201Z 12 240-259 15 15 15 27 27 27 F F F
17152024-04-17T17:11:43.201Z 13 260-279 15 15 15 27 27 27 F F F
17162024-04-17T17:11:43.201Z 14 280-299 4 4 4 6 6 6 F F F
17172024-04-17T17:11:43.201Z 15 300-319 15 15 15 27 27 27 F F F
17182024-04-17T17:11:43.201Z 16 320-339 6 6 6 10 10 10 F F F
17192024-04-17T17:11:43.201Z 17 340-359 14 14 14 26 26 26 F F F
17202024-04-17T17:11:43.201Z 18 360-379 10 10 10 17 17 17 F F F
17212024-04-17T17:11:43.202Z 19 380-399 8 8 8 15 15 15 F F F
17222024-04-17T17:11:43.202Z 20 400-419 10 10 10 19 19 19 F F F
17232024-04-17T17:11:43.202Z 21 420-439 15 15 15 27 27 27 F F F
17242024-04-17T17:11:43.202Z 22 440-459 15 15 15 27 27 27 F F F
17252024-04-17T17:11:43.202Z 23 460-479 13 13 13 24 24 24 F F F
17262024-04-17T17:11:43.202Z 24 480-499 13 13 13 24 24 24 F F F
17272024-04-17T17:11:43.202Z 25 500-519 12 12 12 21 21 21 F F F
17282024-04-17T17:11:43.202Z 26 520-539 10 10 10 19 19 19 F F F
17292024-04-17T17:11:43.202Z 27 540-559 10 10 10 19 19 19 F F F
17302024-04-17T17:11:43.202Z 28 560-579 14 14 14 25 25 25 F F F
17312024-04-17T17:11:43.202Z 29 580-599 14 14 14 26 26 26 F F F
17322024-04-17T17:11:43.202ZMax gen: 15, Max flush: 27
17332024-04-17T17:11:43.203Z
17342024-04-17T17:11:43.203ZBegin loop 8
17352024-04-17T17:11:43.203ZDownstairs to restart: 2
17362024-04-17T17:11:43.218Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.824580388Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232}
17372024-04-17T17:11:43.218Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.824816523Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232}
17382024-04-17T17:11:43.226Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.824842944Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232}
17392024-04-17T17:11:43.226Z{"msg":"Crucible c13b5f8c-8728-4592-bf14-05cfb85d4b25 has session id: 0af18a23-0e9d-4c1b-866b-33f0a2d67669","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.824871956Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17402024-04-17T17:11:43.226Z{"msg":"Upstairs opts: Upstairs UUID: c13b5f8c-8728-4592-bf14-05cfb85d4b25, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.824916019Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17412024-04-17T17:11:43.226Z{"msg":"Crucible stats registered with UUID: c13b5f8c-8728-4592-bf14-05cfb85d4b25","v":0,"name":"crucible","level":30Crucible runtime is spawned
17422024-04-17T17:11:43.226Z,"time":"2024-04-17T17:11:41.824952561Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17432024-04-17T17:11:43.226Z{"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.825056267Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232}
17442024-04-17T17:11:43.227Z{"msg":"c13b5f8c-8728-4592-bf14-05cfb85d4b25 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.825123192Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17452024-04-17T17:11:43.227Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.825164314Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"client":"0","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17462024-04-17T17:11:43.227Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.825203396Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"client":"0","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17472024-04-17T17:11:43.227Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.825249589Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"client":"1","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17482024-04-17T17:11:43.227Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.825296742Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"client":"1","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17492024-04-17T17:11:43.227Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.825321884Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"client":"2","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17502024-04-17T17:11:43.227Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.825375277Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"client":"2","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17512024-04-17T17:11:43.227Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.825401618Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17522024-04-17T17:11:43.227Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.82543227Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17532024-04-17T17:11:43.227Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.825458372Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17542024-04-17T17:11:43.227Z{"msg":"ds_connection connected from Ok(127.0.0.1:36849)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.825492464Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17552024-04-17T17:11:43.228Z{"msg":"ds_connection connected from Ok(127.0.0.1:36429)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.825519685Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17562024-04-17T17:11:43.228Z{"msg":"ds_connection connected from Ok(127.0.0.1:50521)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.825566978Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17572024-04-17T17:11:43.228Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.826179645Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"client":"1","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17582024-04-17T17:11:43.228Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.826226898Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"client":"1","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17592024-04-17T17:11:43.228Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.82625582Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"client":"0","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17602024-04-17T17:11:43.228Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.826281822Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"client":"0","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17612024-04-17T17:11:43.228Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.826698687Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"client":"2","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17622024-04-17T17:11:43.228Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.826726198Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"client":"2","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17632024-04-17T17:11:43.228Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.826892669Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"client":"0","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17642024-04-17T17:11:43.228Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.826933771Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"client":"1","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17652024-04-17T17:11:43.229Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.827299423Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"client":"2","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17662024-04-17T17:11:43.229Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.827519047Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"client":"1","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17672024-04-17T17:11:43.229Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.827554959Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"client":"1","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17682024-04-17T17:11:43.229Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.827589631Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17692024-04-17T17:11:43.229Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.827609682Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"client":"0","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17702024-04-17T17:11:43.229Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.827623463Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"client":"0","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17712024-04-17T17:11:43.229Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.827636454Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17722024-04-17T17:11:43.229Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.827837286Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"client":"2","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17732024-04-17T17:11:43.229Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.827873288Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"client":"2","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17742024-04-17T17:11:43.229Z{"msg":"[0]R flush_numbers[0..12]: [27, 23, 20, 16, 26, 27, 27, 3, 27, 26, 26, 21]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.827889059Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"":"downstairs","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17752024-04-17T17:11:43.229Z{"msg":"[0]R generation[0..12]: [15, 12, 11, 9, 14, 15, 15, 2, 15, 14, 14, 12]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.82790458Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"":"downstairs","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17762024-04-17T17:11:43.229Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.827919161Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"":"downstairs","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17772024-04-17T17:11:43.229Z{"msg":"[1]R flush_numbers[0..12]: [27, 23, 20, 16, 26, 27, 27, 3, 27, 26, 26, 21]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.827940392Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"":"downstairs","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17782024-04-17T17:11:43.230Z{"msg":"[1]R generation[0..12]: [15, 12, 11, 9, 14, 15, 15, 2, 15, 14, 14, 12]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.827954873Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"":"downstairs","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17792024-04-17T17:11:43.230Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.827969484Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"":"downstairs","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17802024-04-17T17:11:43.230Z{"msg":"[2]R flush_numbers[0..12]: [27, 23, 20, 16, 26, 27, 27, 3, 27, 26, 26, 21]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.827984925Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"":"downstairs","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17812024-04-17T17:11:43.230ZWait for a query_work_queue command to finish before sending IO
17822024-04-17T17:11:43.230Z{"msg":"[2]R generation[0..12]: [15, 12, 11, 9, 14, 15, 15, 2, 15, 14, 14, 12]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.828016947Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"":"downstairs","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17832024-04-17T17:11:43.230Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.828033248Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"":"downstairs","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17842024-04-17T17:11:43.230Z{"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.82806507Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"":"downstairs","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17852024-04-17T17:11:43.230Z{"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.828083441Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"":"downstairs","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17862024-04-17T17:11:43.230Z{"msg":"Next flush: 28","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.828105752Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"":"downstairs","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17872024-04-17T17:11:43.230Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.828140234Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"":"downstairs","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17882024-04-17T17:11:43.230Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.828171976Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17892024-04-17T17:11:43.230Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.828190117Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"client":"0","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17902024-04-17T17:11:43.231Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.828209319Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"client":"1","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17912024-04-17T17:11:43.231Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.82823844Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"client":"2","":"downstairs","session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17922024-04-17T17:11:43.231Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.828262842Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17932024-04-17T17:11:43.231Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.828289103Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17942024-04-17T17:11:43.231Z{"msg":"c13b5f8c-8728-4592-bf14-05cfb85d4b25 is now active with session: 0af18a23-0e9d-4c1b-866b-33f0a2d67669","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.828320195Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17952024-04-17T17:11:43.231ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
17962024-04-17T17:11:43.231Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.828352717Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232,"session_id":"0af18a23-0e9d-4c1b-866b-33f0a2d67669"}
17972024-04-17T17:11:43.231Z{"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:41.828373598Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232}
17982024-04-17T17:11:43.231ZLoading write count information from file "/tmp/test_repair_verify.data"
17992024-04-17T17:11:43.231ZRun Repair workload
18002024-04-17T17:11:43.236Z01/30 Write block 41 len 24576 data: 3 3 3 3 4 3
18012024-04-17T17:11:43.236Z02/30 Write block 49 len 4096 data: 2
18022024-04-17T17:11:43.236Z03/30 Read block 489 len 36864
18032024-04-17T17:11:43.236Z04/30 Flush
18042024-04-17T17:11:43.236Z05/30 Write block 320 len 12288 data: 3 2 2
18052024-04-17T17:11:43.236Z06/30 Read block 329 len 16384
18062024-04-17T17:11:43.236Z07/30 Read block 195 len 32768
18072024-04-17T17:11:43.237Z08/30 Write block 232 len 12288 data: 2 2 2
18082024-04-17T17:11:43.237Z09/30 Read block 333 len 16384
18092024-04-17T17:11:43.237Z10/30 Read block 192 len 16384
18102024-04-17T17:11:43.237Z11/30 Read block 275 len 16384
18112024-04-17T17:11:43.237Z12/30 Read block 476 len 28672
18122024-04-17T17:11:43.237Z13/30 Write block 334 len 24576 data: 2 3 2 2 2 2
18132024-04-17T17:11:43.237Z14/30 Read block 394 len 24576
18142024-04-17T17:11:43.237Z15/30 Read block 43 len 24576
18152024-04-17T17:11:43.241Z16/30 Write block 269 len 32768 data: 2 2 2 2 3 4 5 5
18162024-04-17T17:11:43.241Z17/30 Write block 409 len 28672 data: 4 3 4 4 4 4 3
18172024-04-17T17:11:43.241Z18/30 Write block 476 len 20480 data: 3 3 4 5 5
18182024-04-17T17:11:43.241Z19/30 Read block 267 len 4096
18192024-04-17T17:11:43.241Z20/30 Read block 309 len 28672
18202024-04-17T17:11:43.241Z21/30 Flush
18212024-04-17T17:11:43.245Z22/30 Write block 368 len 16384 data: 2 2 2 2
18222024-04-17T17:11:43.248Z23/30 Read block 243 len 8192
18232024-04-17T17:11:43.248Z24/30 Read block 31 len 40960
18242024-04-17T17:11:43.248Z25/30 Write block 74 len 4096 data: 2
18252024-04-17T17:11:43.248Z26/30 Read block 175 len 16384
18262024-04-17T17:11:43.248Z27/30 Write block 569 len 16384 data: 3 3 3 3
18272024-04-17T17:11:43.249Z28/30 Read block 390 len 16384
18282024-04-17T17:11:43.253Z29/30 Read block 64 len 36864
18292024-04-17T17:11:43.253Z30/30 Write block 167 len 28672 data: 2 2 2 2 3 3 3
18302024-04-17T17:11:43.253Z----------------------------------------------------------------
18312024-04-17T17:11:43.253Z Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:26
18322024-04-17T17:11:43.253ZStates: Active Active Active
18332024-04-17T17:11:43.253ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
18342024-04-17T17:11:43.253Z 5 Acked 1004 Write 3 Done Done Sent false
18352024-04-17T17:11:43.253Z 6 Acked 1005 Read 4 Done Done Sent false
18362024-04-17T17:11:43.253Z 7 Acked 1006 Read 8 Done Done Sent false
18372024-04-17T17:11:43.253Z 8 Acked 1007 Write 3 Done Done Sent false
18382024-04-17T17:11:43.253Z 9 Acked 1008 Read 4 Done Done Sent false
18392024-04-17T17:11:43.254Z 10 Acked 1009 Read 4 Done Done Sent false
18402024-04-17T17:11:43.254Z 11 Acked 1010 Read 4 Done Done Sent false
18412024-04-17T17:11:43.254Z 12 Acked 1011 Read 7 Done Done Sent false
18422024-04-17T17:11:43.254Z 13 Acked 1012 Write 6 Done Done Sent false
18432024-04-17T17:11:43.254Z 14 Acked 1013 Read 6 Done Done Sent false
18442024-04-17T17:11:43.254Z 15 Acked 1014 Read 6 Done Done Sent false
18452024-04-17T17:11:43.254Z 16 Acked 1015 Write 8 Done Done Sent false
18462024-04-17T17:11:43.254Z 17 Acked 1016 Write 7 Done Done Sent false
18472024-04-17T17:11:43.254Z 18 Acked 1017 Write 5 Done Done Sent false
18482024-04-17T17:11:43.254Z 19 Acked 1018 Read 1 Done Done Sent false
18492024-04-17T17:11:43.254Z 20 Acked 1019 Read 7 Done Done Sent false
18502024-04-17T17:11:43.254Z 21 Acked 1020 Flush 0 Done Done Sent false
18512024-04-17T17:11:43.254Z 22 Acked 1021 Write 4 Done Done Sent false
18522024-04-17T17:11:43.254Z 23 Acked 1022 Read 2 Done Done Sent false
18532024-04-17T17:11:43.254Z 24 Acked 1023 Read 10 Done Done Sent false
18542024-04-17T17:11:43.254Z 25 Acked 1024 Write 1 Done Done Sent false
18552024-04-17T17:11:43.254Z 26 Acked 1025 Read 4 Done Done Sent false
18562024-04-17T17:11:43.254Z 27 Acked 1026 Write 4 Done Done Sent false
18572024-04-17T17:11:43.254Z 28 Acked 1027 Read 4 Done Done Sent false
18582024-04-17T17:11:43.254Z 29 Acked 1028 Read 9 Done Done Sent false
18592024-04-17T17:11:43.254Z 30 Acked 1029 Write 7 Sent Sent Sent false
18602024-04-17T17:11:43.255Z STATES DS:0 DS:1 DS:2 TOTAL
18612024-04-17T17:11:43.255Z New 0 0 0 0
18622024-04-17T17:11:43.255Z Sent 1 1 26 28
18632024-04-17T17:11:43.255Z Done 25 25 0 50
18642024-04-17T17:11:43.255Z Skipped 0 0 0 0
18652024-04-17T17:11:43.255Z Error 0 0 0 0
18662024-04-17T17:11:43.255ZLast Flush: 1020 1020 1003
18672024-04-17T17:11:43.255ZDownstairs last five completed: 1003 1002 1001 1000
18682024-04-17T17:11:43.255ZUpstairs last five completed: 30 29 28 27 26
18692024-04-17T17:11:43.255Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:11:41.858152633Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1232}
18702024-04-17T17:11:43.255ZWrote out file "/tmp/test_repair_verify.data"
18712024-04-17T17:11:43.259Z
18722024-04-17T17:11:43.263ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
18732024-04-17T17:11:43.276ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
18742024-04-17T17:11:43.276Z 0 000-019 15 15 15 27 27 27 F F F
18752024-04-17T17:11:43.276Z 1 020-039 12 12 12 23 23 23 F F F
18762024-04-17T17:11:43.277Z 2 040-059 16 16 16 28 28 28 F F F
18772024-04-17T17:11:43.277Z 3 060-079 9 9 9 16 16 16 T T F <---
18782024-04-17T17:11:43.277Z 4 080-099 14 14 14 26 26 26 F F F
18792024-04-17T17:11:43.277Z 5 100-119 15 15 15 27 27 27 F F F
18802024-04-17T17:11:43.277Z 6 120-139 15 15 15 27 27 27 F F F
18812024-04-17T17:11:43.277Z 7 140-159 2 2 2 3 3 3 F F F
18822024-04-17T17:11:43.281Z 8 160-179 15 15 15 27 27 27 T T F <---
18832024-04-17T17:11:43.281Z 9 180-199 14 14 14 26 26 26 F F F
18842024-04-17T17:11:43.281Z 10 200-219 14 14 14 26 26 26 F F F
18852024-04-17T17:11:43.281Z 11 220-239 16 16 12 29 29 21 F F F <---
18862024-04-17T17:11:43.281Z 12 240-259 15 15 15 27 27 27 F F F
18872024-04-17T17:11:43.281Z 13 260-279 16 16 15 29 29 27 F F F <---
18882024-04-17T17:11:43.281Z 14 280-299 4 4 4 6 6 6 F F F
18892024-04-17T17:11:43.281Z 15 300-319 15 15 15 27 27 27 F F F
18902024-04-17T17:11:43.281Z 16 320-339 16 16 6 29 29 10 F F F <---
18912024-04-17T17:11:43.281Z 17 340-359 14 14 14 26 26 26 F F F
18922024-04-17T17:11:43.281Z 18 360-379 10 10 10 17 17 17 T T F <---
18932024-04-17T17:11:43.281Z 19 380-399 8 8 8 15 15 15 F F F
18942024-04-17T17:11:43.281Z 20 400-419 16 16 10 29 29 19 F F F <---
18952024-04-17T17:11:43.281Z 21 420-439 15 15 15 27 27 27 F F F
18962024-04-17T17:11:43.281Z 22 440-459 15 15 15 27 27 27 F F F
18972024-04-17T17:11:43.282Z 23 460-479 16 16 13 29 29 24 F F F <---
18982024-04-17T17:11:43.282Z 24 480-499 16 16 13 29 29 24 F F F <---
18992024-04-17T17:11:43.282Z 25 500-519 12 12 12 21 21 21 F F F
19002024-04-17T17:11:43.282Z 26 520-539 10 10 10 19 19 19 F F F
19012024-04-17T17:11:43.282Z 27 540-559 10 10 10 19 19 19 F F F
19022024-04-17T17:11:43.282Z 28 560-579 14 14 14 25 25 25 T T F <---
19032024-04-17T17:11:43.282Z 29 580-599 14 14 14 26 26 26 F F F
19042024-04-17T17:11:43.282ZMax gen: 16, Max flush: 29
19052024-04-17T17:11:43.282ZError: Difference in extent metadata found!
19062024-04-17T17:11:43.282ZA Difference in extent metadata is expected here
19072024-04-17T17:11:43.282ZOn loop 8
19082024-04-17T17:11:43.282Z
19092024-04-17T17:11:43.286ZVerifying data now
19102024-04-17T17:11:47.779ZNo change after verify
19112024-04-17T17:11:47.779ZLoop: 8 Downstairs dump after verify (and repair):
19122024-04-17T17:11:47.794ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
19132024-04-17T17:11:47.794Z 0 000-019 15 15 15 27 27 27 F F F
19142024-04-17T17:11:47.803Z 1 020-039 12 12 12 23 23 23 F F F
19152024-04-17T17:11:47.803Z 2 040-059 16 16 16 28 28 28 F F F
19162024-04-17T17:11:47.803Z 3 060-079 17 17 17 30 30 30 F F F
19172024-04-17T17:11:47.803Z 4 080-099 14 14 14 26 26 26 F F F
19182024-04-17T17:11:47.803Z 5 100-119 15 15 15 27 27 27 F F F
19192024-04-17T17:11:47.804Z 6 120-139 15 15 15 27 27 27 F F F
19202024-04-17T17:11:47.804Z 7 140-159 2 2 2 3 3 3 F F F
19212024-04-17T17:11:47.804Z 8 160-179 17 17 17 30 30 30 F F F
19222024-04-17T17:11:47.804Z 9 180-199 14 14 14 26 26 26 F F F
19232024-04-17T17:11:47.804Z 10 200-219 14 14 14 26 26 26 F F F
19242024-04-17T17:11:47.804Z 11 220-239 16 16 16 29 29 29 F F F
19252024-04-17T17:11:47.804Z 12 240-259 15 15 15 27 27 27 F F F
19262024-04-17T17:11:47.804Z 13 260-279 16 16 16 29 29 29 F F F
19272024-04-17T17:11:47.804Z 14 280-299 4 4 4 6 6 6 F F F
19282024-04-17T17:11:47.805Z 15 300-319 15 15 15 27 27 27 F F F
19292024-04-17T17:11:47.805Z 16 320-339 16 16 16 29 29 29 F F F
19302024-04-17T17:11:47.805Z 17 340-359 14 14 14 26 26 26 F F F
19312024-04-17T17:11:47.805Z 18 360-379 17 17 17 30 30 30 F F F
19322024-04-17T17:11:47.805Z 19 380-399 8 8 8 15 15 15 F F F
19332024-04-17T17:11:47.805Z 20 400-419 16 16 16 29 29 29 F F F
19342024-04-17T17:11:47.805Z 21 420-439 15 15 15 27 27 27 F F F
19352024-04-17T17:11:47.806Z 22 440-459 15 15 15 27 27 27 F F F
19362024-04-17T17:11:47.806Z 23 460-479 16 16 16 29 29 29 F F F
19372024-04-17T17:11:47.806Z 24 480-499 16 16 16 29 29 29 F F F
19382024-04-17T17:11:47.806Z 25 500-519 12 12 12 21 21 21 F F F
19392024-04-17T17:11:47.806Z 26 520-539 10 10 10 19 19 19 F F F
19402024-04-17T17:11:47.806Z 27 540-559 10 10 10 19 19 19 F F F
19412024-04-17T17:11:47.807Z 28 560-579 17 17 17 30 30 30 F F F
19422024-04-17T17:11:47.807Z 29 580-599 14 14 14 26 26 26 F F F
19432024-04-17T17:11:47.807ZMax gen: 17, Max flush: 30
19442024-04-17T17:11:47.807Z
19452024-04-17T17:11:47.807ZBegin loop 9
19462024-04-17T17:11:47.808ZDownstairs to restart: 1
19472024-04-17T17:11:47.818Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.421325006Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240}
19482024-04-17T17:11:47.838Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.421587102Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240}
19492024-04-17T17:11:47.838ZCrucible runtime is spawned
19502024-04-17T17:11:47.838Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.42173626Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240}
19512024-04-17T17:11:47.839Z{"msg":"Crucible e35928db-816e-465c-8d4f-dd86e6a63848 has session id: 24b27d0f-abb2-4b44-9286-b73df7cb8b18","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.421768633Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19522024-04-17T17:11:47.839Z{"msg":"Upstairs opts: Upstairs UUID: e35928db-816e-465c-8d4f-dd86e6a63848, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.421812715Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19532024-04-17T17:11:47.839Z{"msg":"Crucible stats registered with UUID: e35928db-816e-465c-8d4f-dd86e6a63848","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.421851198Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19542024-04-17T17:11:47.839Z{"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.421877649Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240}
19552024-04-17T17:11:47.839Z{"msg":"e35928db-816e-465c-8d4f-dd86e6a63848 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.42189009Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19562024-04-17T17:11:47.839Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.421909281Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"client":"0","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19572024-04-17T17:11:47.839Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.421925862Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"client":"0","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19582024-04-17T17:11:47.839Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.421958764Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"client":"1","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19592024-04-17T17:11:47.839Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.421995606Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"client":"1","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19602024-04-17T17:11:47.840Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.422022288Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"client":"2","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19612024-04-17T17:11:47.840Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.4220659Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"client":"2","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19622024-04-17T17:11:47.840Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.422090962Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19632024-04-17T17:11:47.840Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.422120124Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19642024-04-17T17:11:47.840Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.422145325Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19652024-04-17T17:11:47.840Z{"msg":"ds_connection connected from Ok(127.0.0.1:55932)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.422266593Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19662024-04-17T17:11:47.840Z{"msg":"ds_connection connected from Ok(127.0.0.1:51463)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.422315036Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19672024-04-17T17:11:47.840Z{"msg":"ds_connection connected from Ok(127.0.0.1:52845)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.422344237Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19682024-04-17T17:11:47.840Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.423093103Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"client":"2","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19692024-04-17T17:11:47.840Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.423136796Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"client":"2","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19702024-04-17T17:11:47.840Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.423165307Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"client":"0","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19712024-04-17T17:11:47.840Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.423187959Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"client":"0","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19722024-04-17T17:11:47.840Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.423312336Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"client":"1","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19732024-04-17T17:11:47.840Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.42336793Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"client":"1","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19742024-04-17T17:11:47.840Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.423717311Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"client":"0","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19752024-04-17T17:11:47.841Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.423759493Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"client":"2","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19762024-04-17T17:11:47.841Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.423916953Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"client":"1","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19772024-04-17T17:11:47.841Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.424251763Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"client":"0","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19782024-04-17T17:11:47.841Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.424305986Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"client":"0","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19792024-04-17T17:11:47.841Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.42436599Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19802024-04-17T17:11:47.841Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.424414133Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"client":"2","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19812024-04-17T17:11:47.841Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.424432754Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"client":"2","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19822024-04-17T17:11:47.841Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.424455695Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19832024-04-17T17:11:47.841Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.424468656Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"client":"1","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19842024-04-17T17:11:47.841Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.424482977Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"client":"1","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19852024-04-17T17:11:47.841Z{"msg":"[0]R flush_numbers[0..12]: [27, 23, 28, 30, 26, 27, 27, 3, 30, 26, 26, 29]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.424504308Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"":"downstairs","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19862024-04-17T17:11:47.841Z{"msg":"[0]R generation[0..12]: [15, 12, 16, 17, 14, 15, 15, 2, 17, 14, 14, 16]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.424519569Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"":"downstairs","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19872024-04-17T17:11:47.841Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.42453418Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"":"downstairs","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19882024-04-17T17:11:47.842Z{"msg":"[1]R flush_numbers[0..12]: [27, 23, 28, 30, 26, 27, 27, 3, 30, 26, 26, 29]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.424547861Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"":"downstairs","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19892024-04-17T17:11:47.842Z{"msg":"[1]R generation[0..12]: [15, 12, 16, 17, 14, 15, 15, 2, 17, 14, 14, 16]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.424568832Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"":"downstairs","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19902024-04-17T17:11:47.842Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.424583013Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"":"downstairs","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19912024-04-17T17:11:47.842Z{"msg":"Wait for a query_work_queue command to finish before sending IO
19922024-04-17T17:11:47.842Z[2]R flush_numbers[0..12]: [27, 23, 28, 30, 26, 27, 27, 3, 30, 26, 26, 29]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.424599224Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"":"downstairs","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19932024-04-17T17:11:47.842Z{"msg":"[2]R generation[0..12]: [15, 12, 16, 17, 14, 15, 15, 2, 17, 14, 14, 16]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.424613445Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"":"downstairs","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19942024-04-17T17:11:47.842Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.424634556Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"":"downstairs","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19952024-04-17T17:11:47.842Z{"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.424647637Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"":"downstairs","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19962024-04-17T17:11:47.842Z{"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.424660508Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"":"downstairs","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19972024-04-17T17:11:47.843Z{"msg":"Next flush: 31","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.424680169Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"":"downstairs","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19982024-04-17T17:11:47.843Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.42469329Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"":"downstairs","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
19992024-04-17T17:11:47.843Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.424706951Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
20002024-04-17T17:11:47.843Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.424731152Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"client":"0","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
20012024-04-17T17:11:47.843Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.424744873Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"client":"1","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
20022024-04-17T17:11:47.843Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.424758184Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"client":"2","":"downstairs","session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
20032024-04-17T17:11:47.843Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.424771095Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
20042024-04-17T17:11:47.843Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.424783465Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
20052024-04-17T17:11:47.843Z{"msg":"e35928db-816e-465c-8d4f-dd86e6a63848 is now active with session: 24b27d0f-abb2-4b44-9286-b73df7cb8b18","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.424795956Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
20062024-04-17T17:11:47.843Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.424807907Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240,"session_id":"24b27d0f-abb2-4b44-9286-b73df7cb8b18"}
20072024-04-17T17:11:47.844Z{"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:46.424819807Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240}
20082024-04-17T17:11:47.844ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
20092024-04-17T17:11:47.844ZLoading write count information from file "/tmp/test_repair_verify.data"
20102024-04-17T17:11:47.844ZRun Repair workload
20112024-04-17T17:11:47.844Z01/30 Flush
20122024-04-17T17:11:47.844Z02/30 Write block 130 len 32768 data: 2 2 3 3 3 3 3 2
20132024-04-17T17:11:47.844Z03/30 Write block 528 len 24576 data: 2 2 2 2 2 2
20142024-04-17T17:11:47.844Z04/30 Write block 91 len 20480 data: 4 4 4 3 3
20152024-04-17T17:11:47.844Z05/30 Flush
20162024-04-17T17:11:47.869Z06/30 Flush
20172024-04-17T17:11:47.869Z07/30 Write block 453 len 32768 data: 2 2 2 2 2 3 3 3
20182024-04-17T17:11:47.869Z08/30 Read block 366 len 28672
20192024-04-17T17:11:47.869Z09/30 Write block 22 len 24576 data: 3 3 3 4 4 2
20202024-04-17T17:11:47.870Z10/30 Read block 104 len 28672
20212024-04-17T17:11:47.870Z11/30 Read block 562 len 40960
20222024-04-17T17:11:47.870Z12/30 Flush
20232024-04-17T17:11:47.870Z13/30 Write block 412 len 36864 data: 5 5 5 4 3 3 3 3 4
20242024-04-17T17:11:47.870Z14/30 Write block 554 len 28672 data: 3 3 3 3 4 4 3
20252024-04-17T17:11:47.870Z15/30 Read block 44 len 16384
20262024-04-17T17:11:47.871Z16/30 Write block 486 len 4096 data: 4
20272024-04-17T17:11:47.871Z17/30 Read block 50 len 4096
20282024-04-17T17:11:47.871Z18/30 Read block 490 len 32768
20292024-04-17T17:11:47.871Z19/30 Write block 371 len 12288 data: 3 2 2
20302024-04-17T17:11:47.871Z20/30 Write block 566 len 4096 data: 3
20312024-04-17T17:11:47.871Z21/30 Read block 212 len 8192
20322024-04-17T17:11:47.871Z22/30 Write block 215 len 32768 data: 2 2 2 3 2 2 2 2
20332024-04-17T17:11:47.871Z23/30 Flush
20342024-04-17T17:11:47.872Z24/30 Write block 4 len 24576 data: 3 3 3 2 2 2
20352024-04-17T17:11:47.872Z25/30 Read block 350 len 16384
20362024-04-17T17:11:47.872Z26/30 Read block 467 len 16384
20372024-04-17T17:11:47.872Z27/30 Read block 75 len 24576
20382024-04-17T17:11:47.886Z28/30 Write block 60 len 32768 data: 3 3 2 2 2 2 2 2
20392024-04-17T17:11:47.886Z29/30 Write block 377 len 12288 data: 2 3 3
20402024-04-17T17:11:47.886Z30/30 Read block 15 len 36864
20412024-04-17T17:11:47.887Z----------------------------------------------------------------
20422024-04-17T17:11:47.887Z Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:29
20432024-04-17T17:11:47.887ZStates: Active Active Active
20442024-04-17T17:11:47.887ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
20452024-04-17T17:11:47.887Z 2 Acked 1001 Write 8 Done Sent Done false
20462024-04-17T17:11:47.887Z 3 Acked 1002 Write 6 Done Sent Done false
20472024-04-17T17:11:47.887Z 4 Acked 1003 Write 5 Done Sent Done false
20482024-04-17T17:11:47.888Z 5 Acked 1004 Flush 0 Done Sent Done false
20492024-04-17T17:11:47.888Z 6 Acked 1005 Flush 0 Done Sent Done false
20502024-04-17T17:11:47.888Z 7 Acked 1006 Write 8 Done Sent Done false
20512024-04-17T17:11:47.888Z 8 Acked 1007 Read 7 Done Sent Done false
20522024-04-17T17:11:47.888Z 9 Acked 1008 Write 6 Done Sent Done false
20532024-04-17T17:11:47.888Z 10 Acked 1009 Read 7 Done Sent Done false
20542024-04-17T17:11:47.888Z 11 Acked 1010 Read 10 Done Sent Done false
20552024-04-17T17:11:47.888Z 12 Acked 1011 Flush 0 Done Sent Done false
20562024-04-17T17:11:47.889Z 13 Acked 1012 Write 9 Done Sent Done false
20572024-04-17T17:11:47.889Z 14 Acked 1013 Write 7 Done Sent Done false
20582024-04-17T17:11:47.889Z 15 Acked 1014 Read 4 Done Sent Done false
20592024-04-17T17:11:47.889Z 16 Acked 1015 Write 1 Done Sent Done false
20602024-04-17T17:11:47.889Z 17 Acked 1016 Read 1 Done Sent Done false
20612024-04-17T17:11:47.889Z 18 Acked 1017 Read 8 Done Sent Done false
20622024-04-17T17:11:47.889Z 19 Acked 1018 Write 3 Done Sent Done false
20632024-04-17T17:11:47.889Z 20 Acked 1019 Write 1 Done Sent Done false
20642024-04-17T17:11:47.890Z 21 Acked 1020 Read 2 Done Sent Done false
20652024-04-17T17:11:47.890Z 22 Acked 1021 Write 8 Done Sent Done false
20662024-04-17T17:11:47.890Z 23 Acked 1022 Flush 0 Done Sent Done false
20672024-04-17T17:11:47.890Z 24 Acked 1023 Write 6 Done Sent Done false
20682024-04-17T17:11:47.890Z 25 Acked 1024 Read 4 Done Sent Done false
20692024-04-17T17:11:47.890Z 26 Acked 1025 Read 4 Done Sent Done false
20702024-04-17T17:11:47.890Z 27 Acked 1026 Read 6 Done Sent Done false
20712024-04-17T17:11:47.890Z 28 Acked 1027 Write 8 Done Sent Done false
20722024-04-17T17:11:47.890Z 29 Acked 1028 Write 3 Done Sent Done false
20732024-04-17T17:11:47.891Z 30 Acked 1029 Read 9 Done Sent Sent false
20742024-04-17T17:11:47.891Z STATES DS:0 DS:1 DS:2 TOTAL
20752024-04-17T17:11:47.891Z New 0 0 0 0
20762024-04-17T17:11:47.891Z Sent 0 29 1 30
20772024-04-17T17:11:47.891Z Done 29 0 28 57
20782024-04-17T17:11:47.891Z Skipped 0 0 0 0
20792024-04-17T17:11:47.891Z Error 0 0 0 0
20802024-04-17T17:11:47.891ZLast Flush: 1022 1000 1022
20812024-04-17T17:11:47.891ZDownstairs last five completed: 1000
20822024-04-17T17:11:47.891ZUpstairs last five completed: 30 29 28 27 26
20832024-04-17T17:11:47.891Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:11:46.458980668Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1240}
20842024-04-17T17:11:47.891ZWrote out file "/tmp/test_repair_verify.data"
20852024-04-17T17:11:47.892Z
20862024-04-17T17:11:47.892ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
20872024-04-17T17:11:47.892ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
20882024-04-17T17:11:47.892Z 0 000-019 15 15 15 27 27 27 T F T <---
20892024-04-17T17:11:47.892Z 1 020-039 18 12 18 34 23 34 F F F <---
20902024-04-17T17:11:47.892Z 2 040-059 16 16 16 28 28 28 F F F
20912024-04-17T17:11:47.892Z 3 060-079 17 17 17 30 30 30 T F T <---
20922024-04-17T17:11:47.892Z 4 080-099 18 14 18 32 26 32 F F F <---
20932024-04-17T17:11:47.892Z 5 100-119 15 15 15 27 27 27 F F F
20942024-04-17T17:11:47.893Z 6 120-139 18 15 18 32 27 32 F F F <---
20952024-04-17T17:11:47.893Z 7 140-159 2 2 2 3 3 3 F F F
20962024-04-17T17:11:47.893Z 8 160-179 17 17 17 30 30 30 F F F
20972024-04-17T17:11:47.893Z 9 180-199 14 14 14 26 26 26 F F F
20982024-04-17T17:11:47.893Z 10 200-219 18 14 18 35 26 35 F F F <---
20992024-04-17T17:11:47.893Z 11 220-239 18 16 18 35 29 35 F F F <---
21002024-04-17T17:11:47.893Z 12 240-259 15 15 15 27 27 27 F F F
21012024-04-17T17:11:47.894Z 13 260-279 16 16 16 29 29 29 F F F
21022024-04-17T17:11:47.894Z 14 280-299 4 4 4 6 6 6 F F F
21032024-04-17T17:11:47.894Z 15 300-319 15 15 15 27 27 27 F F F
21042024-04-17T17:11:47.894Z 16 320-339 16 16 16 29 29 29 F F F
21052024-04-17T17:11:47.895Z 17 340-359 14 14 14 26 26 26 F F F
21062024-04-17T17:11:47.895Z 18 360-379 18 17 18 35 30 35 T F T <---
21072024-04-17T17:11:47.895Z 19 380-399 8 8 8 15 15 15 F F F
21082024-04-17T17:11:47.895Z 20 400-419 18 16 18 35 29 35 F F F <---
21092024-04-17T17:11:47.895Z 21 420-439 18 15 18 35 27 35 F F F <---
21102024-04-17T17:11:47.895Z 22 440-459 18 15 18 34 27 34 F F F <---
21112024-04-17T17:11:47.895Z 23 460-479 18 16 18 34 29 34 F F F <---
21122024-04-17T17:11:47.896Z 24 480-499 18 16 18 35 29 35 F F F <---
21132024-04-17T17:11:47.896Z 25 500-519 12 12 12 21 21 21 F F F
21142024-04-17T17:11:47.896Z 26 520-539 18 10 18 32 19 32 F F F <---
21152024-04-17T17:11:47.896Z 27 540-559 18 10 18 35 19 35 F F F <---
21162024-04-17T17:11:47.896Z 28 560-579 18 17 18 35 30 35 F F F <---
21172024-04-17T17:11:47.896Z 29 580-599 14 14 14 26 26 26 F F F
21182024-04-17T17:11:47.896ZMax gen: 18, Max flush: 35
21192024-04-17T17:11:47.896ZError: Difference in extent metadata found!
21202024-04-17T17:11:47.907ZA Difference in extent metadata is expected here
21212024-04-17T17:11:47.907ZOn loop 9
21222024-04-17T17:11:47.907Z
21232024-04-17T17:11:47.907ZVerifying data now
21242024-04-17T17:11:52.597ZNo change after verify
21252024-04-17T17:11:52.601ZLoop: 9 Downstairs dump after verify (and repair):
21262024-04-17T17:11:52.611ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
21272024-04-17T17:11:52.611Z 0 000-019 19 19 19 36 36 36 F F F
21282024-04-17T17:11:52.618Z 1 020-039 18 18 18 34 34 34 F F F
21292024-04-17T17:11:52.618Z 2 040-059 16 16 16 28 28 28 F F F
21302024-04-17T17:11:52.618Z 3 060-079 19 19 19 36 36 36 F F F
21312024-04-17T17:11:52.618Z 4 080-099 18 18 18 32 32 32 F F F
21322024-04-17T17:11:52.618Z 5 100-119 15 15 15 27 27 27 F F F
21332024-04-17T17:11:52.618Z 6 120-139 18 18 18 32 32 32 F F F
21342024-04-17T17:11:52.618Z 7 140-159 2 2 2 3 3 3 F F F
21352024-04-17T17:11:52.619Z 8 160-179 17 17 17 30 30 30 F F F
21362024-04-17T17:11:52.619Z 9 180-199 14 14 14 26 26 26 F F F
21372024-04-17T17:11:52.619Z 10 200-219 18 18 18 35 35 35 F F F
21382024-04-17T17:11:52.619Z 11 220-239 18 18 18 35 35 35 F F F
21392024-04-17T17:11:52.619Z 12 240-259 15 15 15 27 27 27 F F F
21402024-04-17T17:11:52.619Z 13 260-279 16 16 16 29 29 29 F F F
21412024-04-17T17:11:52.619Z 14 280-299 4 4 4 6 6 6 F F F
21422024-04-17T17:11:52.619Z 15 300-319 15 15 15 27 27 27 F F F
21432024-04-17T17:11:52.619Z 16 320-339 16 16 16 29 29 29 F F F
21442024-04-17T17:11:52.619Z 17 340-359 14 14 14 26 26 26 F F F
21452024-04-17T17:11:52.619Z 18 360-379 19 19 19 36 36 36 F F F
21462024-04-17T17:11:52.619Z 19 380-399 8 8 8 15 15 15 F F F
21472024-04-17T17:11:52.619Z 20 400-419 18 18 18 35 35 35 F F F
21482024-04-17T17:11:52.619Z 21 420-439 18 18 18 35 35 35 F F F
21492024-04-17T17:11:52.619Z 22 440-459 18 18 18 34 34 34 F F F
21502024-04-17T17:11:52.619Z 23 460-479 18 18 18 34 34 34 F F F
21512024-04-17T17:11:52.619Z 24 480-499 18 18 18 35 35 35 F F F
21522024-04-17T17:11:52.619Z 25 500-519 12 12 12 21 21 21 F F F
21532024-04-17T17:11:52.619Z 26 520-539 18 18 18 32 32 32 F F F
21542024-04-17T17:11:52.619Z 27 540-559 18 18 18 35 35 35 F F F
21552024-04-17T17:11:52.619Z 28 560-579 18 18 18 35 35 35 F F F
21562024-04-17T17:11:52.619Z 29 580-599 14 14 14 26 26 26 F F F
21572024-04-17T17:11:52.620ZMax gen: 19, Max flush: 36
21582024-04-17T17:11:52.624Z
21592024-04-17T17:11:52.624ZBegin loop 10
21602024-04-17T17:11:52.624ZDownstairs to restart: 2
21612024-04-17T17:11:52.635Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.241401737Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248}
21622024-04-17T17:11:52.642Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.241706516Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248}
21632024-04-17T17:11:52.642Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
21642024-04-17T17:11:52.642Z2024-04-17T17:11:51.241760189Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248}
21652024-04-17T17:11:52.642Z{"msg":"Crucible 0afeae5b-4e03-45d9-9ded-2467467344c9 has session id: 5366edeb-e3e5-4aa0-ae27-213f1dd1aca8","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.241807422Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21662024-04-17T17:11:52.642Z{"msg":"Upstairs opts: Upstairs UUID: 0afeae5b-4e03-45d9-9ded-2467467344c9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.241862755Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21672024-04-17T17:11:52.642Z{"msg":"Crucible stats registered with UUID: 0afeae5b-4e03-45d9-9ded-2467467344c9","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.241890937Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21682024-04-17T17:11:52.642Z{"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.241926649Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248}
21692024-04-17T17:11:52.642Z{"msg":"0afeae5b-4e03-45d9-9ded-2467467344c9 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.24194331Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21702024-04-17T17:11:52.642Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.241986673Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"client":"0","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21712024-04-17T17:11:52.642Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.242058057Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"client":"0","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21722024-04-17T17:11:52.642Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.24211148Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"client":"1","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21732024-04-17T17:11:52.642Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.242151403Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"client":"1","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21742024-04-17T17:11:52.642Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.242294751Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"client":"2","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21752024-04-17T17:11:52.642Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.242339784Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"client":"2","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21762024-04-17T17:11:52.642Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.242367206Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21772024-04-17T17:11:52.642Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.242405638Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21782024-04-17T17:11:52.642Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.24242912Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21792024-04-17T17:11:52.642Z{"msg":"ds_connection connected from Ok(127.0.0.1:53244)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.242460492Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21802024-04-17T17:11:52.643Z{"msg":"ds_connection connected from Ok(127.0.0.1:42377)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.242484223Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21812024-04-17T17:11:52.643Z{"msg":"ds_connection connected from Ok(127.0.0.1:54149)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.242508695Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21822024-04-17T17:11:52.643Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.242867376Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"client":"0","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21832024-04-17T17:11:52.643Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.242904199Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"client":"0","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21842024-04-17T17:11:52.643Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.242992254Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"client":"1","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21852024-04-17T17:11:52.643Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.243063818Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"client":"1","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21862024-04-17T17:11:52.643Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.243144723Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"client":"2","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21872024-04-17T17:11:52.643Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.243178005Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"client":"2","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21882024-04-17T17:11:52.643Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.243524206Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"client":"1","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21892024-04-17T17:11:52.643Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.24359807Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"client":"0","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21902024-04-17T17:11:52.643Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.243786272Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"client":"2","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21912024-04-17T17:11:52.643Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.244065199Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"client":"1","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21922024-04-17T17:11:52.643Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.244090971Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"client":"1","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21932024-04-17T17:11:52.643Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.244105881Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21942024-04-17T17:11:52.643Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.244150334Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"client":"0","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21952024-04-17T17:11:52.643Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.244209488Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"client":"0","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21962024-04-17T17:11:52.643Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.244260091Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21972024-04-17T17:11:52.643Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.244287182Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"client":"2","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21982024-04-17T17:11:52.643Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.244317894Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"client":"2","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
21992024-04-17T17:11:52.643Z{"msg":"[0]R flush_numbers[0..12]: [36, 34, 28, 36, 32, 27, 32, 3, 30, 26, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.244333845Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"":"downstairs","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
22002024-04-17T17:11:52.643Z{"msg":"[0]R generation[0..12]: [19, 18, 16, 19, 18, 15, 18, 2, 17, 14, 18, 18]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.244361317Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"":"downstairs","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
22012024-04-17T17:11:52.643Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.244376528Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"Wait for a query_work_queue command to finish before sending IO
22022024-04-17T17:11:52.644Z":"downstairs","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
22032024-04-17T17:11:52.644Z{"msg":"[1]R flush_numbers[0..12]: [36, 34, 28, 36, 32, 27, 32, 3, 30, 26, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.244404139Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"":"downstairs","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
22042024-04-17T17:11:52.644Z{"msg":"[1]R generation[0..12]: [19, 18, 16, 19, 18, 15, 18, 2, 17, 14, 18, 18]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.2444187Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"":"downstairs","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
22052024-04-17T17:11:52.644Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.244433751Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"":"downstairs","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
22062024-04-17T17:11:52.644Z{"msg":"[2]R flush_numbers[0..12]: [36, 34, 28, 36, 32, 27, 32, 3, 30, 26, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.244447732Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"":"downstairs","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
22072024-04-17T17:11:52.644Z{"msg":"[2]R generation[0..12]: [19, 18, 16, 19, 18, 15, 18, 2, 17, 14, 18, 18]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.244461213Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"":"downstairs","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
22082024-04-17T17:11:52.644Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.244475574Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"":"downstairs","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
22092024-04-17T17:11:52.644Z{"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.244488635Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"":"downstairs","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
22102024-04-17T17:11:52.644Z{"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.244502425Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"":"downstairs","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
22112024-04-17T17:11:52.644Z{"msg":"Next flush: 37","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.244522457Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"":"downstairs","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
22122024-04-17T17:11:52.644Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.244535617Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"":"downstairs","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
22132024-04-17T17:11:52.644Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.244549168Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
22142024-04-17T17:11:52.644Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.244560899Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"client":"0","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
22152024-04-17T17:11:52.644Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.24457513Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"client":"1","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
22162024-04-17T17:11:52.644Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.24458817Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"client":"2","":"downstairs","session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
22172024-04-17T17:11:52.644Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.244601011Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
22182024-04-17T17:11:52.644Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.244620172Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
22192024-04-17T17:11:52.644Z{"msg":"0afeae5b-4e03-45d9-9ded-2467467344c9 is now active with session: 5366edeb-e3e5-4aa0-ae27-213f1dd1aca8","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.244633113Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
22202024-04-17T17:11:52.644Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.244656875Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248,"session_id":"5366edeb-e3e5-4aa0-ae27-213f1dd1aca8"}
22212024-04-17T17:11:52.644Z{"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:51.244669135Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248}
22222024-04-17T17:11:52.644ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
22232024-04-17T17:11:52.644ZLoading write count information from file "/tmp/test_repair_verify.data"
22242024-04-17T17:11:52.645ZRun Repair workload
22252024-04-17T17:11:52.648Z01/30 Write block 131 len 8192 data: 3 4
22262024-04-17T17:11:52.648Z02/30 Flush
22272024-04-17T17:11:52.649Z03/30 Flush
22282024-04-17T17:11:52.649Z04/30 Write block 32 len 40960 data: 5 5 5 5 4 4 3 3 4 4
22292024-04-17T17:11:52.649Z05/30 Write block 223 len 20480 data: 2 2 2 2 3
22302024-04-17T17:11:52.649Z06/30 Write block 189 len 24576 data: 2 2 2 2 2 2
22312024-04-17T17:11:52.649Z07/30 Write block 31 len 20480 data: 5 6 6 6 6
22322024-04-17T17:11:52.649Z08/30 Write block 507 len 32768 data: 2 2 2 2 2 2 2 3
22332024-04-17T17:11:52.652Z09/30 Write block 337 len 40960 data: 3 3 3 2 3 3 3 3 3 3
22342024-04-17T17:11:52.652Z10/30 Read block 525 len 4096
22352024-04-17T17:11:52.652Z11/30 Write block 266 len 4096 data: 2
22362024-04-17T17:11:52.652Z12/30 Read block 410 len 8192
22372024-04-17T17:11:52.652Z13/30 Write block 71 len 28672 data: 2 2 2 3 2 2 2
22382024-04-17T17:11:52.656Z14/30 Read block 269 len 4096
22392024-04-17T17:11:52.656Z15/30 Flush
22402024-04-17T17:11:52.659Z16/30 Write block 342 len 36864 data: 4 4 4 4 4 4 4 4 4
22412024-04-17T17:11:52.659Z17/30 Flush
22422024-04-17T17:11:52.663Z18/30 Write block 5 len 40960 data: 4 4 3 3 3 4 4 4 4 4
22432024-04-17T17:11:52.663Z19/30 Write block 273 len 4096 data: 4
22442024-04-17T17:11:52.663Z20/30 Write block 226 len 8192 data: 3 4
22452024-04-17T17:11:52.667Z21/30 Write block 82 len 24576 data: 2 2 2 2 2 3
22462024-04-17T17:11:52.667Z22/30 Write block 312 len 40960 data: 4 3 3 3 3 3 3 4 4 3
22472024-04-17T17:11:52.667Z23/30 Write block 187 len 8192 data: 2 2
22482024-04-17T17:11:52.671Z24/30 Write block 333 len 32768 data: 2 3 4 3 4 4 4 3
22492024-04-17T17:11:52.671Z25/30 Read block 462 len 16384
22502024-04-17T17:11:52.671Z26/30 Write block 324 len 24576 data: 2 2 2 2 3 3
22512024-04-17T17:11:52.671Z27/30 Read block 475 len 20480
22522024-04-17T17:11:52.671Z28/30 Write block 293 len 4096 data: 2
22532024-04-17T17:11:52.676Z29/30 Write block 435 len 36864 data: 2 2 2 2 3 3 3 3 3
22542024-04-17T17:11:52.676Z30/30 Write block 592 len 32768 data: 4 3 3 3 3 3 3 3
22552024-04-17T17:11:52.676Z----------------------------------------------------------------
22562024-04-17T17:11:52.676Z Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:28
22572024-04-17T17:11:52.676ZStates: Active Active Active
22582024-04-17T17:11:52.676ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
22592024-04-17T17:11:52.676Z 3 Acked 1002 Flush 0 Done Done Sent false
22602024-04-17T17:11:52.676Z 4 Acked 1003 Write 10 Done Done Sent false
22612024-04-17T17:11:52.676Z 5 Acked 1004 Write 5 Done Done Sent false
22622024-04-17T17:11:52.676Z 6 Acked 1005 Write 6 Done Done Sent false
22632024-04-17T17:11:52.676Z 7 Acked 1006 Write 5 Done Done Sent false
22642024-04-17T17:11:52.676Z 8 Acked 1007 Write 8 Done Done Sent false
22652024-04-17T17:11:52.676Z 9 Acked 1008 Write 10 Done Done Sent false
22662024-04-17T17:11:52.676Z 10 Acked 1009 Read 1 Done Done Sent false
22672024-04-17T17:11:52.676Z 11 Acked 1010 Write 1 Done Done Sent false
22682024-04-17T17:11:52.676Z 12 Acked 1011 Read 2 Done Done Sent false
22692024-04-17T17:11:52.676Z 13 Acked 1012 Write 7 Done Done Sent false
22702024-04-17T17:11:52.676Z 14 Acked 1013 Read 1 Done Done Sent false
22712024-04-17T17:11:52.676Z 15 Acked 1014 Flush 0 Done Done Sent false
22722024-04-17T17:11:52.676Z 16 Acked 1015 Write 9 Done Done Sent false
22732024-04-17T17:11:52.676Z 17 Acked 1016 Flush 0 Done Done Sent false
22742024-04-17T17:11:52.677Z 18 Acked 1017 Write 10 Done Done Sent false
22752024-04-17T17:11:52.677Z 19 Acked 1018 Write 1 Done Done Sent false
22762024-04-17T17:11:52.677Z 20 Acked 1019 Write 2 Done Done Sent false
22772024-04-17T17:11:52.677Z 21 Acked 1020 Write 6 Done Done Sent false
22782024-04-17T17:11:52.677Z 22 Acked 1021 Write 10 Done Done Sent false
22792024-04-17T17:11:52.677Z 23 Acked 1022 Write 2 Done Done Sent false
22802024-04-17T17:11:52.677Z 24 Acked 1023 Write 8 Done Done Sent false
22812024-04-17T17:11:52.677Z 25 Acked 1024 Read 4 Done Done Sent false
22822024-04-17T17:11:52.677Z 26 Acked 1025 Write 6 Done Done Sent false
22832024-04-17T17:11:52.677Z 27 Acked 1026 Read 5 Done Done Sent false
22842024-04-17T17:11:52.677Z 28 Acked 1027 Write 1 Done Done Sent false
22852024-04-17T17:11:52.677Z 29 Acked 1028 Write 9 Done Done Sent false
22862024-04-17T17:11:52.677Z 30 Acked 1029 Write 8 Sent Sent Sent false
22872024-04-17T17:11:52.677Z STATES DS:0 DS:1 DS:2 TOTAL
22882024-04-17T17:11:52.677Z New 0 0 0 0
22892024-04-17T17:11:52.677Z Sent 1 1 28 30
22902024-04-17T17:11:52.677Z Done 27 27 0 54
22912024-04-17T17:11:52.677Z Skipped 0 0 0 0
22922024-04-17T17:11:52.677Z Error 0 0 0 0
22932024-04-17T17:11:52.677ZLast Flush: 1016 1016 1001
22942024-04-17T17:11:52.677ZDownstairs last five completed: 1001 1000
22952024-04-17T17:11:52.677ZUpstairs last five completed: 30 29 28 27 26
22962024-04-17T17:11:52.677Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:11:51.280429263Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1248}
22972024-04-17T17:11:52.678ZWrote out file "/tmp/test_repair_verify.data"
22982024-04-17T17:11:52.682Z
22992024-04-17T17:11:52.682ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
23002024-04-17T17:11:52.700ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23012024-04-17T17:11:52.700Z 0 000-019 19 19 19 36 36 36 T T F <---
23022024-04-17T17:11:52.700Z 1 020-039 20 20 18 39 39 34 F F F <---
23032024-04-17T17:11:52.700Z 2 040-059 20 20 16 39 39 28 F F F <---
23042024-04-17T17:11:52.700Z 3 060-079 20 20 19 39 39 36 F F F <---
23052024-04-17T17:11:52.700Z 4 080-099 18 18 18 32 32 32 T T F <---
23062024-04-17T17:11:52.704Z 5 100-119 15 15 15 27 27 27 F F F
23072024-04-17T17:11:52.704Z 6 120-139 20 20 20 37 37 37 F F F
23082024-04-17T17:11:52.704Z 7 140-159 2 2 2 3 3 3 F F F
23092024-04-17T17:11:52.704Z 8 160-179 17 17 17 30 30 30 F F F
23102024-04-17T17:11:52.704Z 9 180-199 20 20 14 39 39 26 T T F <---
23112024-04-17T17:11:52.704Z 10 200-219 18 18 18 35 35 35 F F F
23122024-04-17T17:11:52.705Z 11 220-239 20 20 18 39 39 35 T T F <---
23132024-04-17T17:11:52.705Z 12 240-259 15 15 15 27 27 27 F F F
23142024-04-17T17:11:52.705Z 13 260-279 20 20 16 39 39 29 T T F <---
23152024-04-17T17:11:52.705Z 14 280-299 4 4 4 6 6 6 T T F <---
23162024-04-17T17:11:52.705Z 15 300-319 15 15 15 27 27 27 T T F <---
23172024-04-17T17:11:52.705Z 16 320-339 20 20 16 39 39 29 T T F <---
23182024-04-17T17:11:52.705Z 17 340-359 20 20 14 40 40 26 T T F <---
23192024-04-17T17:11:52.705Z 18 360-379 19 19 19 36 36 36 F F F
23202024-04-17T17:11:52.705Z 19 380-399 8 8 8 15 15 15 F F F
23212024-04-17T17:11:52.705Z 20 400-419 18 18 18 35 35 35 F F F
23222024-04-17T17:11:52.705Z 21 420-439 18 18 18 35 35 35 T T F <---
23232024-04-17T17:11:52.705Z 22 440-459 18 18 18 34 34 34 T T F <---
23242024-04-17T17:11:52.705Z 23 460-479 18 18 18 34 34 34 F F F
23252024-04-17T17:11:52.705Z 24 480-499 18 18 18 35 35 35 F F F
23262024-04-17T17:11:52.705Z 25 500-519 20 20 12 39 39 21 F F F <---
23272024-04-17T17:11:52.705Z 26 520-539 18 18 18 32 32 32 F F F
23282024-04-17T17:11:52.705Z 27 540-559 18 18 18 35 35 35 F F F
23292024-04-17T17:11:52.705Z 28 560-579 18 18 18 35 35 35 F F F
23302024-04-17T17:11:52.705Z 29 580-599 14 14 14 26 26 26 T T F <---
23312024-04-17T17:11:52.705ZMax gen: 20, Max flush: 40
23322024-04-17T17:11:52.705ZError: Difference in extent metadata found!
23332024-04-17T17:11:52.705ZA Difference in extent metadata is expected here
23342024-04-17T17:11:52.705ZOn loop 10
23352024-04-17T17:11:52.705Z
23362024-04-17T17:11:52.709ZVerifying data now
23372024-04-17T17:11:57.448ZNo change after verify
23382024-04-17T17:11:57.452ZLoop: 10 Downstairs dump after verify (and repair):
23392024-04-17T17:11:57.463ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23402024-04-17T17:11:57.463Z 0 000-019 21 21 21 41 41 41 F F F
23412024-04-17T17:11:57.468Z 1 020-039 20 20 20 39 39 39 F F F
23422024-04-17T17:11:57.468Z 2 040-059 20 20 20 39 39 39 F F F
23432024-04-17T17:11:57.468Z 3 060-079 20 20 20 39 39 39 F F F
23442024-04-17T17:11:57.468Z 4 080-099 21 21 21 41 41 41 F F F
23452024-04-17T17:11:57.468Z 5 100-119 15 15 15 27 27 27 F F F
23462024-04-17T17:11:57.468Z 6 120-139 20 20 20 37 37 37 F F F
23472024-04-17T17:11:57.469Z 7 140-159 2 2 2 3 3 3 F F F
23482024-04-17T17:11:57.469Z 8 160-179 17 17 17 30 30 30 F F F
23492024-04-17T17:11:57.469Z 9 180-199 21 21 21 41 41 41 F F F
23502024-04-17T17:11:57.469Z 10 200-219 18 18 18 35 35 35 F F F
23512024-04-17T17:11:57.469Z 11 220-239 21 21 21 41 41 41 F F F
23522024-04-17T17:11:57.469Z 12 240-259 15 15 15 27 27 27 F F F
23532024-04-17T17:11:57.469Z 13 260-279 21 21 21 41 41 41 F F F
23542024-04-17T17:11:57.469Z 14 280-299 21 21 21 41 41 41 F F F
23552024-04-17T17:11:57.469Z 15 300-319 21 21 21 41 41 41 F F F
23562024-04-17T17:11:57.469Z 16 320-339 21 21 21 41 41 41 F F F
23572024-04-17T17:11:57.469Z 17 340-359 21 21 21 41 41 41 F F F
23582024-04-17T17:11:57.470Z 18 360-379 19 19 19 36 36 36 F F F
23592024-04-17T17:11:57.470Z 19 380-399 8 8 8 15 15 15 F F F
23602024-04-17T17:11:57.470Z 20 400-419 18 18 18 35 35 35 F F F
23612024-04-17T17:11:57.470Z 21 420-439 21 21 21 41 41 41 F F F
23622024-04-17T17:11:57.470Z 22 440-459 21 21 21 41 41 41 F F F
23632024-04-17T17:11:57.470Z 23 460-479 18 18 18 34 34 34 F F F
23642024-04-17T17:11:57.470Z 24 480-499 18 18 18 35 35 35 F F F
23652024-04-17T17:11:57.470Z 25 500-519 20 20 20 39 39 39 F F F
23662024-04-17T17:11:57.470Z 26 520-539 18 18 18 32 32 32 F F F
23672024-04-17T17:11:57.470Z 27 540-559 18 18 18 35 35 35 F F F
23682024-04-17T17:11:57.470Z 28 560-579 18 18 18 35 35 35 F F F
23692024-04-17T17:11:57.470Z 29 580-599 21 21 21 41 41 41 F F F
23702024-04-17T17:11:57.470ZMax gen: 21, Max flush: 41
23712024-04-17T17:11:57.471Z
23722024-04-17T17:11:57.471ZBegin loop 11
23732024-04-17T17:11:57.471ZDownstairs to restart: 0
23742024-04-17T17:11:57.491Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.092094724Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256}
23752024-04-17T17:11:57.498Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.092326548Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256}
23762024-04-17T17:11:57.498Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.092364941Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256}
23772024-04-17T17:11:57.498Z{"msg":"Crucible a2e8ddb8-39f7-4849-8d8f-9e28539b19db has session id: ef19ce71-ca09-431f-8be7-ab23e3fd35d6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.092394423Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}Crucible runtime is spawned
23782024-04-17T17:11:57.498Z
23792024-04-17T17:11:57.498Z{"msg":"Upstairs opts: Upstairs UUID: a2e8ddb8-39f7-4849-8d8f-9e28539b19db, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.092468327Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
23802024-04-17T17:11:57.498Z{"msg":"Crucible stats registered with UUID: a2e8ddb8-39f7-4849-8d8f-9e28539b19db","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.092502959Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
23812024-04-17T17:11:57.498Z{"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.092529381Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256}
23822024-04-17T17:11:57.498Z{"msg":"a2e8ddb8-39f7-4849-8d8f-9e28539b19db active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.092580684Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
23832024-04-17T17:11:57.498Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.092601045Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"client":"0","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
23842024-04-17T17:11:57.498Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.092645488Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"client":"0","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
23852024-04-17T17:11:57.499Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.092693871Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"client":"1","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
23862024-04-17T17:11:57.499Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.092725903Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"client":"1","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
23872024-04-17T17:11:57.499Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.092755844Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"client":"2","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
23882024-04-17T17:11:57.499Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.092787606Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"client":"2","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
23892024-04-17T17:11:57.499Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.092837319Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
23902024-04-17T17:11:57.499Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.092880592Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
23912024-04-17T17:11:57.499Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.092924395Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
23922024-04-17T17:11:57.499Z{"msg":"ds_connection connected from Ok(127.0.0.1:37379)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.092976278Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
23932024-04-17T17:11:57.499Z{"msg":"ds_connection connected from Ok(127.0.0.1:43040)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.093027091Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
23942024-04-17T17:11:57.499Z{"msg":"ds_connection connected from Ok(127.0.0.1:48281)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.093087725Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
23952024-04-17T17:11:57.499Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.093856271Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"client":"1","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
23962024-04-17T17:11:57.499Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.093892733Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"client":"1","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
23972024-04-17T17:11:57.499Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.093931726Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"client":"2","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
23982024-04-17T17:11:57.499Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.093946557Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"client":"2","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
23992024-04-17T17:11:57.499Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.093959987Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"client":"0","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24002024-04-17T17:11:57.499Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.093981059Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"client":"0","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24012024-04-17T17:11:57.499Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.094444197Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"client":"2","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24022024-04-17T17:11:57.499Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.09449735Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"client":"1","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24032024-04-17T17:11:57.499Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.094531512Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"client":"0","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24042024-04-17T17:11:57.499Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.094969119Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"client":"2","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24052024-04-17T17:11:57.499Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.095016561Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"client":"2","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24062024-04-17T17:11:57.499Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.095044063Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24072024-04-17T17:11:57.500Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.095104647Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"client":"0","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24082024-04-17T17:11:57.500Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.095133158Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"client":"0","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24092024-04-17T17:11:57.500Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.095147569Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24102024-04-17T17:11:57.500Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.095166711Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"client":"1","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24112024-04-17T17:11:57.500Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.095180551Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"client":"1","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24122024-04-17T17:11:57.500Z{"msg":"[0]R flush_numbers[0..12]: [41, 39, 39, 39, 41, 27, 37, 3, 30, 41, 35, 41]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.095200653Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"":"downstairs","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24132024-04-17T17:11:57.500Z{"msg":"[0]R generation[0..12]: [21, 20, 20, 20, 21, 15, 20, 2, 17, 21, 18, 21]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.095214613Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"":"downstairs","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24142024-04-17T17:11:57.500Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.095235385Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"":"downstairs","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24152024-04-17T17:11:57.500Z{"msg":"[1]R flush_numbers[0..12]: [41, 39, 39, 39, 41, 27, 37, 3, 30, 41, 35, 41]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.095249496Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"":"downstairs","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24162024-04-17T17:11:57.500Z{"msg":"[1]R generation[0..12]: [21, 20, 20, 20, 21, 15, 20, 2, 17, 21, 18, 21]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.095269837Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"":"downstairs","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24172024-04-17T17:11:57.500Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.095284067Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"":"downstairs","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24182024-04-17T17:11:57.500Z{"msg":"[2]R flush_numbers[0..12]: [41, 39, 39, 39, 41, 27, 37, 3, 30, 41, 35, 41]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.095308059Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"":"downstairs","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24192024-04-17T17:11:57.500Z{"msg":"[2]R generation[0..12]: [21, 20, 20, 20, 21, 15, 20, 2, 17, 21, 18, 21]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
24202024-04-17T17:11:57.500Z,"time":"2024-04-17T17:11:56.09533257Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"":"downstairs","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24212024-04-17T17:11:57.500Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.095369153Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"":"downstairs","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24222024-04-17T17:11:57.500Z{"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.095385114Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"":"downstairs","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24232024-04-17T17:11:57.500Z{"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.095416176Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"":"downstairs","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24242024-04-17T17:11:57.500Z{"msg":"Next flush: 42","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.095456488Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"":"downstairs","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24252024-04-17T17:11:57.500Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.09549315Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"":"downstairs","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24262024-04-17T17:11:57.500Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.095527192Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24272024-04-17T17:11:57.500Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.095546003Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"client":"0","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24282024-04-17T17:11:57.500Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.095570105Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"client":"1","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24292024-04-17T17:11:57.500Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.095603547Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"client":"2","":"downstairs","session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24302024-04-17T17:11:57.500Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.095620868Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24312024-04-17T17:11:57.501Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.095633729Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24322024-04-17T17:11:57.501Z{"msg":"a2e8ddb8-39f7-4849-8d8f-9e28539b19db is now active with session: ef19ce71-ca09-431f-8be7-ab23e3fd35d6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.0956463Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24332024-04-17T17:11:57.501Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.09566205Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256,"session_id":"ef19ce71-ca09-431f-8be7-ab23e3fd35d6"}
24342024-04-17T17:11:57.501Z{"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:11:56.095696383Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256}
24352024-04-17T17:11:57.501ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
24362024-04-17T17:11:57.501ZLoading write count information from file "/tmp/test_repair_verify.data"
24372024-04-17T17:11:57.501ZRun Repair workload
24382024-04-17T17:11:57.501Z01/30 Write block 219 len 32768 data: 3 3 3 3 3 3 3 4
24392024-04-17T17:11:57.501Z02/30 Flush
24402024-04-17T17:11:57.501Z03/30 Write block 138 len 24576 data: 2 2 2 2 2 3
24412024-04-17T17:11:57.506Z04/30 Write block 442 len 36864 data: 4 4 3 3 3 2 2 2 2
24422024-04-17T17:11:57.506Z05/30 Write block 585 len 12288 data: 4 4 3
24432024-04-17T17:11:57.506Z06/30 Write block 353 len 40960 data: 3 3 2 3 3 3 3 3 3 3
24442024-04-17T17:11:57.506Z07/30 Read block 148 len 24576
24452024-04-17T17:11:57.506Z08/30 Read block 387 len 12288
24462024-04-17T17:11:57.506Z09/30 Write block 481 len 12288 data: 4 4 4
24472024-04-17T17:11:57.506Z10/30 Read block 448 len 40960
24482024-04-17T17:11:57.506Z11/30 Read block 465 len 28672
24492024-04-17T17:11:57.506Z12/30 Read block 78 len 20480
24502024-04-17T17:11:57.507Z13/30 Read block 360 len 20480
24512024-04-17T17:11:57.507Z14/30 Read block 274 len 40960
24522024-04-17T17:11:57.507Z15/30 Read block 105 len 24576
24532024-04-17T17:11:57.511Z16/30 Write block 512 len 16384 data: 3 3 4 3
24542024-04-17T17:11:57.511Z17/30 Read block 499 len 8192
24552024-04-17T17:11:57.511Z18/30 Write block 422 len 32768 data: 3 2 2 2 2 2 3 3
24562024-04-17T17:11:57.511Z19/30 Read block 84 len 40960
24572024-04-17T17:11:57.511Z20/30 Read block 193 len 12288
24582024-04-17T17:11:57.511Z21/30 Write block 265 len 4096 data: 2
24592024-04-17T17:11:57.512Z22/30 Read block 309 len 28672
24602024-04-17T17:11:57.516Z23/30 Read block 101 len 24576
24612024-04-17T17:11:57.516Z24/30 Read block 155 len 16384
24622024-04-17T17:11:57.516Z25/30 Read block 408 len 24576
24632024-04-17T17:11:57.516Z26/30 Read block 487 len 24576
24642024-04-17T17:11:57.516Z27/30 Write block 338 len 32768 data: 5 5 4 4 5 5 5 5
24652024-04-17T17:11:57.516Z28/30 Read block 509 len 36864
24662024-04-17T17:11:57.521Z29/30 Read block 548 len 16384
24672024-04-17T17:11:57.521Z30/30 Read block 378 len 32768
24682024-04-17T17:11:57.521Z----------------------------------------------------------------
24692024-04-17T17:11:57.521Z Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:28
24702024-04-17T17:11:57.521ZStates: Active Active Active
24712024-04-17T17:11:57.521ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
24722024-04-17T17:11:57.521Z 3 Acked 1002 Write 6 Sent Done Done false
24732024-04-17T17:11:57.521Z 4 Acked 1003 Write 9 Sent Done Done false
24742024-04-17T17:11:57.521Z 5 Acked 1004 Write 3 Sent Done Done false
24752024-04-17T17:11:57.521Z 6 Acked 1005 Write 10 Sent Done Done false
24762024-04-17T17:11:57.521Z 7 Acked 1006 Read 6 Sent Done Done false
24772024-04-17T17:11:57.521Z 8 Acked 1007 Read 3 Sent Done Done false
24782024-04-17T17:11:57.521Z 9 Acked 1008 Write 3 Sent Done Done false
24792024-04-17T17:11:57.521Z 10 Acked 1009 Read 10 Sent Done Done false
24802024-04-17T17:11:57.521Z 11 Acked 1010 Read 7 Sent Done Done false
24812024-04-17T17:11:57.522Z 12 Acked 1011 Read 5 Sent Done Done false
24822024-04-17T17:11:57.522Z 13 Acked 1012 Read 5 Sent Done Done false
24832024-04-17T17:11:57.522Z 14 Acked 1013 Read 10 Sent Done Done false
24842024-04-17T17:11:57.522Z 15 Acked 1014 Read 6 Sent Done Done false
24852024-04-17T17:11:57.522Z 16 Acked 1015 Write 4 Sent Done Done false
24862024-04-17T17:11:57.522Z 17 Acked 1016 Read 2 Sent Done Done false
24872024-04-17T17:11:57.522Z 18 Acked 1017 Write 8 Sent Done Done false
24882024-04-17T17:11:57.522Z 19 Acked 1018 Read 10 Sent Done Done false
24892024-04-17T17:11:57.522Z 20 Acked 1019 Read 3 Sent Done Done false
24902024-04-17T17:11:57.522Z 21 Acked 1020 Write 1 Sent Done Done false
24912024-04-17T17:11:57.522Z 22 Acked 1021 Read 7 Sent Done Done false
24922024-04-17T17:11:57.522Z 23 Acked 1022 Read 6 Sent Done Done false
24932024-04-17T17:11:57.522Z 24 Acked 1023 Read 4 Sent Done Done false
24942024-04-17T17:11:57.522Z 25 Acked 1024 Read 6 Sent Done Done false
24952024-04-17T17:11:57.522Z 26 Acked 1025 Read 6 Sent Done Done false
24962024-04-17T17:11:57.522Z 27 Acked 1026 Write 8 Sent Done Done false
24972024-04-17T17:11:57.522Z 28 Acked 1027 Read 9 Sent Done Done false
24982024-04-17T17:11:57.522Z 29 Acked 1028 Read 4 Sent Done Done false
24992024-04-17T17:11:57.522Z 30 Acked 1029 Read 8 Sent Done Done false
25002024-04-17T17:11:57.522Z STATES DS:0 DS:1 DS:2 TOTAL
25012024-04-17T17:11:57.523Z New 0 0 0 0
25022024-04-17T17:11:57.523Z Sent 28 0 0 28
25032024-04-17T17:11:57.523Z Done 0 28 28 56
25042024-04-17T17:11:57.523Z Skipped 0 0 0 0
25052024-04-17T17:11:57.523Z Error 0 0 0 0
25062024-04-17T17:11:57.523ZLast Flush: 1001 1001 1001
25072024-04-17T17:11:57.523ZDownstairs last five completed: 1001 1000
25082024-04-17T17:11:57.523ZUpstairs last five completed: 30 29 28 27 26
25092024-04-17T17:11:57.523Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:11:56.124927284Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1256}
25102024-04-17T17:11:57.523ZWrote out file "/tmp/test_repair_verify.data"
25112024-04-17T17:11:57.527Z
25122024-04-17T17:11:57.527ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
25132024-04-17T17:11:57.544ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25142024-04-17T17:11:57.544Z 0 000-019 21 21 21 41 41 41 F F F
25152024-04-17T17:11:57.544Z 1 020-039 20 20 20 39 39 39 F F F
25162024-04-17T17:11:57.544Z 2 040-059 20 20 20 39 39 39 F F F
25172024-04-17T17:11:57.545Z 3 060-079 20 20 20 39 39 39 F F F
25182024-04-17T17:11:57.545Z 4 080-099 21 21 21 41 41 41 F F F
25192024-04-17T17:11:57.545Z 5 100-119 15 15 15 27 27 27 F F F
25202024-04-17T17:11:57.545Z 6 120-139 20 20 20 37 37 37 F T T <---
25212024-04-17T17:11:57.549Z 7 140-159 2 2 2 3 3 3 F T T <---
25222024-04-17T17:11:57.549Z 8 160-179 17 17 17 30 30 30 F F F
25232024-04-17T17:11:57.549Z 9 180-199 21 21 21 41 41 41 F F F
25242024-04-17T17:11:57.549Z 10 200-219 22 22 22 42 42 42 F F F
25252024-04-17T17:11:57.549Z 11 220-239 22 22 22 42 42 42 F F F
25262024-04-17T17:11:57.549Z 12 240-259 15 15 15 27 27 27 F F F
25272024-04-17T17:11:57.549Z 13 260-279 21 21 21 41 41 41 F T T <---
25282024-04-17T17:11:57.549Z 14 280-299 21 21 21 41 41 41 F F F
25292024-04-17T17:11:57.550Z 15 300-319 21 21 21 41 41 41 F F F
25302024-04-17T17:11:57.550Z 16 320-339 21 21 21 41 41 41 F T T <---
25312024-04-17T17:11:57.550Z 17 340-359 21 21 21 41 41 41 F T T <---
25322024-04-17T17:11:57.550Z 18 360-379 19 19 19 36 36 36 F T T <---
25332024-04-17T17:11:57.550Z 19 380-399 8 8 8 15 15 15 F F F
25342024-04-17T17:11:57.550Z 20 400-419 18 18 18 35 35 35 F F F
25352024-04-17T17:11:57.550Z 21 420-439 21 21 21 41 41 41 F T T <---
25362024-04-17T17:11:57.550Z 22 440-459 21 21 21 41 41 41 F T T <---
25372024-04-17T17:11:57.550Z 23 460-479 18 18 18 34 34 34 F F F
25382024-04-17T17:11:57.550Z 24 480-499 18 18 18 35 35 35 F T T <---
25392024-04-17T17:11:57.550Z 25 500-519 20 20 20 39 39 39 F T T <---
25402024-04-17T17:11:57.550Z 26 520-539 18 18 18 32 32 32 F F F
25412024-04-17T17:11:57.551Z 27 540-559 18 18 18 35 35 35 F F F
25422024-04-17T17:11:57.551Z 28 560-579 18 18 18 35 35 35 F F F
25432024-04-17T17:11:57.551Z 29 580-599 21 21 21 41 41 41 F T T <---
25442024-04-17T17:11:57.551ZMax gen: 22, Max flush: 42
25452024-04-17T17:11:57.551ZError: Difference in extent metadata found!
25462024-04-17T17:11:57.551ZA Difference in extent metadata is expected here
25472024-04-17T17:11:57.551ZOn loop 11
25482024-04-17T17:11:57.551Z
25492024-04-17T17:11:57.557ZVerifying data now
25502024-04-17T17:12:02.116ZNo change after verify
25512024-04-17T17:12:02.117ZLoop: 11 Downstairs dump after verify (and repair):
25522024-04-17T17:12:02.136ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25532024-04-17T17:12:02.136Z 0 000-019 21 21 21 41 41 41 F F F
25542024-04-17T17:12:02.136Z 1 020-039 20 20 20 39 39 39 F F F
25552024-04-17T17:12:02.137Z 2 040-059 20 20 20 39 39 39 F F F
25562024-04-17T17:12:02.137Z 3 060-079 20 20 20 39 39 39 F F F
25572024-04-17T17:12:02.137Z 4 080-099 21 21 21 41 41 41 F F F
25582024-04-17T17:12:02.150Z 5 100-119 15 15 15 27 27 27 F F F
25592024-04-17T17:12:02.151Z 6 120-139 23 23 23 43 43 43 F F F
25602024-04-17T17:12:02.151Z 7 140-159 23 23 23 43 43 43 F F F
25612024-04-17T17:12:02.151Z 8 160-179 17 17 17 30 30 30 F F F
25622024-04-17T17:12:02.151Z 9 180-199 21 21 21 41 41 41 F F F
25632024-04-17T17:12:02.151Z 10 200-219 22 22 22 42 42 42 F F F
25642024-04-17T17:12:02.152Z 11 220-239 22 22 22 42 42 42 F F F
25652024-04-17T17:12:02.152Z 12 240-259 15 15 15 27 27 27 F F F
25662024-04-17T17:12:02.152Z 13 260-279 23 23 23 43 43 43 F F F
25672024-04-17T17:12:02.153Z 14 280-299 21 21 21 41 41 41 F F F
25682024-04-17T17:12:02.153Z 15 300-319 21 21 21 41 41 41 F F F
25692024-04-17T17:12:02.153Z 16 320-339 23 23 23 43 43 43 F F F
25702024-04-17T17:12:02.153Z 17 340-359 23 23 23 43 43 43 F F F
25712024-04-17T17:12:02.154Z 18 360-379 23 23 23 43 43 43 F F F
25722024-04-17T17:12:02.154Z 19 380-399 8 8 8 15 15 15 F F F
25732024-04-17T17:12:02.154Z 20 400-419 18 18 18 35 35 35 F F F
25742024-04-17T17:12:02.154Z 21 420-439 23 23 23 43 43 43 F F F
25752024-04-17T17:12:02.155Z 22 440-459 23 23 23 43 43 43 F F F
25762024-04-17T17:12:02.155Z 23 460-479 18 18 18 34 34 34 F F F
25772024-04-17T17:12:02.155Z 24 480-499 23 23 23 43 43 43 F F F
25782024-04-17T17:12:02.155Z 25 500-519 23 23 23 43 43 43 F F F
25792024-04-17T17:12:02.155Z 26 520-539 18 18 18 32 32 32 F F F
25802024-04-17T17:12:02.156Z 27 540-559 18 18 18 35 35 35 F F F
25812024-04-17T17:12:02.156Z 28 560-579 18 18 18 35 35 35 F F F
25822024-04-17T17:12:02.156Z 29 580-599 23 23 23 43 43 43 F F F
25832024-04-17T17:12:02.156ZMax gen: 23, Max flush: 43
25842024-04-17T17:12:02.157Z
25852024-04-17T17:12:02.157ZBegin loop 12
25862024-04-17T17:12:02.157ZDownstairs to restart: 2
25872024-04-17T17:12:02.173Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.758138631Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264}
25882024-04-17T17:12:02.173Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.758396737Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264}
25892024-04-17T17:12:02.174Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.75844301Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264}
25902024-04-17T17:12:02.174ZCrucible runtime is spawned
25912024-04-17T17:12:02.174Z{"msg":"Crucible 6358786f-37a6-4c34-a200-af4bedd8bb5c has session id: ba449628-1f0a-4914-93aa-3e778272e312","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.758493183Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
25922024-04-17T17:12:02.174Z{"msg":"Upstairs opts: Upstairs UUID: 6358786f-37a6-4c34-a200-af4bedd8bb5c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.758549546Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
25932024-04-17T17:12:02.174Z{"msg":"Crucible stats registered with UUID: 6358786f-37a6-4c34-a200-af4bedd8bb5c","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.758579648Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
25942024-04-17T17:12:02.175Z{"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.758597259Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264}
25952024-04-17T17:12:02.175Z{"msg":"6358786f-37a6-4c34-a200-af4bedd8bb5c active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.758621121Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
25962024-04-17T17:12:02.175Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.758636991Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"client":"0","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
25972024-04-17T17:12:02.175Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.758654512Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"client":"0","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
25982024-04-17T17:12:02.175Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.758671264Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"client":"1","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
25992024-04-17T17:12:02.175Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.758702366Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"client":"1","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26002024-04-17T17:12:02.175Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.758726187Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"client":"2","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26012024-04-17T17:12:02.176Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.758756719Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"client":"2","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26022024-04-17T17:12:02.176Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.75877997Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26032024-04-17T17:12:02.176Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.758824083Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26042024-04-17T17:12:02.176Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.758852575Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26052024-04-17T17:12:02.177Z{"msg":"ds_connection connected from Ok(127.0.0.1:49570)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.758902718Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26062024-04-17T17:12:02.177Z{"msg":"ds_connection connected from Ok(127.0.0.1:41837)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.758957401Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26072024-04-17T17:12:02.177Z{"msg":"ds_connection connected from Ok(127.0.0.1:34072)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.759051596Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26082024-04-17T17:12:02.178Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.759678394Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"client":"0","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26092024-04-17T17:12:02.178Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.759734328Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"client":"0","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26102024-04-17T17:12:02.178Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.75976571Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"client":"1","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26112024-04-17T17:12:02.178Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.759818633Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"client":"1","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26122024-04-17T17:12:02.178Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.759966862Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"client":"2","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26132024-04-17T17:12:02.179Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.760012745Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"client":"2","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26142024-04-17T17:12:02.179Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.76042001Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"client":"0","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26152024-04-17T17:12:02.179Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.760474963Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"client":"1","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26162024-04-17T17:12:02.179Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.760658224Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"client":"2","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26172024-04-17T17:12:02.179Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.760944741Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"client":"0","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26182024-04-17T17:12:02.180Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.760982654Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"client":"0","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26192024-04-17T17:12:02.180Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.761041367Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26202024-04-17T17:12:02.180Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.761065949Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"client":"1","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26212024-04-17T17:12:02.180Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.76109714Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"client":"1","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26222024-04-17T17:12:02.181Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.761121422Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26232024-04-17T17:12:02.181Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.761150074Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"client":"2","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26242024-04-17T17:12:02.181Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.761173855Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"client":"2","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26252024-04-17T17:12:02.181Z{"msg":"[0]R flush_numbers[0..12]: [41, 39, 39, 39, 41, 27, 43, 43, 30, 41, 42, 42]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.761206337Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"":"downstairs","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26262024-04-17T17:12:02.182Z{"msg":"[0]R generation[0..12]: [21, 20, 20, 20, 21, 15, 23, 23, 17, 21, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.761232169Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"":"downstairs","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26272024-04-17T17:12:02.182Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.761263821Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"":"downstairs","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26282024-04-17T17:12:02.182Z{"msg":"[1]R flush_numbers[0..12]: [41, 39, 39, 39, 41, 27, 43, 43, 30, 41, 42, 42]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.761288992Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"":"downstairs","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26292024-04-17T17:12:02.182Z{"msg":"Wait for a query_work_queue command to finish before sending IO
26302024-04-17T17:12:02.183Z[1]R generation[0..12]: [21, 20, 20, 20, 21, 15, 23, 23, 17, 21, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.761322324Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"":"downstairs","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26312024-04-17T17:12:02.183Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.761348666Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"":"downstairs","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26322024-04-17T17:12:02.183Z{"msg":"[2]R flush_numbers[0..12]: [41, 39, 39, 39, 41, 27, 43, 43, 30, 41, 42, 42]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.761378328Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"":"downstairs","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26332024-04-17T17:12:02.183Z{"msg":"[2]R generation[0..12]: [21, 20, 20, 20, 21, 15, 23, 23, 17, 21, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.761403349Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"":"downstairs","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26342024-04-17T17:12:02.183Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.761444292Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"":"downstairs","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26352024-04-17T17:12:02.184Z{"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.761464553Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"":"downstairs","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26362024-04-17T17:12:02.184Z{"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.761491375Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"":"downstairs","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26372024-04-17T17:12:02.184Z{"msg":"Next flush: 44","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.761510466Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"":"downstairs","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26382024-04-17T17:12:02.184Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.761529727Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"":"downstairs","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26392024-04-17T17:12:02.184Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.761547838Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26402024-04-17T17:12:02.185Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.761564299Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"client":"0","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26412024-04-17T17:12:02.185Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
26422024-04-17T17:12:02.185Z,"time":"2024-04-17T17:12:00.76158548Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"client":"1","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26432024-04-17T17:12:02.185Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.761617072Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"client":"2","":"downstairs","session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26442024-04-17T17:12:02.185Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.761649204Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26452024-04-17T17:12:02.186Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.761671865Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26462024-04-17T17:12:02.186Z{"msg":"6358786f-37a6-4c34-a200-af4bedd8bb5c is now active with session: ba449628-1f0a-4914-93aa-3e778272e312","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.761698567Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26472024-04-17T17:12:02.186Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.761719788Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264,"session_id":"ba449628-1f0a-4914-93aa-3e778272e312"}
26482024-04-17T17:12:02.186Z{"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:00.76174085Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264}
26492024-04-17T17:12:02.186ZLoading write count information from file "/tmp/test_repair_verify.data"
26502024-04-17T17:12:02.186ZRun Repair workload
26512024-04-17T17:12:02.187Z01/30 Write block 285 len 36864 data: 3 3 3 3 3 3 3 3 3
26522024-04-17T17:12:02.187Z02/30 Write block 155 len 12288 data: 2 2 2
26532024-04-17T17:12:02.187Z03/30 Write block 178 len 24576 data: 3 3 2 2 2 2
26542024-04-17T17:12:02.187Z04/30 Write block 444 len 28672 data: 4 4 4 3 3 3 3
26552024-04-17T17:12:02.188Z05/30 Write block 355 len 28672 data: 3 4 4 4 4 4 4
26562024-04-17T17:12:02.188Z06/30 Read block 582 len 32768
26572024-04-17T17:12:02.202Z07/30 Write block 410 len 40960 data: 4 5 6 6 6 5 4 4 4 4
26582024-04-17T17:12:02.202Z08/30 Write block 452 len 8192 data: 2 3
26592024-04-17T17:12:02.202Z09/30 Read block 319 len 20480
26602024-04-17T17:12:02.202Z10/30 Read block 118 len 8192
26612024-04-17T17:12:02.203Z11/30 Read block 115 len 32768
26622024-04-17T17:12:02.203Z12/30 Write block 99 len 40960 data: 2 2 2 2 2 2 2 3 3 3
26632024-04-17T17:12:02.203Z13/30 Write block 299 len 32768 data: 2 2 3 2 3 3 4 3
26642024-04-17T17:12:02.203Z14/30 Write block 495 len 40960 data: 2 3 3 3 3 4 4 4 3 3
26652024-04-17T17:12:02.204Z15/30 Flush
26662024-04-17T17:12:02.204Z16/30 Write block 0 len 40960 data: 3 3 3 3 4 5 5 4 4 4
26672024-04-17T17:12:02.204Z17/30 Read block 175 len 24576
26682024-04-17T17:12:02.204Z18/30 Read block 99 len 32768
26692024-04-17T17:12:02.204Z19/30 Write block 87 len 36864 data: 4 3 4 4 5 5 5 4 4
26702024-04-17T17:12:02.205Z20/30 Write block 523 len 4096 data: 2
26712024-04-17T17:12:02.205Z21/30 Read block 65 len 20480
26722024-04-17T17:12:02.205Z22/30 Write block 438 len 36864 data: 3 4 4 4 5 5 5 5 5
26732024-04-17T17:12:02.205Z23/30 Read block 388 len 28672
26742024-04-17T17:12:02.206Z24/30 Read block 510 len 40960
26752024-04-17T17:12:02.206Z25/30 Read block 404 len 28672
26762024-04-17T17:12:02.206Z26/30 Write block 279 len 24576 data: 2 2 2 2 2 3
26772024-04-17T17:12:02.206Z27/30 Read block 542 len 24576
26782024-04-17T17:12:02.206Z28/30 Read block 445 len 16384
26792024-04-17T17:12:02.206Z29/30 Read block 315 len 16384
26802024-04-17T17:12:02.206Z30/30 Read block 464 len 28672
26812024-04-17T17:12:02.206Z----------------------------------------------------------------
26822024-04-17T17:12:02.206Z Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:30
26832024-04-17T17:12:02.206ZStates: Active Active Active
26842024-04-17T17:12:02.206ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
26852024-04-17T17:12:02.206Z 1 Acked 1000 Write 9 Done Done Done false
26862024-04-17T17:12:02.206Z 2 Acked 1001 Write 3 Done Done Done false
26872024-04-17T17:12:02.206Z 3 Acked 1002 Write 6 Done Done Done false
26882024-04-17T17:12:02.207Z 4 Acked 1003 Write 7 Done Done Done false
26892024-04-17T17:12:02.207Z 5 Acked 1004 Write 7 Done Done Sent false
26902024-04-17T17:12:02.207Z 6 Acked 1005 Read 8 Done Done Sent false
26912024-04-17T17:12:02.207Z 7 Acked 1006 Write 10 Done Done Sent false
26922024-04-17T17:12:02.207Z 8 Acked 1007 Write 2 Done Done Sent false
26932024-04-17T17:12:02.207Z 9 Acked 1008 Read 5 Done Done Sent false
26942024-04-17T17:12:02.207Z 10 Acked 1009 Read 2 Done Done Sent false
26952024-04-17T17:12:02.207Z 11 Acked 1010 Read 8 Done Done Sent false
26962024-04-17T17:12:02.207Z 12 Acked 1011 Write 10 Done Done Sent false
26972024-04-17T17:12:02.207Z 13 Acked 1012 Write 8 Done Done Sent false
26982024-04-17T17:12:02.207Z 14 Acked 1013 Write 10 Done Done Sent false
26992024-04-17T17:12:02.207Z 15 Acked 1014 Flush 0 Done Done Sent false
27002024-04-17T17:12:02.208Z 16 Acked 1015 Write 10 Done Done Sent false
27012024-04-17T17:12:02.208Z 17 Acked 1016 Read 6 Done Done Sent false
27022024-04-17T17:12:02.208Z 18 Acked 1017 Read 8 Done Done Sent false
27032024-04-17T17:12:02.208Z 19 Acked 1018 Write 9 Done Done Sent false
27042024-04-17T17:12:02.208Z 20 Acked 1019 Write 1 Done Done Sent false
27052024-04-17T17:12:02.208Z 21 Acked 1020 Read 5 Done Done Sent false
27062024-04-17T17:12:02.208Z 22 Acked 1021 Write 9 Done Done Sent false
27072024-04-17T17:12:02.208Z 23 Acked 1022 Read 7 Done Done Sent false
27082024-04-17T17:12:02.209Z 24 Acked 1023 Read 10 Done Done Sent false
27092024-04-17T17:12:02.209Z 25 Acked 1024 Read 7 Done Done Sent false
27102024-04-17T17:12:02.209Z 26 Acked 1025 Write 6 Done Done Sent false
27112024-04-17T17:12:02.209Z 27 Acked 1026 Read 6 Done Done Sent false
27122024-04-17T17:12:02.209Z 28 Acked 1027 Read 4 Done Done Sent false
27132024-04-17T17:12:02.209Z 29 Acked 1028 Read 4 Done Done Sent false
27142024-04-17T17:12:02.209Z 30 Acked 1029 Read 7 Done Done Sent false
27152024-04-17T17:12:02.209Z STATES DS:0 DS:1 DS:2 TOTAL
27162024-04-17T17:12:02.209Z New 0 0 0 0
27172024-04-17T17:12:02.209Z Sent 0 0 26 26
27182024-04-17T17:12:02.209Z Done 30 30 4 64
27192024-04-17T17:12:02.209Z Skipped 0 0 0 0
27202024-04-17T17:12:02.209Z Error 0 0 0 0
27212024-04-17T17:12:02.209ZLast Flush: 1014 1014 0
27222024-04-17T17:12:02.209ZDownstairs last five completed:
27232024-04-17T17:12:02.210ZUpstairs last five completed: 30 29 28 27 26
27242024-04-17T17:12:02.210Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:12:00.797988546Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1264}
27252024-04-17T17:12:02.210ZWrote out file "/tmp/test_repair_verify.data"
27262024-04-17T17:12:02.210Z
27272024-04-17T17:12:02.215ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
27282024-04-17T17:12:02.220ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27292024-04-17T17:12:02.220Z 0 000-019 21 21 21 41 41 41 T T F <---
27302024-04-17T17:12:02.220Z 1 020-039 20 20 20 39 39 39 F F F
27312024-04-17T17:12:02.220Z 2 040-059 20 20 20 39 39 39 F F F
27322024-04-17T17:12:02.220Z 3 060-079 20 20 20 39 39 39 F F F
27332024-04-17T17:12:02.220Z 4 080-099 24 24 21 44 44 41 T T F <---
27342024-04-17T17:12:02.220Z 5 100-119 24 24 15 44 44 27 F F F <---
27352024-04-17T17:12:02.220Z 6 120-139 23 23 23 43 43 43 F F F
27362024-04-17T17:12:02.220Z 7 140-159 24 24 23 44 44 43 F F T <---
27372024-04-17T17:12:02.220Z 8 160-179 24 24 17 44 44 30 F F T <---
27382024-04-17T17:12:02.220Z 9 180-199 24 24 21 44 44 41 F F T <---
27392024-04-17T17:12:02.220Z 10 200-219 22 22 22 42 42 42 F F F
27402024-04-17T17:12:02.220Z 11 220-239 22 22 22 42 42 42 F F F
27412024-04-17T17:12:02.220Z 12 240-259 15 15 15 27 27 27 F F F
27422024-04-17T17:12:02.220Z 13 260-279 23 23 23 43 43 43 T T F <---
27432024-04-17T17:12:02.220Z 14 280-299 24 24 21 44 44 41 T T T <---
27442024-04-17T17:12:02.220Z 15 300-319 24 24 21 44 44 41 F F F <---
27452024-04-17T17:12:02.220Z 16 320-339 23 23 23 43 43 43 F F F
27462024-04-17T17:12:02.220Z 17 340-359 24 24 23 44 44 43 F F F <---
27472024-04-17T17:12:02.220Z 18 360-379 24 24 23 44 44 43 F F F <---
27482024-04-17T17:12:02.220Z 19 380-399 8 8 8 15 15 15 F F F
27492024-04-17T17:12:02.220Z 20 400-419 24 24 18 44 44 35 F F F <---
27502024-04-17T17:12:02.220Z 21 420-439 23 23 23 43 43 43 T T F <---
27512024-04-17T17:12:02.221Z 22 440-459 24 24 23 44 44 43 T T T <---
27522024-04-17T17:12:02.221Z 23 460-479 18 18 18 34 34 34 F F F
27532024-04-17T17:12:02.221Z 24 480-499 24 24 23 44 44 43 F F F <---
27542024-04-17T17:12:02.221Z 25 500-519 24 24 23 44 44 43 F F F <---
27552024-04-17T17:12:02.221Z 26 520-539 18 18 18 32 32 32 T T F <---
27562024-04-17T17:12:02.221Z 27 540-559 18 18 18 35 35 35 F F F
27572024-04-17T17:12:02.221Z 28 560-579 18 18 18 35 35 35 F F F
27582024-04-17T17:12:02.221Z 29 580-599 23 23 23 43 43 43 F F F
27592024-04-17T17:12:02.221ZMax gen: 24, Max flush: 44
27602024-04-17T17:12:02.221ZError: Difference in extent metadata found!
27612024-04-17T17:12:02.230ZA Difference in extent metadata is expected here
27622024-04-17T17:12:02.230ZOn loop 12
27632024-04-17T17:12:02.230Z
27642024-04-17T17:12:02.230ZVerifying data now
27652024-04-17T17:12:16.993ZNo change after verify
27662024-04-17T17:12:16.993ZLoop: 12 Downstairs dump after verify (and repair):
27672024-04-17T17:12:17.008ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27682024-04-17T17:12:17.008Z 0 000-019 25 25 25 45 45 45 F F F
27692024-04-17T17:12:17.008Z 1 020-039 20 20 20 39 39 39 F F F
27702024-04-17T17:12:17.008Z 2 040-059 20 20 20 39 39 39 F F F
27712024-04-17T17:12:17.008Z 3 060-079 20 20 20 39 39 39 F F F
27722024-04-17T17:12:17.008Z 4 080-099 25 25 25 45 45 45 F F F
27732024-04-17T17:12:17.008Z 5 100-119 24 24 24 44 44 44 F F F
27742024-04-17T17:12:17.008Z 6 120-139 23 23 23 43 43 43 F F F
27752024-04-17T17:12:17.008Z 7 140-159 24 24 24 44 44 44 F F F
27762024-04-17T17:12:17.008Z 8 160-179 24 24 24 44 44 44 F F F
27772024-04-17T17:12:17.008Z 9 180-199 24 24 24 44 44 44 F F F
27782024-04-17T17:12:17.008Z 10 200-219 22 22 22 42 42 42 F F F
27792024-04-17T17:12:17.012Z 11 220-239 22 22 22 42 42 42 F F F
27802024-04-17T17:12:17.012Z 12 240-259 15 15 15 27 27 27 F F F
27812024-04-17T17:12:17.012Z 13 260-279 25 25 25 45 45 45 F F F
27822024-04-17T17:12:17.012Z 14 280-299 25 25 25 45 45 45 F F F
27832024-04-17T17:12:17.013Z 15 300-319 24 24 24 44 44 44 F F F
27842024-04-17T17:12:17.013Z 16 320-339 23 23 23 43 43 43 F F F
27852024-04-17T17:12:17.013Z 17 340-359 24 24 24 44 44 44 F F F
27862024-04-17T17:12:17.013Z 18 360-379 24 24 24 44 44 44 F F F
27872024-04-17T17:12:17.013Z 19 380-399 8 8 8 15 15 15 F F F
27882024-04-17T17:12:17.013Z 20 400-419 24 24 24 44 44 44 F F F
27892024-04-17T17:12:17.013Z 21 420-439 25 25 25 45 45 45 F F F
27902024-04-17T17:12:17.013Z 22 440-459 25 25 25 45 45 45 F F F
27912024-04-17T17:12:17.013Z 23 460-479 18 18 18 34 34 34 F F F
27922024-04-17T17:12:17.013Z 24 480-499 24 24 24 44 44 44 F F F
27932024-04-17T17:12:17.013Z 25 500-519 24 24 24 44 44 44 F F F
27942024-04-17T17:12:17.013Z 26 520-539 25 25 25 45 45 45 F F F
27952024-04-17T17:12:17.013Z 27 540-559 18 18 18 35 35 35 F F F
27962024-04-17T17:12:17.013Z 28 560-579 18 18 18 35 35 35 F F F
27972024-04-17T17:12:17.013Z 29 580-599 23 23 23 43 43 43 F F F
27982024-04-17T17:12:17.013ZMax gen: 25, Max flush: 45
27992024-04-17T17:12:17.013Z
28002024-04-17T17:12:17.013ZBegin loop 13
28012024-04-17T17:12:17.013ZDownstairs to restart: 2
28022024-04-17T17:12:17.030Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.636895359Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272}
28032024-04-17T17:12:17.030Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.637168275Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272}
28042024-04-17T17:12:17.037Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30Crucible runtime is spawned
28052024-04-17T17:12:17.037Z,"time":"2024-04-17T17:12:15.637211878Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272}
28062024-04-17T17:12:17.037Z{"msg":"Crucible 442d1eec-b0fa-49d2-8c8d-2d8bdea2e960 has session id: 25aceaab-2025-4059-aecf-fa62a4fe1add","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.6372505Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28072024-04-17T17:12:17.037Z{"msg":"Upstairs opts: Upstairs UUID: 442d1eec-b0fa-49d2-8c8d-2d8bdea2e960, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.637318924Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28082024-04-17T17:12:17.037Z{"msg":"Crucible stats registered with UUID: 442d1eec-b0fa-49d2-8c8d-2d8bdea2e960","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.637362247Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28092024-04-17T17:12:17.037Z{"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.637392909Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272}
28102024-04-17T17:12:17.037Z{"msg":"442d1eec-b0fa-49d2-8c8d-2d8bdea2e960 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.637430471Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28112024-04-17T17:12:17.037Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.637472264Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"client":"0","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28122024-04-17T17:12:17.037Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.637518737Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"client":"0","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28132024-04-17T17:12:17.037Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.637550888Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"client":"1","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28142024-04-17T17:12:17.037Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.63758046Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"client":"1","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28152024-04-17T17:12:17.037Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.637630523Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"client":"2","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28162024-04-17T17:12:17.037Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.637676616Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"client":"2","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28172024-04-17T17:12:17.037Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.637709518Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28182024-04-17T17:12:17.037Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.63774553Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28192024-04-17T17:12:17.037Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.637790593Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28202024-04-17T17:12:17.037Z{"msg":"ds_connection connected from Ok(127.0.0.1:52544)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.637824955Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28212024-04-17T17:12:17.037Z{"msg":"ds_connection connected from Ok(127.0.0.1:37686)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.637842476Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28222024-04-17T17:12:17.037Z{"msg":"ds_connection connected from Ok(127.0.0.1:38998)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.637856657Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28232024-04-17T17:12:17.037Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.638478405Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"client":"0","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28242024-04-17T17:12:17.037Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.638528638Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"client":"0","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28252024-04-17T17:12:17.037Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.638578941Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"client":"1","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28262024-04-17T17:12:17.038Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.638608383Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"client":"1","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28272024-04-17T17:12:17.038Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.638716779Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"client":"2","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28282024-04-17T17:12:17.038Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.638752341Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"client":"2","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28292024-04-17T17:12:17.038Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.639162726Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"client":"0","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28302024-04-17T17:12:17.038Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.639199828Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"client":"1","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28312024-04-17T17:12:17.038Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.639216099Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"client":"2","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28322024-04-17T17:12:17.038Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.639691568Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"client":"1","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28332024-04-17T17:12:17.038Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.639728851Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"client":"1","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28342024-04-17T17:12:17.038Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.639746861Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28352024-04-17T17:12:17.038Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.639771993Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"client":"0","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28362024-04-17T17:12:17.038Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.639804175Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"client":"0","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28372024-04-17T17:12:17.038Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.639819606Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28382024-04-17T17:12:17.038Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.639834587Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"client":"2","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28392024-04-17T17:12:17.038Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.639864669Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"client":"2","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28402024-04-17T17:12:17.038Z{"msg":"[0]R flush_numbers[0..12]: [45, 39, 39, 39, 45, 44, 43, 44, 44, 44, 42, 42]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.63988071Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"":"downstairs","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28412024-04-17T17:12:17.038Z{"msg":"[0]R generation[0..12]: [25, 20, 20, 20, 25, 24, 23, 24, 24, 24, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.639912551Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"":"downstairs","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28422024-04-17T17:12:17.038Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.639930303Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"":"downstairs","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28432024-04-17T17:12:17.038Z{"msg":"[1]R flush_numbers[0..12]: [45, 39, 39, 39, 45, 44, 43, 44, 44, 44, 42, 42]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.639960674Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"":"downstairs","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28442024-04-17T17:12:17.038ZWait for a query_work_queue command to finish before sending IO
28452024-04-17T17:12:17.038Z{"msg":"[1]R generation[0..12]: [25, 20, 20, 20, 25, 24, 23, 24, 24, 24, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.639981006Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"":"downstairs","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28462024-04-17T17:12:17.038Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.640011708Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"":"downstairs","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28472024-04-17T17:12:17.038Z{"msg":"[2]R flush_numbers[0..12]: [45, 39, 39, 39, 45, 44, 43, 44, 44, 44, 42, 42]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.640027069Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"":"downstairs","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28482024-04-17T17:12:17.038Z{"msg":"[2]R generation[0..12]: [25, 20, 20, 20, 25, 24, 23, 24, 24, 24, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.640040609Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"":"downstairs","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28492024-04-17T17:12:17.039Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.640062841Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"":"downstairs","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28502024-04-17T17:12:17.039Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.640075992Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"":"downstairs","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28512024-04-17T17:12:17.039Z{"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.640088992Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"":"downstairs","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28522024-04-17T17:12:17.039Z{"msg":"Next flush: 46","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.640112344Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"":"downstairs","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28532024-04-17T17:12:17.039Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.640128045Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"":"downstairs","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28542024-04-17T17:12:17.039Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.640140956Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28552024-04-17T17:12:17.039Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.640153736Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"client":"0","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28562024-04-17T17:12:17.039Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.640166837Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"client":"1","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28572024-04-17T17:12:17.039Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.640179918Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"client":"2","":"downstairs","session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28582024-04-17T17:12:17.039Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.640192928Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28592024-04-17T17:12:17.039Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.64021248Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28602024-04-17T17:12:17.039Z{"msg":"442d1eec-b0fa-49d2-8c8d-2d8bdea2e960 is now active with session: 25aceaab-2025-4059-aecf-fa62a4fe1add","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.64022549Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28612024-04-17T17:12:17.039Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.640237361Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272,"session_id":"25aceaab-2025-4059-aecf-fa62a4fe1add"}
28622024-04-17T17:12:17.039Z{"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:15.640261263Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272}
28632024-04-17T17:12:17.039ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
28642024-04-17T17:12:17.039ZLoading write count information from file "/tmp/test_repair_verify.data"
28652024-04-17T17:12:17.039ZRun Repair workload
28662024-04-17T17:12:17.039Z01/30 Flush
28672024-04-17T17:12:17.043Z02/30 Write block 488 len 32768 data: 3 3 3 3 3 2 2 3
28682024-04-17T17:12:17.043Z03/30 Write block 105 len 32768 data: 3 4 4 4 3 3 3 3
28692024-04-17T17:12:17.043Z04/30 Read block 538 len 4096
28702024-04-17T17:12:17.043Z05/30 Read block 201 len 4096
28712024-04-17T17:12:17.043Z06/30 Write block 279 len 8192 data: 3 3
28722024-04-17T17:12:17.043Z07/30 Write block 17 len 4096 data: 4
28732024-04-17T17:12:17.044Z08/30 Read block 253 len 8192
28742024-04-17T17:12:17.044Z09/30 Read block 199 len 20480
28752024-04-17T17:12:17.044Z10/30 Read block 161 len 12288
28762024-04-17T17:12:17.044Z11/30 Read block 259 len 12288
28772024-04-17T17:12:17.044Z12/30 Read block 251 len 40960
28782024-04-17T17:12:17.047Z13/30 Flush
28792024-04-17T17:12:17.050Z14/30 Write block 517 len 36864 data: 2 3 2 2 2 2 3 2 2
28802024-04-17T17:12:17.051Z15/30 Read block 333 len 4096
28812024-04-17T17:12:17.051Z16/30 Write block 526 len 20480 data: 2 2 3 3 3
28822024-04-17T17:12:17.051Z17/30 Flush
28832024-04-17T17:12:17.054Z18/30 Write block 57 len 32768 data: 3 3 3 4 4 3 3 3
28842024-04-17T17:12:17.054Z19/30 Read block 434 len 4096
28852024-04-17T17:12:17.058Z20/30 Write block 383 len 8192 data: 3 3
28862024-04-17T17:12:17.058Z21/30 Flush
28872024-04-17T17:12:17.061Z22/30 Write block 163 len 28672 data: 2 2 2 2 3 3 3
28882024-04-17T17:12:17.061Z23/30 Read block 178 len 32768
28892024-04-17T17:12:17.062Z24/30 Read block 350 len 8192
28902024-04-17T17:12:17.062Z25/30 Read block 230 len 4096
28912024-04-17T17:12:17.065Z26/30 Write block 435 len 40960 data: 3 3 3 4 5 5 5 6 6 6
28922024-04-17T17:12:17.065Z27/30 Write block 546 len 32768 data: 2 2 2 2 2 2 3 3
28932024-04-17T17:12:17.065Z28/30 Write block 134 len 24576 data: 4 4 4 3 3 3
28942024-04-17T17:12:17.070Z29/30 Write block 36 len 28672 data: 5 5 4 4 5 5 4
28952024-04-17T17:12:17.070Z30/30 Write block 541 len 16384 data: 4 4 4 3
28962024-04-17T17:12:17.070Z----------------------------------------------------------------
28972024-04-17T17:12:17.070Z Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:29
28982024-04-17T17:12:17.070ZStates: Active Active Active
28992024-04-17T17:12:17.070ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
29002024-04-17T17:12:17.070Z 2 Acked 1001 Write 8 Done Done Done false
29012024-04-17T17:12:17.070Z 3 Acked 1002 Write 8 Done Done Done false
29022024-04-17T17:12:17.070Z 4 Acked 1003 Read 1 Done Done Done false
29032024-04-17T17:12:17.071Z 5 Acked 1004 Read 1 Done Done Sent false
29042024-04-17T17:12:17.071Z 6 Acked 1005 Write 2 Done Done Sent false
29052024-04-17T17:12:17.071Z 7 Acked 1006 Write 1 Done Done Sent false
29062024-04-17T17:12:17.071Z 8 Acked 1007 Read 2 Done Done Sent false
29072024-04-17T17:12:17.071Z 9 Acked 1008 Read 5 Done Done Sent false
29082024-04-17T17:12:17.071Z 10 Acked 1009 Read 3 Done Done Sent false
29092024-04-17T17:12:17.071Z 11 Acked 1010 Read 3 Done Done Sent false
29102024-04-17T17:12:17.071Z 12 Acked 1011 Read 10 Done Done Sent false
29112024-04-17T17:12:17.071Z 13 Acked 1012 Flush 0 Done Done Sent false
29122024-04-17T17:12:17.071Z 14 Acked 1013 Write 9 Done Done Sent false
29132024-04-17T17:12:17.071Z 15 Acked 1014 Read 1 Done Done Sent false
29142024-04-17T17:12:17.071Z 16 Acked 1015 Write 5 Done Done Sent false
29152024-04-17T17:12:17.071Z 17 Acked 1016 Flush 0 Done Done Sent false
29162024-04-17T17:12:17.071Z 18 Acked 1017 Write 8 Done Done Sent false
29172024-04-17T17:12:17.071Z 19 Acked 1018 Read 1 Done Done Sent false
29182024-04-17T17:12:17.071Z 20 Acked 1019 Write 2 Done Done Sent false
29192024-04-17T17:12:17.071Z 21 Acked 1020 Flush 0 Done Done Sent false
29202024-04-17T17:12:17.071Z 22 Acked 1021 Write 7 Done Done Sent false
29212024-04-17T17:12:17.071Z 23 Acked 1022 Read 8 Done Done Sent false
29222024-04-17T17:12:17.071Z 24 Acked 1023 Read 2 Done Done Sent false
29232024-04-17T17:12:17.071Z 25 Acked 1024 Read 1 Done Done Sent false
29242024-04-17T17:12:17.071Z 26 Acked 1025 Write 10 Done Done Sent false
29252024-04-17T17:12:17.071Z 27 Acked 1026 Write 8 Done Done Sent false
29262024-04-17T17:12:17.071Z 28 Acked 1027 Write 6 Done Done Sent false
29272024-04-17T17:12:17.072Z 29 Acked 1028 Write 7 Done Done Sent false
29282024-04-17T17:12:17.072Z 30 Acked 1029 Write 4 Sent Sent Sent false
29292024-04-17T17:12:17.072Z STATES DS:0 DS:1 DS:2 TOTAL
29302024-04-17T17:12:17.072Z New 0 0 0 0
29312024-04-17T17:12:17.072Z Sent 1 1 26 28
29322024-04-17T17:12:17.072Z Done 28 28 3 59
29332024-04-17T17:12:17.072Z Skipped 0 0 0 0
29342024-04-17T17:12:17.072Z Error 0 0 0 0
29352024-04-17T17:12:17.072ZLast Flush: 1020 1020 1000
29362024-04-17T17:12:17.072ZDownstairs last five completed: 1000
29372024-04-17T17:12:17.072ZUpstairs last five completed: 30 29 28 27 26
29382024-04-17T17:12:17.072Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:12:15.674001737Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1272}
29392024-04-17T17:12:17.072ZWrote out file "/tmp/test_repair_verify.data"
29402024-04-17T17:12:17.076Z
29412024-04-17T17:12:17.076ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
29422024-04-17T17:12:17.091ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
29432024-04-17T17:12:17.091Z 0 000-019 26 26 25 47 47 45 F F F <---
29442024-04-17T17:12:17.096Z 1 020-039 20 20 20 39 39 39 T T F <---
29452024-04-17T17:12:17.096Z 2 040-059 26 26 20 49 49 39 T T F <---
29462024-04-17T17:12:17.096Z 3 060-079 26 26 20 49 49 39 F F F <---
29472024-04-17T17:12:17.096Z 4 080-099 25 25 25 45 45 45 F F F
29482024-04-17T17:12:17.096Z 5 100-119 26 26 24 47 47 44 F F T <---
29492024-04-17T17:12:17.096Z 6 120-139 23 23 23 43 43 43 T T F <---
29502024-04-17T17:12:17.096Z 7 140-159 24 24 24 44 44 44 F F F
29512024-04-17T17:12:17.096Z 8 160-179 24 24 24 44 44 44 T T F <---
29522024-04-17T17:12:17.096Z 9 180-199 24 24 24 44 44 44 F F F
29532024-04-17T17:12:17.096Z 10 200-219 22 22 22 42 42 42 F F F
29542024-04-17T17:12:17.096Z 11 220-239 22 22 22 42 42 42 F F F
29552024-04-17T17:12:17.096Z 12 240-259 15 15 15 27 27 27 F F F
29562024-04-17T17:12:17.096Z 13 260-279 26 26 25 47 47 45 F F F <---
29572024-04-17T17:12:17.096Z 14 280-299 26 26 25 47 47 45 F F F <---
29582024-04-17T17:12:17.096Z 15 300-319 24 24 24 44 44 44 F F F
29592024-04-17T17:12:17.096Z 16 320-339 23 23 23 43 43 43 F F F
29602024-04-17T17:12:17.096Z 17 340-359 24 24 24 44 44 44 F F F
29612024-04-17T17:12:17.096Z 18 360-379 24 24 24 44 44 44 F F F
29622024-04-17T17:12:17.096Z 19 380-399 26 26 8 49 49 15 F F F <---
29632024-04-17T17:12:17.097Z 20 400-419 24 24 24 44 44 44 F F F
29642024-04-17T17:12:17.097Z 21 420-439 25 25 25 45 45 45 T T F <---
29652024-04-17T17:12:17.097Z 22 440-459 25 25 25 45 45 45 T T F <---
29662024-04-17T17:12:17.097Z 23 460-479 18 18 18 34 34 34 F F F
29672024-04-17T17:12:17.097Z 24 480-499 26 26 24 47 47 44 F F T <---
29682024-04-17T17:12:17.097Z 25 500-519 26 26 24 48 48 44 F F F <---
29692024-04-17T17:12:17.097Z 26 520-539 26 26 25 48 48 45 F F F <---
29702024-04-17T17:12:17.097Z 27 540-559 18 18 18 35 35 35 T T F <---
29712024-04-17T17:12:17.097Z 28 560-579 18 18 18 35 35 35 F F F
29722024-04-17T17:12:17.097Z 29 580-599 23 23 23 43 43 43 F F F
29732024-04-17T17:12:17.097ZMax gen: 26, Max flush: 49
29742024-04-17T17:12:17.097ZError: Difference in extent metadata found!
29752024-04-17T17:12:17.097ZA Difference in extent metadata is expected here
29762024-04-17T17:12:17.097ZOn loop 13
29772024-04-17T17:12:17.097Z
29782024-04-17T17:12:17.101ZVerifying data now
29792024-04-17T17:12:31.832ZNo change after verify
29802024-04-17T17:12:31.832ZLoop: 13 Downstairs dump after verify (and repair):
29812024-04-17T17:12:31.843ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
29822024-04-17T17:12:31.843Z 0 000-019 26 26 26 47 47 47 F F F
29832024-04-17T17:12:31.843Z 1 020-039 27 27 27 50 50 50 F F F
29842024-04-17T17:12:31.843Z 2 040-059 27 27 27 50 50 50 F F F
29852024-04-17T17:12:31.843Z 3 060-079 26 26 26 49 49 49 F F F
29862024-04-17T17:12:31.843Z 4 080-099 25 25 25 45 45 45 F F F
29872024-04-17T17:12:31.843Z 5 100-119 26 26 26 47 47 47 F F F
29882024-04-17T17:12:31.844Z 6 120-139 27 27 27 50 50 50 F F F
29892024-04-17T17:12:31.848Z 7 140-159 24 24 24 44 44 44 F F F
29902024-04-17T17:12:31.848Z 8 160-179 27 27 27 50 50 50 F F F
29912024-04-17T17:12:31.848Z 9 180-199 24 24 24 44 44 44 F F F
29922024-04-17T17:12:31.848Z 10 200-219 22 22 22 42 42 42 F F F
29932024-04-17T17:12:31.848Z 11 220-239 22 22 22 42 42 42 F F F
29942024-04-17T17:12:31.848Z 12 240-259 15 15 15 27 27 27 F F F
29952024-04-17T17:12:31.848Z 13 260-279 26 26 26 47 47 47 F F F
29962024-04-17T17:12:31.848Z 14 280-299 26 26 26 47 47 47 F F F
29972024-04-17T17:12:31.848Z 15 300-319 24 24 24 44 44 44 F F F
29982024-04-17T17:12:31.848Z 16 320-339 23 23 23 43 43 43 F F F
29992024-04-17T17:12:31.848Z 17 340-359 24 24 24 44 44 44 F F F
30002024-04-17T17:12:31.848Z 18 360-379 24 24 24 44 44 44 F F F
30012024-04-17T17:12:31.848Z 19 380-399 26 26 26 49 49 49 F F F
30022024-04-17T17:12:31.848Z 20 400-419 24 24 24 44 44 44 F F F
30032024-04-17T17:12:31.848Z 21 420-439 27 27 27 50 50 50 F F F
30042024-04-17T17:12:31.848Z 22 440-459 27 27 27 50 50 50 F F F
30052024-04-17T17:12:31.848Z 23 460-479 18 18 18 34 34 34 F F F
30062024-04-17T17:12:31.848Z 24 480-499 26 26 26 47 47 47 F F F
30072024-04-17T17:12:31.848Z 25 500-519 26 26 26 48 48 48 F F F
30082024-04-17T17:12:31.848Z 26 520-539 26 26 26 48 48 48 F F F
30092024-04-17T17:12:31.848Z 27 540-559 27 27 27 50 50 50 F F F
30102024-04-17T17:12:31.849Z 28 560-579 18 18 18 35 35 35 F F F
30112024-04-17T17:12:31.849Z 29 580-599 23 23 23 43 43 43 F F F
30122024-04-17T17:12:31.849ZMax gen: 27, Max flush: 50
30132024-04-17T17:12:31.849Z
30142024-04-17T17:12:31.849ZBegin loop 14
30152024-04-17T17:12:31.849ZDownstairs to restart: 2
30162024-04-17T17:12:31.866Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.472542534Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280}
30172024-04-17T17:12:31.866Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.472863523Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280}
30182024-04-17T17:12:31.866Z{Crucible runtime is spawned
30192024-04-17T17:12:31.873Z"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.472909066Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280}
30202024-04-17T17:12:31.873Z{"msg":"Crucible d6e103e5-9a2f-4ce0-8cf2-c4af10ba4e42 has session id: d7707197-a88a-496a-b098-a21c58516777","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.472941258Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30212024-04-17T17:12:31.873Z{"msg":"Upstairs opts: Upstairs UUID: d6e103e5-9a2f-4ce0-8cf2-c4af10ba4e42, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.472987881Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30222024-04-17T17:12:31.873Z{"msg":"Crucible stats registered with UUID: d6e103e5-9a2f-4ce0-8cf2-c4af10ba4e42","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.473004622Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30232024-04-17T17:12:31.873Z{"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.473017603Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280}
30242024-04-17T17:12:31.873Z{"msg":"d6e103e5-9a2f-4ce0-8cf2-c4af10ba4e42 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.47313442Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30252024-04-17T17:12:31.873Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.473167852Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"client":"0","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30262024-04-17T17:12:31.873Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.473195183Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"client":"0","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30272024-04-17T17:12:31.873Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.473238326Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"client":"1","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30282024-04-17T17:12:31.873Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.473276288Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"client":"1","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30292024-04-17T17:12:31.873Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.473311461Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"client":"2","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30302024-04-17T17:12:31.873Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.473346013Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"client":"2","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30312024-04-17T17:12:31.873Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.473370904Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30322024-04-17T17:12:31.873Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.473409427Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30332024-04-17T17:12:31.874Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.473447289Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30342024-04-17T17:12:31.874Z{"msg":"ds_connection connected from Ok(127.0.0.1:53656)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.473475711Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30352024-04-17T17:12:31.874Z{"msg":"ds_connection connected from Ok(127.0.0.1:56366)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.473520123Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30362024-04-17T17:12:31.874Z{"msg":"ds_connection connected from Ok(127.0.0.1:42328)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.473602488Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30372024-04-17T17:12:31.874Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.47429487Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"client":"1","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30382024-04-17T17:12:31.874Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.474341703Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"client":"1","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30392024-04-17T17:12:31.874Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.474394516Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"client":"0","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30402024-04-17T17:12:31.874Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.474469451Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"client":"0","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30412024-04-17T17:12:31.874Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.474535295Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"client":"2","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30422024-04-17T17:12:31.874Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.474579787Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"client":"2","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30432024-04-17T17:12:31.874Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.474870125Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"client":"1","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30442024-04-17T17:12:31.874Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.474968771Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"client":"0","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30452024-04-17T17:12:31.874Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.475266979Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"client":"2","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30462024-04-17T17:12:31.874Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.475432949Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"client":"0","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30472024-04-17T17:12:31.874Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.475467441Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"client":"0","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30482024-04-17T17:12:31.874Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.475482792Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30492024-04-17T17:12:31.874Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.475505694Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"client":"1","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30502024-04-17T17:12:31.874Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.475532365Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"client":"1","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30512024-04-17T17:12:31.874Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.475547996Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30522024-04-17T17:12:31.874Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.475709256Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"client":"2","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30532024-04-17T17:12:31.874Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.475754919Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"client":"2","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30542024-04-17T17:12:31.874Z{"msg":"[0]R flush_numbers[0..12]: [47, 50, 50, 49, 45, 47, 50, 44, 50, 44, 42, 42]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.475803821Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"":"downstairs","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30552024-04-17T17:12:31.874Z{"msg":"[0]R generation[0..12]: [26, 27, 27, 26, 25, 26, 27, 24, 27, 24, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.475829903Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"":"downstairs","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30562024-04-17T17:12:31.875Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.475860545Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"":"downstairs","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30572024-04-17T17:12:31.875Z{"msg":"[1]R flush_numbers[0..12]: [47, 50, 50, 49, 45, 47, 50, 44, 50, 44, 42, 42]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.475885357Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"":"downstairs","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30582024-04-17T17:12:31.875Z{"msg":"Wait for a query_work_queue command to finish before sending IO
30592024-04-17T17:12:31.875Z[1]R generation[0..12]: [26, 27, 27, 26, 25, 26, 27, 24, 27, 24, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.475916448Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"":"downstairs","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30602024-04-17T17:12:31.875Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.47594318Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"":"downstairs","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30612024-04-17T17:12:31.875Z{"msg":"[2]R flush_numbers[0..12]: [47, 50, 50, 49, 45, 47, 50, 44, 50, 44, 42, 42]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.475973332Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"":"downstairs","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30622024-04-17T17:12:31.875Z{"msg":"[2]R generation[0..12]: [26, 27, 27, 26, 25, 26, 27, 24, 27, 24, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.475997823Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"":"downstairs","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30632024-04-17T17:12:31.875Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.476028205Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"":"downstairs","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30642024-04-17T17:12:31.875Z{"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.476042896Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"":"downstairs","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30652024-04-17T17:12:31.875Z{"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.476063847Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"":"downstairs","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30662024-04-17T17:12:31.875Z{"msg":"Next flush: 51","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.476077068Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"":"downstairs","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30672024-04-17T17:12:31.875Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.476096529Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"":"downstairs","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30682024-04-17T17:12:31.875Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.47610986Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30692024-04-17T17:12:31.875Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.476121581Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"client":"0","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30702024-04-17T17:12:31.875Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.476134552Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"client":"1","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30712024-04-17T17:12:31.875Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.476153883Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"client":"2","":"downstairs","session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30722024-04-17T17:12:31.875Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.476172174Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30732024-04-17T17:12:31.875Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.476193415Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30742024-04-17T17:12:31.875Z{"msg":"d6e103e5-9a2f-4ce0-8cf2-c4af10ba4e42 is now active with session: d7707197-a88a-496a-b098-a21c58516777","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.476206706Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30752024-04-17T17:12:31.875Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.476233648Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280,"session_id":"d7707197-a88a-496a-b098-a21c58516777"}
30762024-04-17T17:12:31.875Z{"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:30.476248168Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280}
30772024-04-17T17:12:31.875ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
30782024-04-17T17:12:31.875ZLoading write count information from file "/tmp/test_repair_verify.data"
30792024-04-17T17:12:31.875ZRun Repair workload
30802024-04-17T17:12:31.880Z01/30 Write block 239 len 8192 data: 3 3
30812024-04-17T17:12:31.880Z02/30 Write block 410 len 28672 data: 5 6 7 7 7 6 5
30822024-04-17T17:12:31.880Z03/30 Write block 111 len 8192 data: 4 4
30832024-04-17T17:12:31.880Z04/30 Read block 267 len 20480
30842024-04-17T17:12:31.880Z05/30 Write block 25 len 16384 data: 5 5 3 2
30852024-04-17T17:12:31.880Z06/30 Write block 103 len 8192 data: 3 3
30862024-04-17T17:12:31.880Z07/30 Write block 193 len 4096 data: 3
30872024-04-17T17:12:31.880Z08/30 Write block 289 len 12288 data: 4 4 4
30882024-04-17T17:12:31.880Z09/30 Read block 120 len 16384
30892024-04-17T17:12:31.880Z10/30 Read block 556 len 4096
30902024-04-17T17:12:31.880Z11/30 Flush
30912024-04-17T17:12:31.884Z12/30 Flush
30922024-04-17T17:12:31.884Z13/30 Write block 32 len 20480 data: 7 7 7 7 6
30932024-04-17T17:12:31.884Z14/30 Write block 14 len 8192 data: 5 4
30942024-04-17T17:12:31.884Z15/30 Write block 350 len 24576 data: 5 4 4 4 4 4
30952024-04-17T17:12:31.884Z16/30 Read block 49 len 8192
30962024-04-17T17:12:31.887Z17/30 Flush
30972024-04-17T17:12:31.887Z18/30 Flush
30982024-04-17T17:12:31.887Z19/30 Write block 9 len 16384 data: 5 5 5 5
30992024-04-17T17:12:31.891Z20/30 Read block 340 len 16384
31002024-04-17T17:12:31.891Z21/30 Read block 130 len 20480
31012024-04-17T17:12:31.891Z22/30 Read block 284 len 4096
31022024-04-17T17:12:31.891Z23/30 Flush
31032024-04-17T17:12:31.895Z24/30 Write block 522 len 8192 data: 3 4
31042024-04-17T17:12:31.895Z25/30 Read block 212 len 40960
31052024-04-17T17:12:31.895Z26/30 Write block 212 len 16384 data: 2 2 2 3
31062024-04-17T17:12:31.899Z27/30 Write block 241 len 36864 data: 3 3 3 4 4 5 5 4 4
31072024-04-17T17:12:31.899Z28/30 Write block 32 len 12288 data: 8 8 8
31082024-04-17T17:12:31.899Z29/30 Read block 12 len 32768
31092024-04-17T17:12:31.899Z30/30 Read block 44 len 4096
31102024-04-17T17:12:31.899Z----------------------------------------------------------------
31112024-04-17T17:12:31.899Z Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:30
31122024-04-17T17:12:31.899ZStates: Active Active Active
31132024-04-17T17:12:31.900ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
31142024-04-17T17:12:31.900Z 1 Acked 1000 Write 2 Done Done Sent false
31152024-04-17T17:12:31.900Z 2 Acked 1001 Write 7 Done Done Sent false
31162024-04-17T17:12:31.900Z 3 Acked 1002 Write 2 Done Done Sent false
31172024-04-17T17:12:31.900Z 4 Acked 1003 Read 5 Done Done Sent false
31182024-04-17T17:12:31.900Z 5 Acked 1004 Write 4 Done Done Sent false
31192024-04-17T17:12:31.900Z 6 Acked 1005 Write 2 Done Done Sent false
31202024-04-17T17:12:31.900Z 7 Acked 1006 Write 1 Done Done Sent false
31212024-04-17T17:12:31.900Z 8 Acked 1007 Write 3 Done Done Sent false
31222024-04-17T17:12:31.900Z 9 Acked 1008 Read 4 Done Done Sent false
31232024-04-17T17:12:31.900Z 10 Acked 1009 Read 1 Done Done Sent false
31242024-04-17T17:12:31.900Z 11 Acked 1010 Flush 0 Done Done Sent false
31252024-04-17T17:12:31.900Z 12 Acked 1011 Flush 0 Done Done Sent false
31262024-04-17T17:12:31.900Z 13 Acked 1012 Write 5 Done Done Sent false
31272024-04-17T17:12:31.900Z 14 Acked 1013 Write 2 Done Done Sent false
31282024-04-17T17:12:31.900Z 15 Acked 1014 Write 6 Done Done Sent false
31292024-04-17T17:12:31.900Z 16 Acked 1015 Read 2 Done Done Sent false
31302024-04-17T17:12:31.900Z 17 Acked 1016 Flush 0 Done Done Sent false
31312024-04-17T17:12:31.900Z 18 Acked 1017 Flush 0 Done Done Sent false
31322024-04-17T17:12:31.900Z 19 Acked 1018 Write 4 Done Done Sent false
31332024-04-17T17:12:31.900Z 20 Acked 1019 Read 4 Done Done Sent false
31342024-04-17T17:12:31.900Z 21 Acked 1020 Read 5 Done Done Sent false
31352024-04-17T17:12:31.901Z 22 Acked 1021 Read 1 Done Done Sent false
31362024-04-17T17:12:31.901Z 23 Acked 1022 Flush 0 Done Done Sent false
31372024-04-17T17:12:31.901Z 24 Acked 1023 Write 2 Done Done Sent false
31382024-04-17T17:12:31.901Z 25 Acked 1024 Read 10 Done Done Sent false
31392024-04-17T17:12:31.901Z 26 Acked 1025 Write 4 Done Done Sent false
31402024-04-17T17:12:31.901Z 27 Acked 1026 Write 9 Done Done Sent false
31412024-04-17T17:12:31.901Z 28 Acked 1027 Write 3 Done Done Sent false
31422024-04-17T17:12:31.901Z 29 Acked 1028 Read 8 Done Done Sent false
31432024-04-17T17:12:31.901Z 30 Acked 1029 Read 1 Done Done Sent false
31442024-04-17T17:12:31.901Z STATES DS:0 DS:1 DS:2 TOTAL
31452024-04-17T17:12:31.901Z New 0 0 0 0
31462024-04-17T17:12:31.901Z Sent 0 0 30 30
31472024-04-17T17:12:31.901Z Done 30 30 0 60
31482024-04-17T17:12:31.901Z Skipped 0 0 0 0
31492024-04-17T17:12:31.901Z Error 0 0 0 0
31502024-04-17T17:12:31.901ZLast Flush: 1022 1022 0
31512024-04-17T17:12:31.901ZDownstairs last five completed:
31522024-04-17T17:12:31.901ZUpstairs last five completed: 30 29 28 27 26
31532024-04-17T17:12:31.901Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:12:30.504669501Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1280}
31542024-04-17T17:12:31.901ZWrote out file "/tmp/test_repair_verify.data"
31552024-04-17T17:12:31.905Z
31562024-04-17T17:12:31.909ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
31572024-04-17T17:12:31.921ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
31582024-04-17T17:12:31.921Z 0 000-019 28 28 26 55 55 47 F F F <---
31592024-04-17T17:12:31.925Z 1 020-039 28 28 27 53 53 50 T T F <---
31602024-04-17T17:12:31.925Z 2 040-059 27 27 27 50 50 50 F F F
31612024-04-17T17:12:31.925Z 3 060-079 26 26 26 49 49 49 F F F
31622024-04-17T17:12:31.925Z 4 080-099 25 25 25 45 45 45 F F F
31632024-04-17T17:12:31.925Z 5 100-119 28 28 26 51 51 47 F F F <---
31642024-04-17T17:12:31.925Z 6 120-139 27 27 27 50 50 50 F F F
31652024-04-17T17:12:31.925Z 7 140-159 24 24 24 44 44 44 F F F
31662024-04-17T17:12:31.925Z 8 160-179 27 27 27 50 50 50 F F F
31672024-04-17T17:12:31.926Z 9 180-199 28 28 24 51 51 44 F F F <---
31682024-04-17T17:12:31.926Z 10 200-219 22 22 22 42 42 42 T T F <---
31692024-04-17T17:12:31.926Z 11 220-239 28 28 22 51 51 42 F F F <---
31702024-04-17T17:12:31.926Z 12 240-259 28 28 15 51 51 27 T T F <---
31712024-04-17T17:12:31.926Z 13 260-279 26 26 26 47 47 47 F F F
31722024-04-17T17:12:31.926Z 14 280-299 28 28 26 51 51 47 F F F <---
31732024-04-17T17:12:31.926Z 15 300-319 24 24 24 44 44 44 F F F
31742024-04-17T17:12:31.926Z 16 320-339 23 23 23 43 43 43 F F F
31752024-04-17T17:12:31.926Z 17 340-359 28 28 24 53 53 44 F F F <---
31762024-04-17T17:12:31.926Z 18 360-379 24 24 24 44 44 44 F F F
31772024-04-17T17:12:31.926Z 19 380-399 26 26 26 49 49 49 F F F
31782024-04-17T17:12:31.926Z 20 400-419 28 28 24 51 51 44 F F F <---
31792024-04-17T17:12:31.926Z 21 420-439 27 27 27 50 50 50 F F F
31802024-04-17T17:12:31.926Z 22 440-459 27 27 27 50 50 50 F F F
31812024-04-17T17:12:31.926Z 23 460-479 18 18 18 34 34 34 F F F
31822024-04-17T17:12:31.926Z 24 480-499 26 26 26 47 47 47 F F F
31832024-04-17T17:12:31.926Z 25 500-519 26 26 26 48 48 48 F F F
31842024-04-17T17:12:31.926Z 26 520-539 26 26 26 48 48 48 T T F <---
31852024-04-17T17:12:31.926Z 27 540-559 27 27 27 50 50 50 F F F
31862024-04-17T17:12:31.926Z 28 560-579 18 18 18 35 35 35 F F F
31872024-04-17T17:12:31.926Z 29 580-599 23 23 23 43 43 43 F F F
31882024-04-17T17:12:31.926ZMax gen: 28, Max flush: 55
31892024-04-17T17:12:31.926ZError: Difference in extent metadata found!
31902024-04-17T17:12:31.926ZA Difference in extent metadata is expected here
31912024-04-17T17:12:31.927ZOn loop 14
31922024-04-17T17:12:31.927Z
31932024-04-17T17:12:31.930ZVerifying data now
31942024-04-17T17:12:32.456ZNo change after verify
31952024-04-17T17:12:32.456ZLoop: 14 Downstairs dump after verify (and repair):
31962024-04-17T17:12:32.470ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
31972024-04-17T17:12:32.470Z 0 000-019 28 28 28 55 55 55 F F F
31982024-04-17T17:12:32.474Z 1 020-039 29 29 29 56 56 56 F F F
31992024-04-17T17:12:32.474Z 2 040-059 27 27 27 50 50 50 F F F
32002024-04-17T17:12:32.474Z 3 060-079 26 26 26 49 49 49 F F F
32012024-04-17T17:12:32.474Z 4 080-099 25 25 25 45 45 45 F F F
32022024-04-17T17:12:32.475Z 5 100-119 28 28 28 51 51 51 F F F
32032024-04-17T17:12:32.475Z 6 120-139 27 27 27 50 50 50 F F F
32042024-04-17T17:12:32.475Z 7 140-159 24 24 24 44 44 44 F F F
32052024-04-17T17:12:32.475Z 8 160-179 27 27 27 50 50 50 F F F
32062024-04-17T17:12:32.475Z 9 180-199 28 28 28 51 51 51 F F F
32072024-04-17T17:12:32.475Z 10 200-219 29 29 29 56 56 56 F F F
32082024-04-17T17:12:32.475Z 11 220-239 28 28 28 51 51 51 F F F
32092024-04-17T17:12:32.475Z 12 240-259 29 29 29 56 56 56 F F F
32102024-04-17T17:12:32.475Z 13 260-279 26 26 26 47 47 47 F F F
32112024-04-17T17:12:32.475Z 14 280-299 28 28 28 51 51 51 F F F
32122024-04-17T17:12:32.475Z 15 300-319 24 24 24 44 44 44 F F F
32132024-04-17T17:12:32.475Z 16 320-339 23 23 23 43 43 43 F F F
32142024-04-17T17:12:32.475Z 17 340-359 28 28 28 53 53 53 F F F
32152024-04-17T17:12:32.475Z 18 360-379 24 24 24 44 44 44 F F F
32162024-04-17T17:12:32.475Z 19 380-399 26 26 26 49 49 49 F F F
32172024-04-17T17:12:32.475Z 20 400-419 28 28 28 51 51 51 F F F
32182024-04-17T17:12:32.475Z 21 420-439 27 27 27 50 50 50 F F F
32192024-04-17T17:12:32.475Z 22 440-459 27 27 27 50 50 50 F F F
32202024-04-17T17:12:32.475Z 23 460-479 18 18 18 34 34 34 F F F
32212024-04-17T17:12:32.475Z 24 480-499 26 26 26 47 47 47 F F F
32222024-04-17T17:12:32.475Z 25 500-519 26 26 26 48 48 48 F F F
32232024-04-17T17:12:32.475Z 26 520-539 29 29 29 56 56 56 F F F
32242024-04-17T17:12:32.475Z 27 540-559 27 27 27 50 50 50 F F F
32252024-04-17T17:12:32.475Z 28 560-579 18 18 18 35 35 35 F F F
32262024-04-17T17:12:32.476Z 29 580-599 23 23 23 43 43 43 F F F
32272024-04-17T17:12:32.476ZMax gen: 29, Max flush: 56
32282024-04-17T17:12:32.476Z
32292024-04-17T17:12:32.476ZBegin loop 15
32302024-04-17T17:12:32.476ZDownstairs to restart: 1
32312024-04-17T17:12:32.494Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.100707544Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288}
32322024-04-17T17:12:32.494Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.100987441Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288}
32332024-04-17T17:12:32.499Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.101031323Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288}
32342024-04-17T17:12:32.499Z{"msg":"Crucible d6b9b286-53eb-4d7b-835d-78cba04bfdcd has session id: 658c437a-602f-4b71-bf2d-e557e46d1e2a","v":0,"name":"crucible","level":30Crucible runtime is spawned
32352024-04-17T17:12:32.500Z,"time":"2024-04-17T17:12:31.101063855Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32362024-04-17T17:12:32.500Z{"msg":"Upstairs opts: Upstairs UUID: d6b9b286-53eb-4d7b-835d-78cba04bfdcd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.10113536Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32372024-04-17T17:12:32.500Z{"msg":"Crucible stats registered with UUID: d6b9b286-53eb-4d7b-835d-78cba04bfdcd","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.101179522Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32382024-04-17T17:12:32.500Z{"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.101220705Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288}
32392024-04-17T17:12:32.500Z{"msg":"d6b9b286-53eb-4d7b-835d-78cba04bfdcd active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.101268028Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32402024-04-17T17:12:32.500Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.10131158Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"client":"0","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32412024-04-17T17:12:32.500Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.101365374Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"client":"0","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32422024-04-17T17:12:32.500Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.101398986Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"client":"1","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32432024-04-17T17:12:32.500Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.101519133Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"client":"1","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32442024-04-17T17:12:32.500Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.101572026Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"client":"2","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32452024-04-17T17:12:32.500Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.101628249Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"client":"2","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32462024-04-17T17:12:32.500Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.101670742Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32472024-04-17T17:12:32.500Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.101738136Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32482024-04-17T17:12:32.500Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.10180176Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32492024-04-17T17:12:32.500Z{"msg":"ds_connection connected from Ok(127.0.0.1:43132)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.101840182Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32502024-04-17T17:12:32.500Z{"msg":"ds_connection connected from Ok(127.0.0.1:49226)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.101874004Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32512024-04-17T17:12:32.500Z{"msg":"ds_connection connected from Ok(127.0.0.1:62915)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.101897336Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32522024-04-17T17:12:32.500Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.102324212Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"client":"0","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32532024-04-17T17:12:32.500Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.102370094Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"client":"0","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32542024-04-17T17:12:32.500Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.102408637Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"client":"2","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32552024-04-17T17:12:32.500Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.102436469Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"client":"2","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32562024-04-17T17:12:32.500Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.102778599Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"client":"1","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32572024-04-17T17:12:32.500Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.102828092Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"client":"1","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32582024-04-17T17:12:32.501Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.102854964Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"client":"2","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32592024-04-17T17:12:32.501Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.103056066Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"client":"0","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32602024-04-17T17:12:32.501Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.103400997Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"client":"1","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32612024-04-17T17:12:32.501Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.10344519Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"client":"2","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32622024-04-17T17:12:32.501Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.103466291Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"client":"2","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32632024-04-17T17:12:32.501Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.103499333Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32642024-04-17T17:12:32.501Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.103516074Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"client":"0","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32652024-04-17T17:12:32.501Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.103536635Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"client":"0","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32662024-04-17T17:12:32.501Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.103550256Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32672024-04-17T17:12:32.501Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.103878296Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"client":"1","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32682024-04-17T17:12:32.501Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.103920388Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"client":"1","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32692024-04-17T17:12:32.501Z{"msg":"[0]R flush_numbers[0..12]: [55, 56, 50, 49, 45, 51, 50, 44, 50, 51, 56, 51]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.1039375Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"":"downstairs","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32702024-04-17T17:12:32.501Z{"msg":"[0]R generation[0..12]: [28, 29, 27, 26, 25, 28, 27, 24, 27, 28, 29, 28]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.103959821Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"":"downstairs","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32712024-04-17T17:12:32.501Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.103974822Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"":"downstairs","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32722024-04-17T17:12:32.501Z{"msg":"[1]R flush_numbers[0..12]: [55, 56, 50, 49, 45, 51, 50, 44, 50, 51, 56, 51]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.103988773Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"":"downstairs","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32732024-04-17T17:12:32.501Z{"msg":"[1]R generation[0..12]: [28, 29, 27, 26, 25, 28, 27, 24, 27, 28, 29, 28]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.104009244Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"":"downstairs","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32742024-04-17T17:12:32.501Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.104023795Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"":"downstairs","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32752024-04-17T17:12:32.501ZWait for a query_work_queue command to finish before sending IO
32762024-04-17T17:12:32.501Z{"msg":"[2]R flush_numbers[0..12]: [55, 56, 50, 49, 45, 51, 50, 44, 50, 51, 56, 51]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.104047536Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"":"downstairs","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32772024-04-17T17:12:32.501Z{"msg":"[2]R generation[0..12]: [28, 29, 27, 26, 25, 28, 27, 24, 27, 28, 29, 28]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.104061937Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"":"downstairs","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32782024-04-17T17:12:32.501Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.104082478Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"":"downstairs","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32792024-04-17T17:12:32.501Z{"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.104095659Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"":"downstairs","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32802024-04-17T17:12:32.502Z{"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.104117851Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"":"downstairs","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32812024-04-17T17:12:32.502Z{"msg":"Next flush: 57","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.104130311Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"":"downstairs","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32822024-04-17T17:12:32.502Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.104158393Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"":"downstairs","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32832024-04-17T17:12:32.502Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.104181314Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32842024-04-17T17:12:32.502Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.104207556Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"client":"0","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32852024-04-17T17:12:32.502Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.104242248Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"client":"1","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32862024-04-17T17:12:32.502Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
32872024-04-17T17:12:32.502Z,"time":"2024-04-17T17:12:31.10427331Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"client":"2","":"downstairs","session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32882024-04-17T17:12:32.502Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.104308292Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32892024-04-17T17:12:32.502Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.104322523Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32902024-04-17T17:12:32.502Z{"msg":"d6b9b286-53eb-4d7b-835d-78cba04bfdcd is now active with session: 658c437a-602f-4b71-bf2d-e557e46d1e2a","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.104335854Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32912024-04-17T17:12:32.502Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.104348514Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288,"session_id":"658c437a-602f-4b71-bf2d-e557e46d1e2a"}
32922024-04-17T17:12:32.502Z{"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:31.104373546Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288}
32932024-04-17T17:12:32.507ZLoading write count information from file "/tmp/test_repair_verify.data"
32942024-04-17T17:12:32.507ZRun Repair workload
32952024-04-17T17:12:32.507Z01/30 Write block 201 len 28672 data: 3 3 4 4 4 3 3
32962024-04-17T17:12:32.507Z02/30 Write block 111 len 12288 data: 5 5 4
32972024-04-17T17:12:32.507Z03/30 Read block 136 len 40960
32982024-04-17T17:12:32.507Z04/30 Read block 516 len 36864
32992024-04-17T17:12:32.507Z05/30 Flush
33002024-04-17T17:12:32.507Z06/30 Write block 196 len 24576 data: 2 2 3 3 3 4
33012024-04-17T17:12:32.513Z07/30 Write block 433 len 8192 data: 3 2
33022024-04-17T17:12:32.513Z08/30 Read block 38 len 16384
33032024-04-17T17:12:32.513Z09/30 Write block 371 len 8192 data: 4 3
33042024-04-17T17:12:32.513Z10/30 Flush
33052024-04-17T17:12:32.517Z11/30 Write block 223 len 28672 data: 4 4 4 5 5 3 3
33062024-04-17T17:12:32.517Z12/30 Write block 364 len 32768 data: 3 2 2 2 3 3 3 5
33072024-04-17T17:12:32.517Z13/30 Read block 349 len 36864
33082024-04-17T17:12:32.517Z14/30 Read block 31 len 32768
33092024-04-17T17:12:32.517Z15/30 Flush
33102024-04-17T17:12:32.521Z16/30 Write block 464 len 16384 data: 4 3 3 2
33112024-04-17T17:12:32.521Z17/30 Write block 572 len 36864 data: 4 3 3 4 4 4 4 6 5
33122024-04-17T17:12:32.521Z18/30 Write block 288 len 12288 data: 4 5 5
33132024-04-17T17:12:32.521Z19/30 Write block 429 len 12288 data: 4 2 2
33142024-04-17T17:12:32.528Z20/30 Write block 180 len 4096 data: 3
33152024-04-17T17:12:32.528Z21/30 Write block 240 len 12288 data: 4 4 4
33162024-04-17T17:12:32.528Z22/30 Write block 90 len 20480 data: 5 6 6 6 5
33172024-04-17T17:12:32.528Z23/30 Flush
33182024-04-17T17:12:32.532Z24/30 Write block 57 len 4096 data: 4
33192024-04-17T17:12:32.532Z25/30 Read block 433 len 24576
33202024-04-17T17:12:32.532Z26/30 Write block 493 len 28672 data: 3 3 4 4 4 4 4
33212024-04-17T17:12:32.533Z27/30 Read block 221 len 4096
33222024-04-17T17:12:32.533Z28/30 Read block 383 len 32768
33232024-04-17T17:12:32.533Z29/30 Read block 587 len 28672
33242024-04-17T17:12:32.533Z30/30 Write block 224 len 20480 data: 5 5 6 6 4
33252024-04-17T17:12:32.533Z----------------------------------------------------------------
33262024-04-17T17:12:32.533Z Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:25
33272024-04-17T17:12:32.533ZStates: Active Active Active
33282024-04-17T17:12:32.533ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
33292024-04-17T17:12:32.533Z 6 Acked 1005 Write 6 Done Done Done false
33302024-04-17T17:12:32.533Z 7 Acked 1006 Write 2 Done Done Done false
33312024-04-17T17:12:32.533Z 8 Acked 1007 Read 4 Done Sent Done false
33322024-04-17T17:12:32.533Z 9 Acked 1008 Write 2 Done Sent Done false
33332024-04-17T17:12:32.533Z 10 Acked 1009 Flush 0 Done Sent Done false
33342024-04-17T17:12:32.533Z 11 Acked 1010 Write 7 Done Sent Done false
33352024-04-17T17:12:32.533Z 12 Acked 1011 Write 8 Done Sent Done false
33362024-04-17T17:12:32.533Z 13 Acked 1012 Read 9 Done Sent Done false
33372024-04-17T17:12:32.533Z 14 Acked 1013 Read 8 Done Sent Done false
33382024-04-17T17:12:32.533Z 15 Acked 1014 Flush 0 Done Sent Done false
33392024-04-17T17:12:32.533Z 16 Acked 1015 Write 4 Done Sent Done false
33402024-04-17T17:12:32.533Z 17 Acked 1016 Write 9 Done Sent Done false
33412024-04-17T17:12:32.533Z 18 Acked 1017 Write 3 Done Sent Done false
33422024-04-17T17:12:32.533Z 19 Acked 1018 Write 3 Done Sent Done false
33432024-04-17T17:12:32.533Z 20 Acked 1019 Write 1 Done Sent Done false
33442024-04-17T17:12:32.533Z 21 Acked 1020 Write 3 Done Sent Done false
33452024-04-17T17:12:32.534Z 22 Acked 1021 Write 5 Done Sent Done false
33462024-04-17T17:12:32.534Z 23 Acked 1022 Flush 0 Done Sent Done false
33472024-04-17T17:12:32.534Z 24 Acked 1023 Write 1 Done Sent Done false
33482024-04-17T17:12:32.534Z 25 Acked 1024 Read 6 Done Sent Done false
33492024-04-17T17:12:32.534Z 26 Acked 1025 Write 7 Done Sent Done false
33502024-04-17T17:12:32.534Z 27 Acked 1026 Read 1 Done Sent Done false
33512024-04-17T17:12:32.534Z 28 Acked 1027 Read 8 Done Sent Done false
33522024-04-17T17:12:32.534Z 29 Acked 1028 Read 7 Done Sent Done false
33532024-04-17T17:12:32.534Z 30 Acked 1029 Write 5 Sent Sent Sent false
33542024-04-17T17:12:32.534Z STATES DS:0 DS:1 DS:2 TOTAL
33552024-04-17T17:12:32.534Z New 0 0 0 0
33562024-04-17T17:12:32.534Z Sent 1 23 1 25
33572024-04-17T17:12:32.534Z Done 24 2 24 50
33582024-04-17T17:12:32.534Z Skipped 0 0 0 0
33592024-04-17T17:12:32.534Z Error 0 0 0 0
33602024-04-17T17:12:32.534ZLast Flush: 1022 1004 1022
33612024-04-17T17:12:32.534ZDownstairs last five completed: 1004 1003 1002 1001 1000
33622024-04-17T17:12:32.534ZUpstairs last five completed: 30 29 28 27 26
33632024-04-17T17:12:32.538Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:12:31.138819223Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1288}
33642024-04-17T17:12:32.538ZWrote out file "/tmp/test_repair_verify.data"
33652024-04-17T17:12:32.538Z
33662024-04-17T17:12:32.541ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
33672024-04-17T17:12:32.556ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
33682024-04-17T17:12:32.556Z 0 000-019 28 28 28 55 55 55 F F F
33692024-04-17T17:12:32.560Z 1 020-039 29 29 29 56 56 56 F F F
33702024-04-17T17:12:32.560Z 2 040-059 27 27 27 50 50 50 T F T <---
33712024-04-17T17:12:32.560Z 3 060-079 26 26 26 49 49 49 F F F
33722024-04-17T17:12:32.560Z 4 080-099 30 25 30 60 45 60 F F F <---
33732024-04-17T17:12:32.560Z 5 100-119 30 30 30 57 57 57 F F F
33742024-04-17T17:12:32.560Z 6 120-139 27 27 27 50 50 50 F F F
33752024-04-17T17:12:32.561Z 7 140-159 24 24 24 44 44 44 F F F
33762024-04-17T17:12:32.561Z 8 160-179 27 27 27 50 50 50 F F F
33772024-04-17T17:12:32.561Z 9 180-199 30 28 30 60 51 60 F T F <---
33782024-04-17T17:12:32.561Z 10 200-219 30 30 30 58 57 58 F T F <---
33792024-04-17T17:12:32.561Z 11 220-239 30 28 30 59 51 59 T F T <---
33802024-04-17T17:12:32.561Z 12 240-259 30 29 30 60 56 60 F F F <---
33812024-04-17T17:12:32.561Z 13 260-279 26 26 26 47 47 47 F F F
33822024-04-17T17:12:32.561Z 14 280-299 30 28 30 60 51 60 F F F <---
33832024-04-17T17:12:32.561Z 15 300-319 24 24 24 44 44 44 F F F
33842024-04-17T17:12:32.561Z 16 320-339 23 23 23 43 43 43 F F F
33852024-04-17T17:12:32.561Z 17 340-359 28 28 28 53 53 53 F F F
33862024-04-17T17:12:32.561Z 18 360-379 30 24 30 59 44 59 F F F <---
33872024-04-17T17:12:32.561Z 19 380-399 26 26 26 49 49 49 F F F
33882024-04-17T17:12:32.561Z 20 400-419 28 28 28 51 51 51 F F F
33892024-04-17T17:12:32.561Z 21 420-439 30 27 30 60 50 60 F T F <---
33902024-04-17T17:12:32.561Z 22 440-459 27 27 27 50 50 50 F F F
33912024-04-17T17:12:32.561Z 23 460-479 30 18 30 60 34 60 F F F <---
33922024-04-17T17:12:32.561Z 24 480-499 26 26 26 47 47 47 T F T <---
33932024-04-17T17:12:32.561Z 25 500-519 26 26 26 48 48 48 F F F
33942024-04-17T17:12:32.561Z 26 520-539 29 29 29 56 56 56 F F F
33952024-04-17T17:12:32.561Z 27 540-559 27 27 27 50 50 50 F F F
33962024-04-17T17:12:32.561Z 28 560-579 30 18 30 60 35 60 F F F <---
33972024-04-17T17:12:32.562Z 29 580-599 30 23 30 60 43 60 F F F <---
33982024-04-17T17:12:32.562ZMax gen: 30, Max flush: 60
33992024-04-17T17:12:32.562ZError: Difference in extent metadata found!
34002024-04-17T17:12:32.562ZA Difference in extent metadata is expected here
34012024-04-17T17:12:32.562ZOn loop 15
34022024-04-17T17:12:32.562Z
34032024-04-17T17:12:32.565ZVerifying data now
34042024-04-17T17:12:47.172ZNo change after verify
34052024-04-17T17:12:47.172ZLoop: 15 Downstairs dump after verify (and repair):
34062024-04-17T17:12:47.183ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
34072024-04-17T17:12:47.183Z 0 000-019 28 28 28 55 55 55 F F F
34082024-04-17T17:12:47.183Z 1 020-039 29 29 29 56 56 56 F F F
34092024-04-17T17:12:47.183Z 2 040-059 31 31 31 61 61 61 F F F
34102024-04-17T17:12:47.183Z 3 060-079 26 26 26 49 49 49 F F F
34112024-04-17T17:12:47.183Z 4 080-099 30 30 30 60 60 60 F F F
34122024-04-17T17:12:47.187Z 5 100-119 30 30 30 57 57 57 F F F
34132024-04-17T17:12:47.187Z 6 120-139 27 27 27 50 50 50 F F F
34142024-04-17T17:12:47.187Z 7 140-159 24 24 24 44 44 44 F F F
34152024-04-17T17:12:47.187Z 8 160-179 27 27 27 50 50 50 F F F
34162024-04-17T17:12:47.187Z 9 180-199 30 30 30 60 60 60 F F F
34172024-04-17T17:12:47.187Z 10 200-219 30 30 30 58 58 58 F F F
34182024-04-17T17:12:47.187Z 11 220-239 31 31 31 61 61 61 F F F
34192024-04-17T17:12:47.188Z 12 240-259 30 30 30 60 60 60 F F F
34202024-04-17T17:12:47.188Z 13 260-279 26 26 26 47 47 47 F F F
34212024-04-17T17:12:47.188Z 14 280-299 30 30 30 60 60 60 F F F
34222024-04-17T17:12:47.188Z 15 300-319 24 24 24 44 44 44 F F F
34232024-04-17T17:12:47.188Z 16 320-339 23 23 23 43 43 43 F F F
34242024-04-17T17:12:47.188Z 17 340-359 28 28 28 53 53 53 F F F
34252024-04-17T17:12:47.188Z 18 360-379 30 30 30 59 59 59 F F F
34262024-04-17T17:12:47.188Z 19 380-399 26 26 26 49 49 49 F F F
34272024-04-17T17:12:47.188Z 20 400-419 28 28 28 51 51 51 F F F
34282024-04-17T17:12:47.188Z 21 420-439 30 30 30 60 60 60 F F F
34292024-04-17T17:12:47.188Z 22 440-459 27 27 27 50 50 50 F F F
34302024-04-17T17:12:47.188Z 23 460-479 30 30 30 60 60 60 F F F
34312024-04-17T17:12:47.188Z 24 480-499 31 31 31 61 61 61 F F F
34322024-04-17T17:12:47.188Z 25 500-519 26 26 26 48 48 48 F F F
34332024-04-17T17:12:47.188Z 26 520-539 29 29 29 56 56 56 F F F
34342024-04-17T17:12:47.188Z 27 540-559 27 27 27 50 50 50 F F F
34352024-04-17T17:12:47.188Z 28 560-579 30 30 30 60 60 60 F F F
34362024-04-17T17:12:47.188Z 29 580-599 30 30 30 60 60 60 F F F
34372024-04-17T17:12:47.188ZMax gen: 31, Max flush: 61
34382024-04-17T17:12:47.188Z
34392024-04-17T17:12:47.188ZBegin loop 16
34402024-04-17T17:12:47.188ZDownstairs to restart: 2
34412024-04-17T17:12:47.205Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.812445752Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296}
34422024-04-17T17:12:47.206Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.812695177Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296}
34432024-04-17T17:12:47.212Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.812737359Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296}
34442024-04-17T17:12:47.212Z{"msg":"Crucible runtime is spawned
34452024-04-17T17:12:47.212ZCrucible adbbd68b-10e0-4b88-9e25-1afcdca3bf01 has session id: b398098a-96df-4350-87c6-96b9966e49f1","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.812839356Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34462024-04-17T17:12:47.213Z{"msg":"Upstairs opts: Upstairs UUID: adbbd68b-10e0-4b88-9e25-1afcdca3bf01, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.812900399Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34472024-04-17T17:12:47.213Z{"msg":"Crucible stats registered with UUID: adbbd68b-10e0-4b88-9e25-1afcdca3bf01","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.812945212Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34482024-04-17T17:12:47.213Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.812969904Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296}
34492024-04-17T17:12:47.213Z{"msg":"adbbd68b-10e0-4b88-9e25-1afcdca3bf01 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.812989855Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34502024-04-17T17:12:47.213Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.813011646Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"client":"0","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34512024-04-17T17:12:47.213Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.813036288Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"client":"0","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34522024-04-17T17:12:47.213Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.8130724Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"client":"1","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34532024-04-17T17:12:47.213Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.813119602Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"client":"1","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34542024-04-17T17:12:47.213Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.813156755Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"client":"2","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34552024-04-17T17:12:47.213Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.813183746Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"client":"2","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34562024-04-17T17:12:47.213Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.813206118Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34572024-04-17T17:12:47.213Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.81324666Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34582024-04-17T17:12:47.213Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.813352176Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34592024-04-17T17:12:47.213Z{"msg":"ds_connection connected from Ok(127.0.0.1:51193)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.813399399Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34602024-04-17T17:12:47.213Z{"msg":"ds_connection connected from Ok(127.0.0.1:49716)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.813441582Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34612024-04-17T17:12:47.213Z{"msg":"ds_connection connected from Ok(127.0.0.1:36322)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.813479544Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34622024-04-17T17:12:47.213Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.814134314Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"client":"0","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34632024-04-17T17:12:47.213Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.814175277Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"client":"0","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34642024-04-17T17:12:47.213Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.814200968Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"client":"1","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34652024-04-17T17:12:47.213Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.8142329Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"client":"1","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34662024-04-17T17:12:47.213Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.814273883Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"client":"2","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34672024-04-17T17:12:47.213Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.814308065Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"client":"2","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34682024-04-17T17:12:47.213Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.814712589Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"client":"1","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34692024-04-17T17:12:47.214Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.814841897Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"client":"0","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34702024-04-17T17:12:47.214Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.814880199Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"client":"2","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34712024-04-17T17:12:47.214Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.815200639Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"client":"1","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34722024-04-17T17:12:47.214Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.815247111Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"client":"1","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34732024-04-17T17:12:47.214Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.815290114Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34742024-04-17T17:12:47.214Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.815315755Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"client":"0","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34752024-04-17T17:12:47.214Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.815347268Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"client":"0","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34762024-04-17T17:12:47.214Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.815368149Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34772024-04-17T17:12:47.214Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.815401561Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"client":"2","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34782024-04-17T17:12:47.214Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.815431102Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"client":"2","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34792024-04-17T17:12:47.214Z{"msg":"[0]R flush_numbers[0..12]: [55, 56, 61, 49, 60, 57, 50, 44, 50, 60, 58, 61]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.815446574Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"":"downstairs","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34802024-04-17T17:12:47.214Z{"msg":"[0]R generation[0..12]: [28, 29, 31, 26, 30, 30, 27, 24, 27, 30, 30, 31]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.815468395Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"":"downstairs","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34812024-04-17T17:12:47.214Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.815483076Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"":"downstairs","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34822024-04-17T17:12:47.214Z{"msg":"[1]R flush_numbers[0..12]: [55, 56, 61, 49, 60, 57, 50, 44, 50, 60, 58, 61]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.815503607Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"":"downstairs","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34832024-04-17T17:12:47.214Z{"msg":"[1]R generation[0..12]: [28, 29, 31, 26, 30, 30, 27, 24, 27, 30, 30, 31]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.815517898Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"":"downstairs","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34842024-04-17T17:12:47.214Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.815538339Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"":"downstairs","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34852024-04-17T17:12:47.214Z{"msg":"[2]R flush_numbers[0..12]: [55, 56, 61, 49, 60, 57, 50, 44, 50, 60, 58, 61]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.81555215Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"":"downstairs","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34862024-04-17T17:12:47.214Z{"msg":"[2]R generation[0..12]: [28, 29, 31, 26, 30, 30, 27, 24, 27, 30, 30, 31]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
34872024-04-17T17:12:47.214Z,"time":"2024-04-17T17:12:45.815566351Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"":"downstairs","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34882024-04-17T17:12:47.214Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.815584592Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"":"downstairs","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34892024-04-17T17:12:47.214Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.815604493Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"":"downstairs","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34902024-04-17T17:12:47.214Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.815618744Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"":"downstairs","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34912024-04-17T17:12:47.214Z{"msg":"Next flush: 62","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.815638085Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"":"downstairs","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34922024-04-17T17:12:47.215Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.815650956Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"":"downstairs","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34932024-04-17T17:12:47.215Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.815671047Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34942024-04-17T17:12:47.215Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.815683058Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"client":"0","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34952024-04-17T17:12:47.215Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.815708499Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"client":"1","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34962024-04-17T17:12:47.215Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.81572213Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"client":"2","":"downstairs","session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34972024-04-17T17:12:47.215Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.815749622Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34982024-04-17T17:12:47.215Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.815762533Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
34992024-04-17T17:12:47.215Z{"msg":"adbbd68b-10e0-4b88-9e25-1afcdca3bf01 is now active with session: b398098a-96df-4350-87c6-96b9966e49f1","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.815775114Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
35002024-04-17T17:12:47.215Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.815786784Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296,"session_id":"b398098a-96df-4350-87c6-96b9966e49f1"}
35012024-04-17T17:12:47.215Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:45.815798845Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296}
35022024-04-17T17:12:47.215ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
35032024-04-17T17:12:47.215ZLoading write count information from file "/tmp/test_repair_verify.data"
35042024-04-17T17:12:47.215ZRun Repair workload
35052024-04-17T17:12:47.233Z01/30 Write block 568 len 8192 data: 3 4
35062024-04-17T17:12:47.233Z02/30 Write block 221 len 36864 data: 4 4 5 6 6 7 7 5 4
35072024-04-17T17:12:47.233Z03/30 Read block 563 len 36864
35082024-04-17T17:12:47.233Z04/30 Read block 125 len 36864
35092024-04-17T17:12:47.233Z05/30 Read block 123 len 36864
35102024-04-17T17:12:47.233Z06/30 Flush
35112024-04-17T17:12:47.233Z07/30 Write block 337 len 40960 data: 5 6 6 5 5 6 6 6 6 5
35122024-04-17T17:12:47.233Z08/30 Read block 156 len 24576
35132024-04-17T17:12:47.233Z09/30 Flush
35142024-04-17T17:12:47.233Z10/30 Write block 313 len 8192 data: 4 4
35152024-04-17T17:12:47.234Z11/30 Read block 567 len 36864
35162024-04-17T17:12:47.234Z12/30 Write block 154 len 16384 data: 2 3 3 3
35172024-04-17T17:12:47.234Z13/30 Write block 331 len 28672 data: 2 2 3 4 5 4 6
35182024-04-17T17:12:47.234Z14/30 Read block 233 len 12288
35192024-04-17T17:12:47.234Z15/30 Read block 293 len 12288
35202024-04-17T17:12:47.234Z16/30 Read block 515 len 16384
35212024-04-17T17:12:47.234Z17/30 Write block 132 len 12288 data: 5 4 5
35222024-04-17T17:12:47.234Z18/30 Write block 116 len 8192 data: 3 3
35232024-04-17T17:12:47.234Z19/30 Write block 105 len 24576 data: 4 5 5 5 4 4
35242024-04-17T17:12:47.234Z20/30 Read block 499 len 24576
35252024-04-17T17:12:47.234Z21/30 Write block 412 len 36864 data: 8 8 8 7 6 5 5 5 5
35262024-04-17T17:12:47.234Z22/30 Write block 346 len 8192 data: 6 5
35272024-04-17T17:12:47.234Z23/30 Read block 390 len 36864
35282024-04-17T17:12:47.243Z24/30 Write block 33 len 8192 data: 9 9
35292024-04-17T17:12:47.243Z25/30 Write block 477 len 32768 data: 4 5 6 6 5 5 5 4
35302024-04-17T17:12:47.244Z26/30 Read block 0 len 36864
35312024-04-17T17:12:47.248Z27/30 Write block 51 len 32768 data: 2 2 2 3 3 3 5 4
35322024-04-17T17:12:47.248Z28/30 Write block 235 len 4096 data: 3
35332024-04-17T17:12:47.248Z29/30 Read block 105 len 24576
35342024-04-17T17:12:47.248Z30/30 Write block 103 len 16384 data: 4 4 5 6
35352024-04-17T17:12:47.248Z----------------------------------------------------------------
35362024-04-17T17:12:47.248Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30
35372024-04-17T17:12:47.248ZStates: Active Active Active
35382024-04-17T17:12:47.248ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
35392024-04-17T17:12:47.248Z 1 Acked 1000 Write 2 Done Done Done false
35402024-04-17T17:12:47.248Z 2 Acked 1001 Write 9 Done Done Sent false
35412024-04-17T17:12:47.248Z 3 Acked 1002 Read 9 Done Done Sent false
35422024-04-17T17:12:47.248Z 4 Acked 1003 Read 9 Done Done Sent false
35432024-04-17T17:12:47.248Z 5 Acked 1004 Read 9 Done Done Sent false
35442024-04-17T17:12:47.248Z 6 Acked 1005 Flush 0 Done Done Sent false
35452024-04-17T17:12:47.249Z 7 Acked 1006 Write 10 Done Done Sent false
35462024-04-17T17:12:47.249Z 8 Acked 1007 Read 6 Done Done Sent false
35472024-04-17T17:12:47.249Z 9 Acked 1008 Flush 0 Done Done Sent false
35482024-04-17T17:12:47.249Z 10 Acked 1009 Write 2 Done Done Sent false
35492024-04-17T17:12:47.249Z 11 Acked 1010 Read 9 Done Done Sent false
35502024-04-17T17:12:47.249Z 12 Acked 1011 Write 4 Done Done Sent false
35512024-04-17T17:12:47.249Z 13 Acked 1012 Write 7 Done Done Sent false
35522024-04-17T17:12:47.249Z 14 Acked 1013 Read 3 Done Done Sent false
35532024-04-17T17:12:47.249Z 15 Acked 1014 Read 3 Done Done Sent false
35542024-04-17T17:12:47.249Z 16 Acked 1015 Read 4 Done Done Sent false
35552024-04-17T17:12:47.249Z 17 Acked 1016 Write 3 Done Done Sent false
35562024-04-17T17:12:47.249Z 18 Acked 1017 Write 2 Done Done Sent false
35572024-04-17T17:12:47.249Z 19 Acked 1018 Write 6 Done Done Sent false
35582024-04-17T17:12:47.249Z 20 Acked 1019 Read 6 Done Done Sent false
35592024-04-17T17:12:47.249Z 21 Acked 1020 Write 9 Done Done Sent false
35602024-04-17T17:12:47.249Z 22 Acked 1021 Write 2 Done Done Sent false
35612024-04-17T17:12:47.249Z 23 Acked 1022 Read 9 Done Done Sent false
35622024-04-17T17:12:47.249Z 24 Acked 1023 Write 2 Done Done Sent false
35632024-04-17T17:12:47.249Z 25 Acked 1024 Write 8 Done Done Sent false
35642024-04-17T17:12:47.249Z 26 Acked 1025 Read 9 Done Done Sent false
35652024-04-17T17:12:47.249Z 27 Acked 1026 Write 8 Done Done Sent false
35662024-04-17T17:12:47.249Z 28 Acked 1027 Write 1 Done Done Sent false
35672024-04-17T17:12:47.249Z 29 Acked 1028 Read 6 Done Done Sent false
35682024-04-17T17:12:47.249Z 30 Acked 1029 Write 4 Sent Sent Sent false
35692024-04-17T17:12:47.250Z STATES DS:0 DS:1 DS:2 TOTAL
35702024-04-17T17:12:47.250Z New 0 0 0 0
35712024-04-17T17:12:47.250Z Sent 1 1 29 31
35722024-04-17T17:12:47.250Z Done 29 29 1 59
35732024-04-17T17:12:47.250Z Skipped 0 0 0 0
35742024-04-17T17:12:47.250Z Error 0 0 0 0
35752024-04-17T17:12:47.250ZLast Flush: 1008 1008 0
35762024-04-17T17:12:47.250ZDownstairs last five completed:
35772024-04-17T17:12:47.250ZUpstairs last five completed: 30 29 28 27 26
35782024-04-17T17:12:47.250Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:12:45.846521447Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1296}
35792024-04-17T17:12:47.250ZWrote out file "/tmp/test_repair_verify.data"
35802024-04-17T17:12:47.250Z
35812024-04-17T17:12:47.250ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
35822024-04-17T17:12:47.266ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
35832024-04-17T17:12:47.266Z 0 000-019 28 28 28 55 55 55 F F F
35842024-04-17T17:12:47.270Z 1 020-039 29 29 29 56 56 56 T T F <---
35852024-04-17T17:12:47.270Z 2 040-059 31 31 31 61 61 61 T T F <---
35862024-04-17T17:12:47.270Z 3 060-079 26 26 26 49 49 49 F F F
35872024-04-17T17:12:47.270Z 4 080-099 30 30 30 60 60 60 F F F
35882024-04-17T17:12:47.270Z 5 100-119 30 30 30 57 57 57 T T F <---
35892024-04-17T17:12:47.270Z 6 120-139 27 27 27 50 50 50 T T F <---
35902024-04-17T17:12:47.270Z 7 140-159 24 24 24 44 44 44 T T F <---
35912024-04-17T17:12:47.270Z 8 160-179 27 27 27 50 50 50 F F F
35922024-04-17T17:12:47.270Z 9 180-199 30 30 30 60 60 60 F F F
35932024-04-17T17:12:47.270Z 10 200-219 30 30 30 58 58 58 F F F
35942024-04-17T17:12:47.270Z 11 220-239 32 32 31 62 62 61 T T F <---
35952024-04-17T17:12:47.270Z 12 240-259 30 30 30 60 60 60 F F F
35962024-04-17T17:12:47.270Z 13 260-279 26 26 26 47 47 47 F F F
35972024-04-17T17:12:47.271Z 14 280-299 30 30 30 60 60 60 F F F
35982024-04-17T17:12:47.271Z 15 300-319 24 24 24 44 44 44 T T F <---
35992024-04-17T17:12:47.271Z 16 320-339 32 32 23 63 63 43 T T F <---
36002024-04-17T17:12:47.271Z 17 340-359 32 32 28 63 63 53 T T F <---
36012024-04-17T17:12:47.271Z 18 360-379 30 30 30 59 59 59 F F F
36022024-04-17T17:12:47.271Z 19 380-399 26 26 26 49 49 49 F F F
36032024-04-17T17:12:47.271Z 20 400-419 28 28 28 51 51 51 T T F <---
36042024-04-17T17:12:47.271Z 21 420-439 30 30 30 60 60 60 T T F <---
36052024-04-17T17:12:47.271Z 22 440-459 27 27 27 50 50 50 F F F
36062024-04-17T17:12:47.271Z 23 460-479 30 30 30 60 60 60 T T F <---
36072024-04-17T17:12:47.271Z 24 480-499 31 31 31 61 61 61 T T F <---
36082024-04-17T17:12:47.271Z 25 500-519 26 26 26 48 48 48 F F F
36092024-04-17T17:12:47.271Z 26 520-539 29 29 29 56 56 56 F F F
36102024-04-17T17:12:47.271Z 27 540-559 27 27 27 50 50 50 F F F
36112024-04-17T17:12:47.271Z 28 560-579 32 32 30 62 62 60 F F T <---
36122024-04-17T17:12:47.271Z 29 580-599 30 30 30 60 60 60 F F F
36132024-04-17T17:12:47.271ZMax gen: 32, Max flush: 63
36142024-04-17T17:12:47.271ZError: Difference in extent metadata found!
36152024-04-17T17:12:47.271ZA Difference in extent metadata is expected here
36162024-04-17T17:12:47.271ZOn loop 16
36172024-04-17T17:12:47.271Z
36182024-04-17T17:12:47.275ZVerifying data now
36192024-04-17T17:12:51.937ZNo change after verify
36202024-04-17T17:12:51.937ZLoop: 16 Downstairs dump after verify (and repair):
36212024-04-17T17:12:51.952ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
36222024-04-17T17:12:51.952Z 0 000-019 28 28 28 55 55 55 F F F
36232024-04-17T17:12:51.956Z 1 020-039 33 33 33 64 64 64 F F F
36242024-04-17T17:12:51.956Z 2 040-059 33 33 33 64 64 64 F F F
36252024-04-17T17:12:51.956Z 3 060-079 26 26 26 49 49 49 F F F
36262024-04-17T17:12:51.956Z 4 080-099 30 30 30 60 60 60 F F F
36272024-04-17T17:12:51.956Z 5 100-119 33 33 33 64 64 64 F F F
36282024-04-17T17:12:51.956Z 6 120-139 33 33 33 64 64 64 F F F
36292024-04-17T17:12:51.956Z 7 140-159 33 33 33 64 64 64 F F F
36302024-04-17T17:12:51.956Z 8 160-179 27 27 27 50 50 50 F F F
36312024-04-17T17:12:51.956Z 9 180-199 30 30 30 60 60 60 F F F
36322024-04-17T17:12:51.956Z 10 200-219 30 30 30 58 58 58 F F F
36332024-04-17T17:12:51.956Z 11 220-239 33 33 33 64 64 64 F F F
36342024-04-17T17:12:51.956Z 12 240-259 30 30 30 60 60 60 F F F
36352024-04-17T17:12:51.956Z 13 260-279 26 26 26 47 47 47 F F F
36362024-04-17T17:12:51.956Z 14 280-299 30 30 30 60 60 60 F F F
36372024-04-17T17:12:51.956Z 15 300-319 33 33 33 64 64 64 F F F
36382024-04-17T17:12:51.956Z 16 320-339 33 33 33 64 64 64 F F F
36392024-04-17T17:12:51.956Z 17 340-359 33 33 33 64 64 64 F F F
36402024-04-17T17:12:51.956Z 18 360-379 30 30 30 59 59 59 F F F
36412024-04-17T17:12:51.956Z 19 380-399 26 26 26 49 49 49 F F F
36422024-04-17T17:12:51.957Z 20 400-419 33 33 33 64 64 64 F F F
36432024-04-17T17:12:51.957Z 21 420-439 33 33 33 64 64 64 F F F
36442024-04-17T17:12:51.957Z 22 440-459 27 27 27 50 50 50 F F F
36452024-04-17T17:12:51.957Z 23 460-479 33 33 33 64 64 64 F F F
36462024-04-17T17:12:51.957Z 24 480-499 33 33 33 64 64 64 F F F
36472024-04-17T17:12:51.957Z 25 500-519 26 26 26 48 48 48 F F F
36482024-04-17T17:12:51.957Z 26 520-539 29 29 29 56 56 56 F F F
36492024-04-17T17:12:51.957Z 27 540-559 27 27 27 50 50 50 F F F
36502024-04-17T17:12:51.957Z 28 560-579 32 32 32 62 62 62 F F F
36512024-04-17T17:12:51.957Z 29 580-599 30 30 30 60 60 60 F F F
36522024-04-17T17:12:51.957ZMax gen: 33, Max flush: 64
36532024-04-17T17:12:51.960Z
36542024-04-17T17:12:51.961ZBegin loop 17
36552024-04-17T17:12:51.961ZDownstairs to restart: 2
36562024-04-17T17:12:51.976Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.583140201Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304}
36572024-04-17T17:12:51.976ZCrucible runtime is spawned
36582024-04-17T17:12:51.982Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.583440459Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304}
36592024-04-17T17:12:51.982Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.583478091Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304}
36602024-04-17T17:12:51.982Z{"msg":"Crucible c31d0438-babe-4a0f-9c60-48c946ee23cb has session id: 3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.583500992Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36612024-04-17T17:12:51.982Z{"msg":"Upstairs opts: Upstairs UUID: c31d0438-babe-4a0f-9c60-48c946ee23cb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.583544855Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36622024-04-17T17:12:51.982Z{"msg":"Crucible stats registered with UUID: c31d0438-babe-4a0f-9c60-48c946ee23cb","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.583570207Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36632024-04-17T17:12:51.982Z{"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.583584048Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304}
36642024-04-17T17:12:51.982Z{"msg":"c31d0438-babe-4a0f-9c60-48c946ee23cb active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.583602278Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36652024-04-17T17:12:51.982Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.583614949Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"client":"0","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36662024-04-17T17:12:51.982Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.58362942Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"client":"0","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36672024-04-17T17:12:51.982Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.583642991Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"client":"1","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36682024-04-17T17:12:51.982Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.583656712Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"client":"1","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36692024-04-17T17:12:51.982Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.583693244Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"client":"2","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36702024-04-17T17:12:51.982Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.583710245Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"client":"2","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36712024-04-17T17:12:51.982Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.583723796Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36722024-04-17T17:12:51.982Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.583753208Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36732024-04-17T17:12:51.982Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.58378662Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36742024-04-17T17:12:51.982Z{"msg":"ds_connection connected from Ok(127.0.0.1:34588)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.583921958Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36752024-04-17T17:12:51.983Z{"msg":"ds_connection connected from Ok(127.0.0.1:63537)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.583973141Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36762024-04-17T17:12:51.983Z{"msg":"ds_connection connected from Ok(127.0.0.1:34245)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.584206405Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36772024-04-17T17:12:51.983Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.584698855Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"client":"1","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36782024-04-17T17:12:51.983Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.584757729Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"client":"1","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36792024-04-17T17:12:51.983Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.584816732Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"client":"0","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36802024-04-17T17:12:51.983Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.584882146Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"client":"0","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36812024-04-17T17:12:51.983Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.585298571Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"client":"0","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36822024-04-17T17:12:51.983Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.585341814Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"client":"1","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36832024-04-17T17:12:51.983Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.585371736Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"client":"2","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36842024-04-17T17:12:51.983Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.58543194Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"client":"2","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36852024-04-17T17:12:51.983Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.585956051Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"client":"2","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36862024-04-17T17:12:51.983Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.586005354Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"client":"1","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36872024-04-17T17:12:51.983Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.586030206Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"client":"1","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36882024-04-17T17:12:51.983Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.586045147Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36892024-04-17T17:12:51.983Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.586064278Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"client":"0","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36902024-04-17T17:12:51.983Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.586078069Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"client":"0","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36912024-04-17T17:12:51.983Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.58609826Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36922024-04-17T17:12:51.983Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.586497724Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"client":"2","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36932024-04-17T17:12:51.983Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.586533276Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"client":"2","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36942024-04-17T17:12:51.983Z{"msg":"[0]R flush_numbers[0..12]: [55, 64, 64, 49, 60, 64, 64, 64, 50, 60, 58, 64]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.586577279Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"":"downstairs","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36952024-04-17T17:12:51.983Z{"msg":"[0]R generation[0..12]: [28, 33, 33, 26, 30, 33, 33, 33, 27, 30, 30, 33]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.586611921Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"":"downstairs","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36962024-04-17T17:12:51.983Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
36972024-04-17T17:12:51.983Z,"time":"2024-04-17T17:12:50.586636202Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"":"downstairs","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36982024-04-17T17:12:51.984Z{"msg":"[1]R flush_numbers[0..12]: [55, 64, 64, 49, 60, 64, 64, 64, 50, 60, 58, 64]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.586664734Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"":"downstairs","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
36992024-04-17T17:12:51.984Z{"msg":"[1]R generation[0..12]: [28, 33, 33, 26, 30, 33, 33, 33, 27, 30, 30, 33]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.586679555Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"":"downstairs","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
37002024-04-17T17:12:51.984Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.586711767Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"":"downstairs","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
37012024-04-17T17:12:51.984Z{"msg":"[2]R flush_numbers[0..12]: [55, 64, 64, 49, 60, 64, 64, 64, 50, 60, 58, 64]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.586728738Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"":"downstairs","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
37022024-04-17T17:12:51.984Z{"msg":"[2]R generation[0..12]: [28, 33, 33, 26, 30, 33, 33, 33, 27, 30, 30, 33]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.58675984Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"":"downstairs","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
37032024-04-17T17:12:51.984Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.586777741Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"":"downstairs","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
37042024-04-17T17:12:51.984Z{"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.586807083Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"":"downstairs","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
37052024-04-17T17:12:51.984Z{"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.586822064Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"":"downstairs","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
37062024-04-17T17:12:51.984Z{"msg":"Next flush: 65","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.586851955Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"":"downstairs","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
37072024-04-17T17:12:51.984Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.586869837Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"":"downstairs","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
37082024-04-17T17:12:51.984Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.586898018Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
37092024-04-17T17:12:51.984Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.586914039Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"client":"0","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
37102024-04-17T17:12:51.988Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.586946511Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"client":"1","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
37112024-04-17T17:12:51.988Z{Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
37122024-04-17T17:12:51.988Z"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.586981273Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"client":"2","":"downstairs","session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
37132024-04-17T17:12:51.988Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.586998334Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
37142024-04-17T17:12:51.988Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.587027956Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
37152024-04-17T17:12:51.988Z{"msg":"c31d0438-babe-4a0f-9c60-48c946ee23cb is now active with session: 3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.587042657Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
37162024-04-17T17:12:51.988Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.587070909Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304,"session_id":"3bc2dc45-fab4-44bf-88eb-8d15df6ff0d7"}
37172024-04-17T17:12:51.988Z{"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:50.58708724Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304}
37182024-04-17T17:12:51.988ZLoading write count information from file "/tmp/test_repair_verify.data"
37192024-04-17T17:12:51.988ZRun Repair workload
37202024-04-17T17:12:51.988Z01/30 Write block 130 len 12288 data: 3 4 6
37212024-04-17T17:12:51.988Z02/30 Read block 475 len 40960
37222024-04-17T17:12:51.989Z03/30 Write block 433 len 16384 data: 4 3 4 4
37232024-04-17T17:12:51.989Z04/30 Write block 484 len 40960 data: 5 4 5 3 4 4 4 4 4 4
37242024-04-17T17:12:51.989Z05/30 Write block 4 len 8192 data: 5 6
37252024-04-17T17:12:51.989Z06/30 Read block 409 len 24576
37262024-04-17T17:12:51.989Z07/30 Write block 92 len 40960 data: 7 7 6 5 2 2 2 3 3 3
37272024-04-17T17:12:51.992Z08/30 Write block 539 len 8192 data: 4 4
37282024-04-17T17:12:51.992Z09/30 Read block 365 len 36864
37292024-04-17T17:12:51.992Z10/30 Read block 399 len 40960
37302024-04-17T17:12:51.992Z11/30 Flush
37312024-04-17T17:12:51.995Z12/30 Write block 149 len 16384 data: 2 2 2 2
37322024-04-17T17:12:51.995Z13/30 Write block 449 len 40960 data: 4 4 2 3 4 3 3 3 3 4
37332024-04-17T17:12:51.999Z14/30 Write block 455 len 24576 data: 4 4 4 5 4 4
37342024-04-17T17:12:51.999Z15/30 Write block 551 len 28672 data: 3 4 4 4 4 4 4
37352024-04-17T17:12:51.999Z16/30 Write block 66 len 28672 data: 3 3 2 2 2 3 3
37362024-04-17T17:12:51.999Z17/30 Read block 104 len 40960
37372024-04-17T17:12:52.002Z18/30 Read block 139 len 32768
37382024-04-17T17:12:52.003Z19/30 Read block 525 len 24576
37392024-04-17T17:12:52.003Z20/30 Flush
37402024-04-17T17:12:52.006Z21/30 Write block 373 len 12288 data: 3 2 2
37412024-04-17T17:12:52.006Z22/30 Write block 593 len 20480 data: 4 4 4 4 4
37422024-04-17T17:12:52.006Z23/30 Flush
37432024-04-17T17:12:52.010Z24/30 Write block 434 len 8192 data: 4 5
37442024-04-17T17:12:52.010Z25/30 Read block 551 len 36864
37452024-04-17T17:12:52.010Z26/30 Write block 345 len 12288 data: 7 7 6
37462024-04-17T17:12:52.013Z27/30 Write block 156 len 32768 data: 4 4 2 2 2 3 3 3
37472024-04-17T17:12:52.013Z28/30 Write block 390 len 20480 data: 3 2 3 3 3
37482024-04-17T17:12:52.013Z29/30 Write block 29 len 24576 data: 4 4 6 9 10 10
37492024-04-17T17:12:52.017Z30/30 Read block 350 len 40960
37502024-04-17T17:12:52.017Z----------------------------------------------------------------
37512024-04-17T17:12:52.017Z Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30
37522024-04-17T17:12:52.017ZStates: Active Active Active
37532024-04-17T17:12:52.017ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
37542024-04-17T17:12:52.017Z 1 Acked 1000 Write 3 Done Done Done false
37552024-04-17T17:12:52.017Z 2 Acked 1001 Read 10 Done Done Sent false
37562024-04-17T17:12:52.017Z 3 Acked 1002 Write 4 Done Done Sent false
37572024-04-17T17:12:52.017Z 4 Acked 1003 Write 10 Done Done Sent false
37582024-04-17T17:12:52.017Z 5 Acked 1004 Write 2 Done Done Sent false
37592024-04-17T17:12:52.017Z 6 Acked 1005 Read 6 Done Done Sent false
37602024-04-17T17:12:52.017Z 7 Acked 1006 Write 10 Done Done Sent false
37612024-04-17T17:12:52.017Z 8 Acked 1007 Write 2 Done Done Sent false
37622024-04-17T17:12:52.017Z 9 Acked 1008 Read 9 Done Done Sent false
37632024-04-17T17:12:52.017Z 10 Acked 1009 Read 10 Done Done Sent false
37642024-04-17T17:12:52.017Z 11 Acked 1010 Flush 0 Done Done Sent false
37652024-04-17T17:12:52.017Z 12 Acked 1011 Write 4 Done Done Sent false
37662024-04-17T17:12:52.018Z 13 Acked 1012 Write 10 Done Done Sent false
37672024-04-17T17:12:52.018Z 14 Acked 1013 Write 6 Done Done Sent false
37682024-04-17T17:12:52.018Z 15 Acked 1014 Write 7 Done Done Sent false
37692024-04-17T17:12:52.018Z 16 Acked 1015 Write 7 Done Done Sent false
37702024-04-17T17:12:52.018Z 17 Acked 1016 Read 10 Done Done Sent false
37712024-04-17T17:12:52.018Z 18 Acked 1017 Read 8 Done Done Sent false
37722024-04-17T17:12:52.018Z 19 Acked 1018 Read 6 Done Done Sent false
37732024-04-17T17:12:52.018Z 20 Acked 1019 Flush 0 Done Done Sent false
37742024-04-17T17:12:52.018Z 21 Acked 1020 Write 3 Done Done Sent false
37752024-04-17T17:12:52.018Z 22 Acked 1021 Write 5 Done Done Sent false
37762024-04-17T17:12:52.018Z 23 Acked 1022 Flush 0 Done Done Sent false
37772024-04-17T17:12:52.018Z 24 Acked 1023 Write 2 Done Done Sent false
37782024-04-17T17:12:52.018Z 25 Acked 1024 Read 9 Done Done Sent false
37792024-04-17T17:12:52.018Z 26 Acked 1025 Write 3 Done Done Sent false
37802024-04-17T17:12:52.018Z 27 Acked 1026 Write 8 Done Done Sent false
37812024-04-17T17:12:52.018Z 28 Acked 1027 Write 5 Done Done Sent false
37822024-04-17T17:12:52.018Z 29 Acked 1028 Write 6 Done Done Sent false
37832024-04-17T17:12:52.018Z 30 Acked 1029 Read 10 Sent Done Sent false
37842024-04-17T17:12:52.018Z STATES DS:0 DS:1 DS:2 TOTAL
37852024-04-17T17:12:52.018Z New 0 0 0 0
37862024-04-17T17:12:52.018Z Sent 1 0 29 30
37872024-04-17T17:12:52.018Z Done 29 30 1 60
37882024-04-17T17:12:52.018Z Skipped 0 0 0 0
37892024-04-17T17:12:52.019Z Error 0 0 0 0
37902024-04-17T17:12:52.019ZLast Flush: 1022 1022 0
37912024-04-17T17:12:52.019ZDownstairs last five completed:
37922024-04-17T17:12:52.019ZUpstairs last five completed: 30 29 28 27 26
37932024-04-17T17:12:52.019Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:12:50.621694697Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1304}
37942024-04-17T17:12:52.019ZWrote out file "/tmp/test_repair_verify.data"
37952024-04-17T17:12:52.022Z
37962024-04-17T17:12:52.023ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
37972024-04-17T17:12:52.038ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
37982024-04-17T17:12:52.038Z 0 000-019 34 34 28 65 65 55 F F F <---
37992024-04-17T17:12:52.038Z 1 020-039 33 33 33 64 64 64 T T F <---
38002024-04-17T17:12:52.038Z 2 040-059 33 33 33 64 64 64 F F F
38012024-04-17T17:12:52.042Z 3 060-079 34 34 26 66 66 49 F F F <---
38022024-04-17T17:12:52.042Z 4 080-099 34 34 30 65 65 60 F F F <---
38032024-04-17T17:12:52.042Z 5 100-119 34 34 33 65 65 64 F F F <---
38042024-04-17T17:12:52.042Z 6 120-139 34 34 33 65 65 64 F F T <---
38052024-04-17T17:12:52.042Z 7 140-159 34 34 33 66 66 64 T T F <---
38062024-04-17T17:12:52.043Z 8 160-179 27 27 27 50 50 50 T T F <---
38072024-04-17T17:12:52.043Z 9 180-199 30 30 30 60 60 60 F F F
38082024-04-17T17:12:52.043Z 10 200-219 30 30 30 58 58 58 F F F
38092024-04-17T17:12:52.043Z 11 220-239 33 33 33 64 64 64 F F F
38102024-04-17T17:12:52.043Z 12 240-259 30 30 30 60 60 60 F F F
38112024-04-17T17:12:52.043Z 13 260-279 26 26 26 47 47 47 F F F
38122024-04-17T17:12:52.043Z 14 280-299 30 30 30 60 60 60 F F F
38132024-04-17T17:12:52.043Z 15 300-319 33 33 33 64 64 64 F F F
38142024-04-17T17:12:52.043Z 16 320-339 33 33 33 64 64 64 F F F
38152024-04-17T17:12:52.043Z 17 340-359 33 33 33 64 64 64 T T F <---
38162024-04-17T17:12:52.043Z 18 360-379 34 34 30 67 67 59 F F F <---
38172024-04-17T17:12:52.043Z 19 380-399 26 26 26 49 49 49 T T F <---
38182024-04-17T17:12:52.043Z 20 400-419 33 33 33 64 64 64 F F F
38192024-04-17T17:12:52.043Z 21 420-439 34 34 33 65 65 64 T T F <---
38202024-04-17T17:12:52.043Z 22 440-459 34 34 27 66 66 50 F F F <---
38212024-04-17T17:12:52.043Z 23 460-479 34 34 33 66 66 64 F F F <---
38222024-04-17T17:12:52.043Z 24 480-499 34 34 33 65 65 64 F F F <---
38232024-04-17T17:12:52.043Z 25 500-519 26 26 26 48 48 48 F F F
38242024-04-17T17:12:52.043Z 26 520-539 34 34 29 65 65 56 F F F <---
38252024-04-17T17:12:52.043Z 27 540-559 34 34 27 66 66 50 F F F <---
38262024-04-17T17:12:52.043Z 28 560-579 32 32 32 62 62 62 F F F
38272024-04-17T17:12:52.043Z 29 580-599 34 34 30 67 67 60 F F F <---
38282024-04-17T17:12:52.043ZMax gen: 34, Max flush: 67
38292024-04-17T17:12:52.043ZError: Difference in extent metadata found!
38302024-04-17T17:12:52.044ZA Difference in extent metadata is expected here
38312024-04-17T17:12:52.044ZOn loop 17
38322024-04-17T17:12:52.044Z
38332024-04-17T17:12:52.047ZVerifying data now
38342024-04-17T17:12:56.846ZNo change after verify
38352024-04-17T17:12:56.846ZLoop: 17 Downstairs dump after verify (and repair):
38362024-04-17T17:12:56.860ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38372024-04-17T17:12:56.860Z 0 000-019 34 34 34 65 65 65 F F F
38382024-04-17T17:12:56.864Z 1 020-039 35 35 35 68 68 68 F F F
38392024-04-17T17:12:56.864Z 2 040-059 33 33 33 64 64 64 F F F
38402024-04-17T17:12:56.864Z 3 060-079 34 34 34 66 66 66 F F F
38412024-04-17T17:12:56.864Z 4 080-099 34 34 34 65 65 65 F F F
38422024-04-17T17:12:56.864Z 5 100-119 34 34 34 65 65 65 F F F
38432024-04-17T17:12:56.864Z 6 120-139 34 34 34 65 65 65 F F F
38442024-04-17T17:12:56.864Z 7 140-159 35 35 35 68 68 68 F F F
38452024-04-17T17:12:56.864Z 8 160-179 35 35 35 68 68 68 F F F
38462024-04-17T17:12:56.864Z 9 180-199 30 30 30 60 60 60 F F F
38472024-04-17T17:12:56.864Z 10 200-219 30 30 30 58 58 58 F F F
38482024-04-17T17:12:56.864Z 11 220-239 33 33 33 64 64 64 F F F
38492024-04-17T17:12:56.864Z 12 240-259 30 30 30 60 60 60 F F F
38502024-04-17T17:12:56.864Z 13 260-279 26 26 26 47 47 47 F F F
38512024-04-17T17:12:56.864Z 14 280-299 30 30 30 60 60 60 F F F
38522024-04-17T17:12:56.864Z 15 300-319 33 33 33 64 64 64 F F F
38532024-04-17T17:12:56.864Z 16 320-339 33 33 33 64 64 64 F F F
38542024-04-17T17:12:56.864Z 17 340-359 35 35 35 68 68 68 F F F
38552024-04-17T17:12:56.864Z 18 360-379 34 34 34 67 67 67 F F F
38562024-04-17T17:12:56.865Z 19 380-399 35 35 35 68 68 68 F F F
38572024-04-17T17:12:56.865Z 20 400-419 33 33 33 64 64 64 F F F
38582024-04-17T17:12:56.865Z 21 420-439 35 35 35 68 68 68 F F F
38592024-04-17T17:12:56.865Z 22 440-459 34 34 34 66 66 66 F F F
38602024-04-17T17:12:56.865Z 23 460-479 34 34 34 66 66 66 F F F
38612024-04-17T17:12:56.865Z 24 480-499 34 34 34 65 65 65 F F F
38622024-04-17T17:12:56.865Z 25 500-519 26 26 26 48 48 48 F F F
38632024-04-17T17:12:56.865Z 26 520-539 34 34 34 65 65 65 F F F
38642024-04-17T17:12:56.865Z 27 540-559 34 34 34 66 66 66 F F F
38652024-04-17T17:12:56.865Z 28 560-579 32 32 32 62 62 62 F F F
38662024-04-17T17:12:56.865Z 29 580-599 34 34 34 67 67 67 F F F
38672024-04-17T17:12:56.865ZMax gen: 35, Max flush: 68
38682024-04-17T17:12:56.865Z
38692024-04-17T17:12:56.865ZBegin loop 18
38702024-04-17T17:12:56.865ZDownstairs to restart: 2
38712024-04-17T17:12:56.883Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.48963407Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312}
38722024-04-17T17:12:56.889Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.489912027Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312}
38732024-04-17T17:12:56.889ZCrucible runtime is spawned
38742024-04-17T17:12:56.889Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.489978431Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312}
38752024-04-17T17:12:56.889Z{"msg":"Crucible 50c49aa7-d3d2-4320-85ea-af3464dbac39 has session id: b1363b27-c3bd-4030-aa88-726733c3d6df","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.490009883Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
38762024-04-17T17:12:56.890Z{"msg":"Upstairs opts: Upstairs UUID: 50c49aa7-d3d2-4320-85ea-af3464dbac39, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.490072297Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
38772024-04-17T17:12:56.890Z{"msg":"Crucible stats registered with UUID: 50c49aa7-d3d2-4320-85ea-af3464dbac39","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.49011911Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
38782024-04-17T17:12:56.890Z{"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.490184674Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312}
38792024-04-17T17:12:56.890Z{"msg":"50c49aa7-d3d2-4320-85ea-af3464dbac39 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.490226166Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
38802024-04-17T17:12:56.890Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.490256948Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"client":"0","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
38812024-04-17T17:12:56.890Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.490305961Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"client":"0","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
38822024-04-17T17:12:56.890Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.490352144Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"client":"1","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
38832024-04-17T17:12:56.890Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.490385426Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"client":"1","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
38842024-04-17T17:12:56.890Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.490416988Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"client":"2","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
38852024-04-17T17:12:56.890Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.49045904Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"client":"2","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
38862024-04-17T17:12:56.890Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.490490062Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
38872024-04-17T17:12:56.890Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.490544486Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
38882024-04-17T17:12:56.890Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.490585038Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
38892024-04-17T17:12:56.890Z{"msg":"ds_connection connected from Ok(127.0.0.1:56698)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.49061503Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
38902024-04-17T17:12:56.890Z{"msg":"ds_connection connected from Ok(127.0.0.1:35025)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.490651782Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
38912024-04-17T17:12:56.890Z{"msg":"ds_connection connected from Ok(127.0.0.1:61101)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.490667613Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
38922024-04-17T17:12:56.890Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.491164813Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"client":"1","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
38932024-04-17T17:12:56.890Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.491202956Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"client":"1","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
38942024-04-17T17:12:56.890Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.491227287Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"client":"0","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
38952024-04-17T17:12:56.890Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.491243508Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"client":"0","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
38962024-04-17T17:12:56.890Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.49128022Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"client":"2","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
38972024-04-17T17:12:56.890Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.491320393Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"client":"2","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
38982024-04-17T17:12:56.891Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.491739148Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"client":"1","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
38992024-04-17T17:12:56.891Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.491793832Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"client":"0","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39002024-04-17T17:12:56.891Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.491835984Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"client":"2","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39012024-04-17T17:12:56.891Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.49226866Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"client":"1","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39022024-04-17T17:12:56.891Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.492325184Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"client":"1","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39032024-04-17T17:12:56.891Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.492344385Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39042024-04-17T17:12:56.891Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.492357856Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"client":"2","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39052024-04-17T17:12:56.891Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.492371776Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"client":"2","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39062024-04-17T17:12:56.891Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.492385277Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39072024-04-17T17:12:56.891Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.492409029Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"client":"0","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39082024-04-17T17:12:56.891Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.49243206Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"client":"0","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39092024-04-17T17:12:56.891Z{"msg":"[0]R flush_numbers[0..12]: [65, 68, 64, 66, 65, 65, 65, 68, 68, 60, 58, 64]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.492446281Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"":"downstairs","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39102024-04-17T17:12:56.891Z{"msg":"[0]R generation[0..12]: [34, 35, 33, 34, 34, 34, 34, 35, 35, 30, 30, 33]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.492460122Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"":"downstairs","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39112024-04-17T17:12:56.891Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.492480983Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"":"downstairs","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39122024-04-17T17:12:56.891Z{"msg":"[1]R flush_numbers[0..12]: [65, 68, 64, 66, 65, 65, 65, 68, 68, 60, 58, 64]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.492494984Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"":"downstairs","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39132024-04-17T17:12:56.891Z{"msg":"[1]R generation[0..12]: [34, 35, 33, 34, 34, 34, 34, 35, 35, 30, 30, 33]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.492515005Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"":"downstairs","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39142024-04-17T17:12:56.891Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.492529176Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"":"downstairs","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39152024-04-17T17:12:56.891Z{"msg":"[2]R flush_numbers[0..12]: [65, 68, 64, 66, 65, 65, 65, 68, 68, 60, 58, 64]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.492549357Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"":"downstairs","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39162024-04-17T17:12:56.891Z{"msg":"[2]R generation[0..12]: [34, 35, 33, 34, 34, 34, 34, 35, 35, 30, 30, 33]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.492562988Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"":"downstairs","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39172024-04-17T17:12:56.891ZWait for a query_work_queue command to finish before sending IO
39182024-04-17T17:12:56.891Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.492585669Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"":"downstairs","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39192024-04-17T17:12:56.891Z{"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.49259925Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"":"downstairs","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39202024-04-17T17:12:56.891Z{"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.492619231Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"":"downstairs","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39212024-04-17T17:12:56.892Z{"msg":"Next flush: 69","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.492632072Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"":"downstairs","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39222024-04-17T17:12:56.892Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.492645253Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"":"downstairs","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39232024-04-17T17:12:56.892Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.492665244Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39242024-04-17T17:12:56.892Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.492677365Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"client":"0","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39252024-04-17T17:12:56.892Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.492690336Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"client":"1","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39262024-04-17T17:12:56.892Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.492710357Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"client":"2","":"downstairs","session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39272024-04-17T17:12:56.892Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.492723908Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39282024-04-17T17:12:56.892Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.492740119Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39292024-04-17T17:12:56.892Z{"msg":"50c49aa7-d3d2-4320-85ea-af3464dbac39 is now active with session: b1363b27-c3bd-4030-aa88-726733c3d6df","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.49275669Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39302024-04-17T17:12:56.892Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.492769151Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312,"session_id":"b1363b27-c3bd-4030-aa88-726733c3d6df"}
39312024-04-17T17:12:56.892Z{"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:12:55.492780881Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312}
39322024-04-17T17:12:56.892ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
39332024-04-17T17:12:56.892ZLoading write count information from file "/tmp/test_repair_verify.data"
39342024-04-17T17:12:56.892ZRun Repair workload
39352024-04-17T17:12:56.896Z01/30 Write block 345 len 40960 data: 8 8 7 5 5 6 5 5 5 5
39362024-04-17T17:12:56.896Z02/30 Write block 373 len 12288 data: 4 3 3
39372024-04-17T17:12:56.896Z03/30 Read block 447 len 16384
39382024-04-17T17:12:56.896Z04/30 Flush
39392024-04-17T17:12:56.896Z05/30 Write block 21 len 24576 data: 3 4 4 4 6 6
39402024-04-17T17:12:56.896Z06/30 Write block 514 len 40960 data: 5 4 2 3 4 3 3 3 4 5
39412024-04-17T17:12:56.897Z07/30 Write block 120 len 8192 data: 4 3
39422024-04-17T17:12:56.897Z08/30 Write block 89 len 4096 data: 5
39432024-04-17T17:12:56.900Z09/30 Write block 384 len 24576 data: 4 3 3 3 3 3
39442024-04-17T17:12:56.900Z10/30 Write block 111 len 8192 data: 6 6
39452024-04-17T17:12:56.900Z11/30 Read block 214 len 24576
39462024-04-17T17:12:56.900Z12/30 Write block 463 len 40960 data: 4 5 4 4 3 3 4 4 4 4
39472024-04-17T17:12:56.900Z13/30 Read block 189 len 32768
39482024-04-17T17:12:56.903Z14/30 Flush
39492024-04-17T17:12:56.903Z15/30 Write block 271 len 8192 data: 3 3
39502024-04-17T17:12:56.907Z16/30 Read block 580 len 24576
39512024-04-17T17:12:56.907Z17/30 Write block 178 len 28672 data: 4 4 4 3 3 3 2
39522024-04-17T17:12:56.907Z18/30 Read block 423 len 32768
39532024-04-17T17:12:56.907Z19/30 Read block 328 len 20480
39542024-04-17T17:12:56.910Z20/30 Write block 130 len 20480 data: 4 5 7 5 6
39552024-04-17T17:12:56.910Z21/30 Write block 21 len 36864 data: 4 5 5 5 7 7 4 3 5
39562024-04-17T17:12:56.914Z22/30 Write block 141 len 36864 data: 3 3 4 3 3 3 2 2 3
39572024-04-17T17:12:56.914Z23/30 Read block 522 len 40960
39582024-04-17T17:12:56.914Z24/30 Read block 55 len 12288
39592024-04-17T17:12:56.914Z25/30 Read block 17 len 20480
39602024-04-17T17:12:56.914Z26/30 Read block 489 len 28672
39612024-04-17T17:12:56.918Z27/30 Read block 54 len 16384
39622024-04-17T17:12:56.918Z28/30 Read block 255 len 4096
39632024-04-17T17:12:56.918Z29/30 Write block 294 len 8192 data: 2 2
39642024-04-17T17:12:56.918Z30/30 Write block 515 len 8192 data: 5 3
39652024-04-17T17:12:56.918Z----------------------------------------------------------------
39662024-04-17T17:12:56.918Z Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:26
39672024-04-17T17:12:56.918ZStates: Active Active Active
39682024-04-17T17:12:56.918ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
39692024-04-17T17:12:56.918Z 5 Acked 1004 Write 6 Done Done Done false
39702024-04-17T17:12:56.918Z 6 Acked 1005 Write 10 Done Done Done false
39712024-04-17T17:12:56.918Z 7 Acked 1006 Write 2 Done Done Sent false
39722024-04-17T17:12:56.918Z 8 Acked 1007 Write 1 Done Done Sent false
39732024-04-17T17:12:56.918Z 9 Acked 1008 Write 6 Done Done Sent false
39742024-04-17T17:12:56.919Z 10 Acked 1009 Write 2 Done Done Sent false
39752024-04-17T17:12:56.919Z 11 Acked 1010 Read 6 Done Done Sent false
39762024-04-17T17:12:56.919Z 12 Acked 1011 Write 10 Done Done Sent false
39772024-04-17T17:12:56.919Z 13 Acked 1012 Read 8 Done Done Sent false
39782024-04-17T17:12:56.919Z 14 Acked 1013 Flush 0 Done Done Sent false
39792024-04-17T17:12:56.919Z 15 Acked 1014 Write 2 Done Done Sent false
39802024-04-17T17:12:56.919Z 16 Acked 1015 Read 6 Done Done Sent false
39812024-04-17T17:12:56.919Z 17 Acked 1016 Write 7 Done Done Sent false
39822024-04-17T17:12:56.919Z 18 Acked 1017 Read 8 Done Done Sent false
39832024-04-17T17:12:56.919Z 19 Acked 1018 Read 5 Done Done Sent false
39842024-04-17T17:12:56.919Z 20 Acked 1019 Write 5 Done Done Sent false
39852024-04-17T17:12:56.919Z 21 Acked 1020 Write 9 Done Done Sent false
39862024-04-17T17:12:56.919Z 22 Acked 1021 Write 9 Done Done Sent false
39872024-04-17T17:12:56.919Z 23 Acked 1022 Read 10 Done Done Sent false
39882024-04-17T17:12:56.919Z 24 Acked 1023 Read 3 Done Done Sent false
39892024-04-17T17:12:56.919Z 25 Acked 1024 Read 5 Done Done Sent false
39902024-04-17T17:12:56.919Z 26 Acked 1025 Read 7 Done Done Sent false
39912024-04-17T17:12:56.919Z 27 Acked 1026 Read 4 Done Done Sent false
39922024-04-17T17:12:56.919Z 28 Acked 1027 Read 1 Done Done Sent false
39932024-04-17T17:12:56.919Z 29 Acked 1028 Write 2 Done Done Sent false
39942024-04-17T17:12:56.919Z 30 Acked 1029 Write 2 Sent Sent Sent false
39952024-04-17T17:12:56.919Z STATES DS:0 DS:1 DS:2 TOTAL
39962024-04-17T17:12:56.919Z New 0 0 0 0
39972024-04-17T17:12:56.919Z Sent 1 1 24 26
39982024-04-17T17:12:56.920Z Done 25 25 2 52
39992024-04-17T17:12:56.920Z Skipped 0 0 0 0
40002024-04-17T17:12:56.920Z Error 0 0 0 0
40012024-04-17T17:12:56.920ZLast Flush: 1013 1013 1003
40022024-04-17T17:12:56.920ZDownstairs last five completed: 1003 1002 1001 1000
40032024-04-17T17:12:56.920ZUpstairs last five completed: 30 29 28 27 26
40042024-04-17T17:12:56.920Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:12:55.524490943Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1312}
40052024-04-17T17:12:56.920ZWrote out file "/tmp/test_repair_verify.data"
40062024-04-17T17:12:56.923Z
40072024-04-17T17:12:56.927ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
40082024-04-17T17:12:56.943ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
40092024-04-17T17:12:56.943Z 0 000-019 34 34 34 65 65 65 F F F
40102024-04-17T17:12:56.947Z 1 020-039 36 36 35 70 70 68 T T T <---
40112024-04-17T17:12:56.947Z 2 040-059 33 33 33 64 64 64 F F F
40122024-04-17T17:12:56.947Z 3 060-079 34 34 34 66 66 66 F F F
40132024-04-17T17:12:56.947Z 4 080-099 36 36 34 70 70 65 F F F <---
40142024-04-17T17:12:56.947Z 5 100-119 36 36 34 70 70 65 F F F <---
40152024-04-17T17:12:56.947Z 6 120-139 36 36 34 70 70 65 T T F <---
40162024-04-17T17:12:56.947Z 7 140-159 35 35 35 68 68 68 T T F <---
40172024-04-17T17:12:56.947Z 8 160-179 35 35 35 68 68 68 T T F <---
40182024-04-17T17:12:56.947Z 9 180-199 30 30 30 60 60 60 T T F <---
40192024-04-17T17:12:56.947Z 10 200-219 30 30 30 58 58 58 F F F
40202024-04-17T17:12:56.947Z 11 220-239 33 33 33 64 64 64 F F F
40212024-04-17T17:12:56.947Z 12 240-259 30 30 30 60 60 60 F F F
40222024-04-17T17:12:56.947Z 13 260-279 26 26 26 47 47 47 T T F <---
40232024-04-17T17:12:56.947Z 14 280-299 30 30 30 60 60 60 T T F <---
40242024-04-17T17:12:56.947Z 15 300-319 33 33 33 64 64 64 F F F
40252024-04-17T17:12:56.947Z 16 320-339 33 33 33 64 64 64 F F F
40262024-04-17T17:12:56.947Z 17 340-359 36 36 36 69 69 69 F F F
40272024-04-17T17:12:56.947Z 18 360-379 36 36 36 69 69 69 F F F
40282024-04-17T17:12:56.947Z 19 380-399 36 36 35 70 70 68 F F F <---
40292024-04-17T17:12:56.948Z 20 400-419 33 33 33 64 64 64 F F F
40302024-04-17T17:12:56.948Z 21 420-439 35 35 35 68 68 68 F F F
40312024-04-17T17:12:56.948Z 22 440-459 34 34 34 66 66 66 F F F
40322024-04-17T17:12:56.948Z 23 460-479 36 36 34 70 70 66 F F F <---
40332024-04-17T17:12:56.948Z 24 480-499 34 34 34 65 65 65 F F F
40342024-04-17T17:12:56.948Z 25 500-519 36 36 26 70 70 48 T T T <---
40352024-04-17T17:12:56.948Z 26 520-539 36 36 34 70 70 65 F F T <---
40362024-04-17T17:12:56.948Z 27 540-559 34 34 34 66 66 66 F F F
40372024-04-17T17:12:56.948Z 28 560-579 32 32 32 62 62 62 F F F
40382024-04-17T17:12:56.948Z 29 580-599 34 34 34 67 67 67 F F F
40392024-04-17T17:12:56.948ZMax gen: 36, Max flush: 70
40402024-04-17T17:12:56.948ZError: Difference in extent metadata found!
40412024-04-17T17:12:56.948ZA Difference in extent metadata is expected here
40422024-04-17T17:12:56.948ZOn loop 18
40432024-04-17T17:12:56.948Z
40442024-04-17T17:12:56.952ZVerifying data now
40452024-04-17T17:13:11.567ZNo change after verify
40462024-04-17T17:13:11.567ZLoop: 18 Downstairs dump after verify (and repair):
40472024-04-17T17:13:11.584ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
40482024-04-17T17:13:11.584Z 0 000-019 34 34 34 65 65 65 F F F
40492024-04-17T17:13:11.584Z 1 020-039 37 37 37 71 71 71 F F F
40502024-04-17T17:13:11.584Z 2 040-059 33 33 33 64 64 64 F F F
40512024-04-17T17:13:11.585Z 3 060-079 34 34 34 66 66 66 F F F
40522024-04-17T17:13:11.585Z 4 080-099 36 36 36 70 70 70 F F F
40532024-04-17T17:13:11.598Z 5 100-119 36 36 36 70 70 70 F F F
40542024-04-17T17:13:11.598Z 6 120-139 37 37 37 71 71 71 F F F
40552024-04-17T17:13:11.598Z 7 140-159 37 37 37 71 71 71 F F F
40562024-04-17T17:13:11.598Z 8 160-179 37 37 37 71 71 71 F F F
40572024-04-17T17:13:11.598Z 9 180-199 37 37 37 71 71 71 F F F
40582024-04-17T17:13:11.598Z 10 200-219 30 30 30 58 58 58 F F F
40592024-04-17T17:13:11.599Z 11 220-239 33 33 33 64 64 64 F F F
40602024-04-17T17:13:11.599Z 12 240-259 30 30 30 60 60 60 F F F
40612024-04-17T17:13:11.599Z 13 260-279 37 37 37 71 71 71 F F F
40622024-04-17T17:13:11.599Z 14 280-299 37 37 37 71 71 71 F F F
40632024-04-17T17:13:11.599Z 15 300-319 33 33 33 64 64 64 F F F
40642024-04-17T17:13:11.599Z 16 320-339 33 33 33 64 64 64 F F F
40652024-04-17T17:13:11.599Z 17 340-359 36 36 36 69 69 69 F F F
40662024-04-17T17:13:11.600Z 18 360-379 36 36 36 69 69 69 F F F
40672024-04-17T17:13:11.600Z 19 380-399 36 36 36 70 70 70 F F F
40682024-04-17T17:13:11.600Z 20 400-419 33 33 33 64 64 64 F F F
40692024-04-17T17:13:11.600Z 21 420-439 35 35 35 68 68 68 F F F
40702024-04-17T17:13:11.600Z 22 440-459 34 34 34 66 66 66 F F F
40712024-04-17T17:13:11.600Z 23 460-479 36 36 36 70 70 70 F F F
40722024-04-17T17:13:11.600Z 24 480-499 34 34 34 65 65 65 F F F
40732024-04-17T17:13:11.600Z 25 500-519 37 37 37 71 71 71 F F F
40742024-04-17T17:13:11.601Z 26 520-539 36 36 36 70 70 70 F F F
40752024-04-17T17:13:11.601Z 27 540-559 34 34 34 66 66 66 F F F
40762024-04-17T17:13:11.601Z 28 560-579 32 32 32 62 62 62 F F F
40772024-04-17T17:13:11.601Z 29 580-599 34 34 34 67 67 67 F F F
40782024-04-17T17:13:11.601ZMax gen: 37, Max flush: 71
40792024-04-17T17:13:11.601Z
40802024-04-17T17:13:11.601ZBegin loop 19
40812024-04-17T17:13:11.601ZDownstairs to restart: 0
40822024-04-17T17:13:11.635Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.20970322Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320}
40832024-04-17T17:13:11.635Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.21001765Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320}
40842024-04-17T17:13:11.636ZCrucible runtime is spawned
40852024-04-17T17:13:11.636Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.210085724Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320}
40862024-04-17T17:13:11.636Z{"msg":"Crucible b98f252e-3b0d-42e9-89f9-52a1b9105874 has session id: 8a410f77-523c-4c62-9941-bd683c8a2c8f","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.210128906Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
40872024-04-17T17:13:11.636Z{"msg":"Upstairs opts: Upstairs UUID: b98f252e-3b0d-42e9-89f9-52a1b9105874, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.21018076Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
40882024-04-17T17:13:11.636Z{"msg":"Crucible stats registered with UUID: b98f252e-3b0d-42e9-89f9-52a1b9105874","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.210216662Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
40892024-04-17T17:13:11.636Z{"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.210262534Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320}
40902024-04-17T17:13:11.637Z{"msg":"b98f252e-3b0d-42e9-89f9-52a1b9105874 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.210287896Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
40912024-04-17T17:13:11.637Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.210325138Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"client":"0","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
40922024-04-17T17:13:11.637Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.210367531Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"client":"0","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
40932024-04-17T17:13:11.637Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.210436775Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"client":"1","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
40942024-04-17T17:13:11.638Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.210465067Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"client":"1","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
40952024-04-17T17:13:11.638Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.210507569Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"client":"2","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
40962024-04-17T17:13:11.638Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.210539111Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"client":"2","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
40972024-04-17T17:13:11.638Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.210580364Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
40982024-04-17T17:13:11.638Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.210622446Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
40992024-04-17T17:13:11.638Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.210652198Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41002024-04-17T17:13:11.639Z{"msg":"ds_connection connected from Ok(127.0.0.1:38712)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.21068809Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41012024-04-17T17:13:11.639Z{"msg":"ds_connection connected from Ok(127.0.0.1:41506)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.210713592Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41022024-04-17T17:13:11.639Z{"msg":"ds_connection connected from Ok(127.0.0.1:52478)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.210750604Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41032024-04-17T17:13:11.639Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.211415164Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"client":"1","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41042024-04-17T17:13:11.639Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.211451677Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"client":"1","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41052024-04-17T17:13:11.639Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.211468187Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"client":"0","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41062024-04-17T17:13:11.639Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.211518751Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"client":"0","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41072024-04-17T17:13:11.640Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.211551352Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"client":"2","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41082024-04-17T17:13:11.640Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.211573874Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"client":"2","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41092024-04-17T17:13:11.640Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.212102336Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"client":"0","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41102024-04-17T17:13:11.640Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.212148419Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"client":"1","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41112024-04-17T17:13:11.640Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.21217217Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"client":"2","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41122024-04-17T17:13:11.640Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.212674881Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"client":"2","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41132024-04-17T17:13:11.640Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.212718073Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"client":"2","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41142024-04-17T17:13:11.641Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.212757566Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41152024-04-17T17:13:11.641Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.212780787Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"client":"0","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41162024-04-17T17:13:11.641Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.212809259Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"client":"0","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41172024-04-17T17:13:11.641Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.21283259Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41182024-04-17T17:13:11.641Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.212862312Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"client":"1","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41192024-04-17T17:13:11.641Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.212884643Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"client":"1","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41202024-04-17T17:13:11.641Z{"msg":"[0]R flush_numbers[0..12]: [65, 71, 64, 66, 70, 70, 71, 71, 71, 71, 58, 64]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
41212024-04-17T17:13:11.641Z,"time":"2024-04-17T17:13:10.212907895Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"":"downstairs","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41222024-04-17T17:13:11.642Z{"msg":"[0]R generation[0..12]: [34, 37, 33, 34, 36, 36, 37, 37, 37, 37, 30, 33]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.212932096Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"":"downstairs","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41232024-04-17T17:13:11.642Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.212965868Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"":"downstairs","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41242024-04-17T17:13:11.642Z{"msg":"[1]R flush_numbers[0..12]: [65, 71, 64, 66, 70, 70, 71, 71, 71, 71, 58, 64]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.21299089Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"":"downstairs","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41252024-04-17T17:13:11.642Z{"msg":"[1]R generation[0..12]: [34, 37, 33, 34, 36, 36, 37, 37, 37, 37, 30, 33]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.213023162Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"":"downstairs","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41262024-04-17T17:13:11.642Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.213063124Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"":"downstairs","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41272024-04-17T17:13:11.642Z{"msg":"[2]R flush_numbers[0..12]: [65, 71, 64, 66, 70, 70, 71, 71, 71, 71, 58, 64]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.213089946Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"":"downstairs","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41282024-04-17T17:13:11.642Z{"msg":"[2]R generation[0..12]: [34, 37, 33, 34, 36, 36, 37, 37, 37, 37, 30, 33]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.213123848Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"":"downstairs","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41292024-04-17T17:13:11.643Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.2131546Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"":"downstairs","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41302024-04-17T17:13:11.643Z{"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.213180841Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"":"downstairs","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41312024-04-17T17:13:11.643Z{"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.213205733Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"":"downstairs","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41322024-04-17T17:13:11.643Z{"msg":"Next flush: 72","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.213221164Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"":"downstairs","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41332024-04-17T17:13:11.643Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.213244245Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"":"downstairs","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41342024-04-17T17:13:11.643Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.213257816Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41352024-04-17T17:13:11.644Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.213278427Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"client":"0","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41362024-04-17T17:13:11.644Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.213297929Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"client":"1","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41372024-04-17T17:13:11.644Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.21332029Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"client":"2","":"downstairs","session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41382024-04-17T17:13:11.644Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.213334261Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41392024-04-17T17:13:11.644Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.213357572Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41402024-04-17T17:13:11.644Z{"msg":"b98f252e-3b0d-42e9-89f9-52a1b9105874 is now active with session: 8a410f77-523c-4c62-9941-bd683c8a2c8f","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.213370933Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41412024-04-17T17:13:11.645Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.213389454Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320,"session_id":"8a410f77-523c-4c62-9941-bd683c8a2c8f"}
41422024-04-17T17:13:11.645Z{"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:10.213401715Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320}
41432024-04-17T17:13:11.645ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
41442024-04-17T17:13:12.075ZLoading write count information from file "/tmp/test_repair_verify.data"
41452024-04-17T17:13:12.076ZRun Repair workload
41462024-04-17T17:13:12.076Z01/30 Write block 364 len 8192 data: 4 3
41472024-04-17T17:13:12.076Z02/30 Read block 524 len 12288
41482024-04-17T17:13:12.076Z03/30 Read block 94 len 28672
41492024-04-17T17:13:12.076Z04/30 Read block 122 len 4096
41502024-04-17T17:13:12.076Z05/30 Read block 172 len 16384
41512024-04-17T17:13:12.076Z06/30 Read block 299 len 28672
41522024-04-17T17:13:12.076Z07/30 Read block 103 len 12288
41532024-04-17T17:13:12.077Z08/30 Read block 503 len 8192
41542024-04-17T17:13:12.077Z09/30 Write block 192 len 4096 data: 3
41552024-04-17T17:13:12.077Z10/30 Write block 421 len 40960 data: 3 4 3 3 3 3 3 4 5 3
41562024-04-17T17:13:12.077Z11/30 Read block 120 len 40960
41572024-04-17T17:13:12.077Z12/30 Write block 385 len 12288 data: 4 4 4
41582024-04-17T17:13:12.077Z13/30 Write block 542 len 12288 data: 5 5 4
41592024-04-17T17:13:12.078Z14/30 Write block 526 len 24576 data: 3 3 4 4 4 3
41602024-04-17T17:13:12.078Z15/30 Read block 574 len 12288
41612024-04-17T17:13:12.078Z16/30 Write block 507 len 8192 data: 3 3
41622024-04-17T17:13:12.078Z17/30 Read block 101 len 20480
41632024-04-17T17:13:12.078Z18/30 Read block 49 len 4096
41642024-04-17T17:13:12.078Z19/30 Flush
41652024-04-17T17:13:12.078Z20/30 Write block 305 len 16384 data: 5 4 3 3
41662024-04-17T17:13:12.079Z21/30 Write block 166 len 8192 data: 3 4
41672024-04-17T17:13:12.079Z22/30 Write block 458 len 40960 data: 6 5 5 4 4 5 6 5 5 4
41682024-04-17T17:13:12.079Z23/30 Read block 56 len 20480
41692024-04-17T17:13:12.079Z24/30 Flush
41702024-04-17T17:13:12.079Z25/30 Write block 267 len 40960 data: 2 2 3 3 4 4 5 5 6 6
41712024-04-17T17:13:12.079Z26/30 Read block 307 len 16384
41722024-04-17T17:13:12.080Z27/30 Read block 304 len 40960
41732024-04-17T17:13:12.080Z28/30 Write block 394 len 40960 data: 4 2 2 2 3 3 3 3 3 4
41742024-04-17T17:13:12.080Z29/30 Read block 384 len 36864
41752024-04-17T17:13:12.080Z30/30 Write block 360 len 16384 data: 5 5 4 3
41762024-04-17T17:13:12.080Z----------------------------------------------------------------
41772024-04-17T17:13:12.080Z Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:30
41782024-04-17T17:13:12.080ZStates: Active Active Active
41792024-04-17T17:13:12.081ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
41802024-04-17T17:13:12.081Z 1 Acked 1000 Write 2 Done Done Done false
41812024-04-17T17:13:12.081Z 2 Acked 1001 Read 3 Sent Done Done false
41822024-04-17T17:13:12.081Z 3 Acked 1002 Read 7 Sent Done Done false
41832024-04-17T17:13:12.081Z 4 Acked 1003 Read 1 Sent Done Done false
41842024-04-17T17:13:12.081Z 5 Acked 1004 Read 4 Sent Done Done false
41852024-04-17T17:13:12.081Z 6 Acked 1005 Read 7 Sent Done Done false
41862024-04-17T17:13:12.081Z 7 Acked 1006 Read 3 Sent Done Done false
41872024-04-17T17:13:12.082Z 8 Acked 1007 Read 2 Sent Done Done false
41882024-04-17T17:13:12.082Z 9 Acked 1008 Write 1 Sent Done Done false
41892024-04-17T17:13:12.082Z 10 Acked 1009 Write 10 Sent Done Done false
41902024-04-17T17:13:12.082Z 11 Acked 1010 Read 10 Sent Done Done false
41912024-04-17T17:13:12.082Z 12 Acked 1011 Write 3 Sent Done Done false
41922024-04-17T17:13:12.083Z 13 Acked 1012 Write 3 Sent Done Done false
41932024-04-17T17:13:12.083Z 14 Acked 1013 Write 6 Sent Done Done false
41942024-04-17T17:13:12.083Z 15 Acked 1014 Read 3 Sent Done Done false
41952024-04-17T17:13:12.083Z 16 Acked 1015 Write 2 Sent Done Done false
41962024-04-17T17:13:12.083Z 17 Acked 1016 Read 5 Sent Done Done false
41972024-04-17T17:13:12.083Z 18 Acked 1017 Read 1 Sent Done Done false
41982024-04-17T17:13:12.083Z 19 Acked 1018 Flush 0 Sent Done Done false
41992024-04-17T17:13:12.083Z 20 Acked 1019 Write 4 Sent Done Done false
42002024-04-17T17:13:12.083Z 21 Acked 1020 Write 2 Sent Done Done false
42012024-04-17T17:13:12.083Z 22 Acked 1021 Write 10 Sent Done Done false
42022024-04-17T17:13:12.084Z 23 Acked 1022 Read 5 Sent Done Done false
42032024-04-17T17:13:12.084Z 24 Acked 1023 Flush 0 Sent Done Done false
42042024-04-17T17:13:12.084Z 25 Acked 1024 Write 10 Sent Done Done false
42052024-04-17T17:13:12.084Z 26 Acked 1025 Read 4 Sent Done Done false
42062024-04-17T17:13:12.084Z 27 Acked 1026 Read 10 Sent Done Done false
42072024-04-17T17:13:12.084Z 28 Acked 1027 Write 10 Sent Done Done false
42082024-04-17T17:13:12.084Z 29 Acked 1028 Read 9 Sent Done Done false
42092024-04-17T17:13:12.085Z 30 Acked 1029 Write 4 Sent Sent Sent false
42102024-04-17T17:13:12.085Z STATES DS:0 DS:1 DS:2 TOTAL
42112024-04-17T17:13:12.085Z New 0 0 0 0
42122024-04-17T17:13:12.085Z Sent 29 1 1 31
42132024-04-17T17:13:12.085Z Done 1 29 29 59
42142024-04-17T17:13:12.085Z Skipped 0 0 0 0
42152024-04-17T17:13:12.085Z Error 0 0 0 0
42162024-04-17T17:13:12.085ZLast Flush: 0 1023 1023
42172024-04-17T17:13:12.086ZDownstairs last five completed:
42182024-04-17T17:13:12.086ZUpstairs last five completed: 30 29 28 27 26
42192024-04-17T17:13:12.086Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:13:10.245628538Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1320}
42202024-04-17T17:13:12.086ZWrote out file "/tmp/test_repair_verify.data"
42212024-04-17T17:13:12.086Z
42222024-04-17T17:13:12.086ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
42232024-04-17T17:13:12.109ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
42242024-04-17T17:13:12.109Z 0 000-019 34 34 34 65 65 65 F F F
42252024-04-17T17:13:12.109Z 1 020-039 37 37 37 71 71 71 F F F
42262024-04-17T17:13:12.110Z 2 040-059 33 33 33 64 64 64 F F F
42272024-04-17T17:13:12.110Z 3 060-079 34 34 34 66 66 66 F F F
42282024-04-17T17:13:12.110Z 4 080-099 36 36 36 70 70 70 F F F
42292024-04-17T17:13:12.110Z 5 100-119 36 36 36 70 70 70 F F F
42302024-04-17T17:13:12.110Z 6 120-139 37 37 37 71 71 71 F F F
42312024-04-17T17:13:12.110Z 7 140-159 37 37 37 71 71 71 F F F
42322024-04-17T17:13:12.110Z 8 160-179 37 38 38 71 73 73 F F F <---
42332024-04-17T17:13:12.111Z 9 180-199 37 38 38 71 72 72 F F F <---
42342024-04-17T17:13:12.111Z 10 200-219 30 30 30 58 58 58 F F F
42352024-04-17T17:13:12.111Z 11 220-239 33 33 33 64 64 64 F F F
42362024-04-17T17:13:12.111Z 12 240-259 30 30 30 60 60 60 F F F
42372024-04-17T17:13:12.111Z 13 260-279 37 37 37 71 71 71 F T T <---
42382024-04-17T17:13:12.112Z 14 280-299 37 37 37 71 71 71 F F F
42392024-04-17T17:13:12.112Z 15 300-319 33 38 38 64 73 73 F F F <---
42402024-04-17T17:13:12.112Z 16 320-339 33 33 33 64 64 64 F F F
42412024-04-17T17:13:12.112Z 17 340-359 36 36 36 69 69 69 F F F
42422024-04-17T17:13:12.112Z 18 360-379 36 38 38 69 72 72 T T T <---
42432024-04-17T17:13:12.112Z 19 380-399 36 38 38 70 72 72 F T T <---
42442024-04-17T17:13:12.113Z 20 400-419 33 33 33 64 64 64 F T T <---
42452024-04-17T17:13:12.113Z 21 420-439 35 38 38 68 72 72 F F F <---
42462024-04-17T17:13:12.113Z 22 440-459 34 38 38 66 73 73 F F F <---
42472024-04-17T17:13:12.113Z 23 460-479 36 38 38 70 73 73 F F F <---
42482024-04-17T17:13:12.113Z 24 480-499 34 34 34 65 65 65 F F F
42492024-04-17T17:13:12.113Z 25 500-519 37 38 38 71 72 72 F F F <---
42502024-04-17T17:13:12.114Z 26 520-539 36 38 38 70 72 72 F F F <---
42512024-04-17T17:13:12.114Z 27 540-559 34 38 38 66 72 72 F F F <---
42522024-04-17T17:13:12.114Z 28 560-579 32 32 32 62 62 62 F F F
42532024-04-17T17:13:12.114Z 29 580-599 34 34 34 67 67 67 F F F
42542024-04-17T17:13:12.114ZMax gen: 38, Max flush: 73
42552024-04-17T17:13:12.114ZError: Difference in extent metadata found!
42562024-04-17T17:13:12.115ZA Difference in extent metadata is expected here
42572024-04-17T17:13:12.115ZOn loop 19
42582024-04-17T17:13:12.115Z
42592024-04-17T17:13:12.115ZVerifying data now
42602024-04-17T17:13:16.269ZNo change after verify
42612024-04-17T17:13:16.269ZLoop: 19 Downstairs dump after verify (and repair):
42622024-04-17T17:13:16.282ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
42632024-04-17T17:13:16.282Z 0 000-019 34 34 34 65 65 65 F F F
42642024-04-17T17:13:16.283Z 1 020-039 37 37 37 71 71 71 F F F
42652024-04-17T17:13:16.283Z 2 040-059 33 33 33 64 64 64 F F F
42662024-04-17T17:13:16.283Z 3 060-079 34 34 34 66 66 66 F F F
42672024-04-17T17:13:16.288Z 4 080-099 36 36 36 70 70 70 F F F
42682024-04-17T17:13:16.288Z 5 100-119 36 36 36 70 70 70 F F F
42692024-04-17T17:13:16.288Z 6 120-139 37 37 37 71 71 71 F F F
42702024-04-17T17:13:16.289Z 7 140-159 37 37 37 71 71 71 F F F
42712024-04-17T17:13:16.289Z 8 160-179 38 38 38 73 73 73 F F F
42722024-04-17T17:13:16.289Z 9 180-199 38 38 38 72 72 72 F F F
42732024-04-17T17:13:16.289Z 10 200-219 30 30 30 58 58 58 F F F
42742024-04-17T17:13:16.289Z 11 220-239 33 33 33 64 64 64 F F F
42752024-04-17T17:13:16.289Z 12 240-259 30 30 30 60 60 60 F F F
42762024-04-17T17:13:16.290Z 13 260-279 39 39 39 74 74 74 F F F
42772024-04-17T17:13:16.290Z 14 280-299 37 37 37 71 71 71 F F F
42782024-04-17T17:13:16.290Z 15 300-319 38 38 38 73 73 73 F F F
42792024-04-17T17:13:16.291Z 16 320-339 33 33 33 64 64 64 F F F
42802024-04-17T17:13:16.291Z 17 340-359 36 36 36 69 69 69 F F F
42812024-04-17T17:13:16.291Z 18 360-379 39 39 39 74 74 74 F F F
42822024-04-17T17:13:16.291Z 19 380-399 39 39 39 74 74 74 F F F
42832024-04-17T17:13:16.291Z 20 400-419 39 39 39 74 74 74 F F F
42842024-04-17T17:13:16.292Z 21 420-439 38 38 38 72 72 72 F F F
42852024-04-17T17:13:16.292Z 22 440-459 38 38 38 73 73 73 F F F
42862024-04-17T17:13:16.292Z 23 460-479 38 38 38 73 73 73 F F F
42872024-04-17T17:13:16.292Z 24 480-499 34 34 34 65 65 65 F F F
42882024-04-17T17:13:16.292Z 25 500-519 38 38 38 72 72 72 F F F
42892024-04-17T17:13:16.293Z 26 520-539 38 38 38 72 72 72 F F F
42902024-04-17T17:13:16.293Z 27 540-559 38 38 38 72 72 72 F F F
42912024-04-17T17:13:16.293Z 28 560-579 32 32 32 62 62 62 F F F
42922024-04-17T17:13:16.293Z 29 580-599 34 34 34 67 67 67 F F F
42932024-04-17T17:13:16.294ZMax gen: 39, Max flush: 74
42942024-04-17T17:13:16.294Z
42952024-04-17T17:13:16.294ZBegin loop 20
42962024-04-17T17:13:16.294ZDownstairs to restart: 1
42972024-04-17T17:13:16.307Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.911555996Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328}
42982024-04-17T17:13:16.307Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.911842173Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328}
42992024-04-17T17:13:16.337Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30Crucible runtime is spawned
43002024-04-17T17:13:16.337Z,"time":"2024-04-17T17:13:14.911886826Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328}
43012024-04-17T17:13:16.337Z{"msg":"Crucible 61644a88-6fee-4457-8c3a-3a6e8d01aea5 has session id: e18799bb-0bc7-45b6-ae80-8f61d4f58e0c","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.911969311Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43022024-04-17T17:13:16.337Z{"msg":"Upstairs opts: Upstairs UUID: 61644a88-6fee-4457-8c3a-3a6e8d01aea5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.912089908Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43032024-04-17T17:13:16.337Z{"msg":"Crucible stats registered with UUID: 61644a88-6fee-4457-8c3a-3a6e8d01aea5","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.912141781Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43042024-04-17T17:13:16.337Z{"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.912174853Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328}
43052024-04-17T17:13:16.338Z{"msg":"61644a88-6fee-4457-8c3a-3a6e8d01aea5 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.912198115Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43062024-04-17T17:13:16.338Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.912220166Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"client":"0","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43072024-04-17T17:13:16.338Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.912259628Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"client":"0","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43082024-04-17T17:13:16.338Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.912300531Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"client":"1","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43092024-04-17T17:13:16.338Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.912339873Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"client":"1","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43102024-04-17T17:13:16.338Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.912388466Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"client":"2","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43112024-04-17T17:13:16.338Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.912434349Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"client":"2","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43122024-04-17T17:13:16.338Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.912465271Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43132024-04-17T17:13:16.338Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.912509673Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43142024-04-17T17:13:16.339Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.912537035Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43152024-04-17T17:13:16.339Z{"msg":"ds_connection connected from Ok(127.0.0.1:43354)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.912563717Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43162024-04-17T17:13:16.339Z{"msg":"ds_connection connected from Ok(127.0.0.1:64533)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.91261089Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43172024-04-17T17:13:16.339Z{"msg":"ds_connection connected from Ok(127.0.0.1:55879)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.912714706Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43182024-04-17T17:13:16.339Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.913232957Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"client":"2","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43192024-04-17T17:13:16.339Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.91327934Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"client":"2","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43202024-04-17T17:13:16.339Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.913307382Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"client":"0","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43212024-04-17T17:13:16.340Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.913335123Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"client":"0","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43222024-04-17T17:13:16.340Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.913797331Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"client":"1","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43232024-04-17T17:13:16.340Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.913833624Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"client":"1","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43242024-04-17T17:13:16.340Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.913852845Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"client":"2","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43252024-04-17T17:13:16.340Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.913879607Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"client":"0","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43262024-04-17T17:13:16.340Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.914468942Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"client":"0","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43272024-04-17T17:13:16.341Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.914521606Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"client":"0","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43282024-04-17T17:13:16.341Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.914558058Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43292024-04-17T17:13:16.341Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.914573709Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"client":"1","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43302024-04-17T17:13:16.341Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.914587979Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"client":"2","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43312024-04-17T17:13:16.341Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.914616011Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"client":"2","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43322024-04-17T17:13:16.341Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.914657754Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43332024-04-17T17:13:16.341Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.915041977Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"client":"1","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43342024-04-17T17:13:16.341Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.915076759Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"client":"1","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43352024-04-17T17:13:16.342Z{"msg":"[0]R flush_numbers[0..12]: [65, 71, 64, 66, 70, 70, 71, 71, 73, 72, 58, 64]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.91509335Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"":"downstairs","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43362024-04-17T17:13:16.342Z{"msg":"[0]R generation[0..12]: [34, 37, 33, 34, 36, 36, 37, 37, 38, 38, 30, 33]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.915116011Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"":"downstairs","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43372024-04-17T17:13:16.342Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.915130892Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"":"downstairs","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43382024-04-17T17:13:16.342Z{"msg":"[1]R flush_numbers[0..12]: [65, 71, 64, 66, 70, 70, 71, 71, 73, 72, 58, 64]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.915156174Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"":"downstairs","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43392024-04-17T17:13:16.342Z{"msg":"[1]R generation[0..12]: [34, 37, 33, 34, 36, 36, 37, 37, 38, 38, 30, 33]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.915182795Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"":"downstairs","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43402024-04-17T17:13:16.342Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.915197236Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"":"downstairs","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43412024-04-17T17:13:16.342ZWait for a query_work_queue command to finish before sending IO
43422024-04-17T17:13:16.342Z{"msg":"[2]R flush_numbers[0..12]: [65, 71, 64, 66, 70, 70, 71, 71, 73, 72, 58, 64]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.915222878Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"":"downstairs","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43432024-04-17T17:13:16.343Z{"msg":"[2]R generation[0..12]: [34, 37, 33, 34, 36, 36, 37, 37, 38, 38, 30, 33]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.915237529Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"":"downstairs","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43442024-04-17T17:13:16.343Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.915273931Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"":"downstairs","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43452024-04-17T17:13:16.343Z{"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.915288192Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"":"downstairs","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43462024-04-17T17:13:16.343Z{"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.915305123Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"":"downstairs","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43472024-04-17T17:13:16.343Z{"msg":"Next flush: 75","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.915324924Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"":"downstairs","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43482024-04-17T17:13:16.343Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.915347225Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"":"downstairs","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43492024-04-17T17:13:16.343Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.915376977Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43502024-04-17T17:13:16.343Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.915389948Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"client":"0","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43512024-04-17T17:13:16.343Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.915414309Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"client":"1","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43522024-04-17T17:13:16.343Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.915447452Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"client":"2","":"downstairs","session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43532024-04-17T17:13:16.343Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.915478673Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43542024-04-17T17:13:16.343Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.915516946Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43552024-04-17T17:13:16.344Z{"msg":"61644a88-6fee-4457-8c3a-3a6e8d01aea5 is now active with session: e18799bb-0bc7-45b6-ae80-8f61d4f58e0c","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.915558368Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43562024-04-17T17:13:16.344Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.915597401Z","Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
43572024-04-17T17:13:16.344Zhostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328,"session_id":"e18799bb-0bc7-45b6-ae80-8f61d4f58e0c"}
43582024-04-17T17:13:16.344Z{"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:14.915636603Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328}
43592024-04-17T17:13:16.344ZLoading write count information from file "/tmp/test_repair_verify.data"
43602024-04-17T17:13:16.344ZRun Repair workload
43612024-04-17T17:13:16.344Z01/30 Write block 581 len 24576 data: 5 4 4 4 5 5
43622024-04-17T17:13:16.344Z02/30 Write block 41 len 24576 data: 6 5 4 4 5 4
43632024-04-17T17:13:16.353Z03/30 Write block 152 len 40960 data: 3 2 3 4 5 5 3 3 3 4
43642024-04-17T17:13:16.354Z04/30 Read block 543 len 32768
43652024-04-17T17:13:16.354Z05/30 Read block 52 len 40960
43662024-04-17T17:13:16.354Z06/30 Read block 223 len 36864
43672024-04-17T17:13:16.354Z07/30 Read block 560 len 40960
43682024-04-17T17:13:16.354Z08/30 Write block 200 len 24576 data: 4 5 4 5 5 5
43692024-04-17T17:13:16.355Z09/30 Read block 334 len 36864
43702024-04-17T17:13:16.355Z10/30 Read block 324 len 20480
43712024-04-17T17:13:16.355Z11/30 Read block 55 len 12288
43722024-04-17T17:13:16.355Z12/30 Write block 437 len 32768 data: 4 5 6 6 6 7 7 7
43732024-04-17T17:13:16.356Z13/30 Read block 419 len 36864
43742024-04-17T17:13:16.356Z14/30 Write block 527 len 40960 data: 4 5 5 5 4 3 3 2 2 3
43752024-04-17T17:13:16.356Z15/30 Read block 214 len 28672
43762024-04-17T17:13:16.356Z16/30 Flush
43772024-04-17T17:13:16.356Z17/30 Write block 403 len 20480 data: 5 4 4 4 5
43782024-04-17T17:13:16.356Z18/30 Read block 450 len 32768
43792024-04-17T17:13:16.357Z19/30 Read block 161 len 40960
43802024-04-17T17:13:16.357Z20/30 Write block 63 len 36864 data: 4 4 3 4 4 3 3 3 4
43812024-04-17T17:13:16.357Z21/30 Write block 483 len 32768 data: 6 6 5 6 4 5 5 5
43822024-04-17T17:13:16.357Z22/30 Read block 186 len 12288
43832024-04-17T17:13:16.357Z23/30 Write block 198 len 40960 data: 4 4 5 6 5 6 6 6 4 4
43842024-04-17T17:13:16.358Z24/30 Read block 512 len 16384
43852024-04-17T17:13:16.358Z25/30 Write block 379 len 4096 data: 4
43862024-04-17T17:13:16.358Z26/30 Write block 88 len 12288 data: 4 6 6
43872024-04-17T17:13:16.358Z27/30 Write block 314 len 20480 data: 5 4 4 4 4
43882024-04-17T17:13:16.358Z28/30 Write block 387 len 4096 data: 5
43892024-04-17T17:13:16.359Z29/30 Write block 587 len 16384 data: 4 2 2 3
43902024-04-17T17:13:16.359Z30/30 Write block 377 len 16384 data: 3 4 5 3
43912024-04-17T17:13:16.359Z----------------------------------------------------------------
43922024-04-17T17:13:16.359Z Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:30
43932024-04-17T17:13:16.359ZStates: Active Active Active
43942024-04-17T17:13:16.360ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
43952024-04-17T17:13:16.360Z 1 Acked 1000 Write 6 Done Done Done false
43962024-04-17T17:13:16.360Z 2 Acked 1001 Write 6 Done Done Done false
43972024-04-17T17:13:16.360Z 3 Acked 1002 Write 10 Done Done Done false
43982024-04-17T17:13:16.360Z 4 Acked 1003 Read 8 Done Done Done false
43992024-04-17T17:13:16.360Z 5 Acked 1004 Read 10 Done Sent Done false
44002024-04-17T17:13:16.361Z 6 Acked 1005 Read 9 Done Sent Done false
44012024-04-17T17:13:16.361Z 7 Acked 1006 Read 10 Done Sent Done false
44022024-04-17T17:13:16.361Z 8 Acked 1007 Write 6 Done Sent Done false
44032024-04-17T17:13:16.361Z 9 Acked 1008 Read 9 Done Sent Done false
44042024-04-17T17:13:16.361Z 10 Acked 1009 Read 5 Done Sent Done false
44052024-04-17T17:13:16.362Z 11 Acked 1010 Read 3 Done Sent Done false
44062024-04-17T17:13:16.362Z 12 Acked 1011 Write 8 Done Sent Done false
44072024-04-17T17:13:16.362Z 13 Acked 1012 Read 9 Done Sent Done false
44082024-04-17T17:13:16.362Z 14 Acked 1013 Write 10 Done Sent Done false
44092024-04-17T17:13:16.363Z 15 Acked 1014 Read 7 Done Sent Done false
44102024-04-17T17:13:16.363Z 16 Acked 1015 Flush 0 Done Sent Done false
44112024-04-17T17:13:16.363Z 17 Acked 1016 Write 5 Done Sent Done false
44122024-04-17T17:13:16.363Z 18 Acked 1017 Read 8 Done Sent Done false
44132024-04-17T17:13:16.363Z 19 Acked 1018 Read 10 Done Sent Done false
44142024-04-17T17:13:16.363Z 20 Acked 1019 Write 9 Done Sent Done false
44152024-04-17T17:13:16.364Z 21 Acked 1020 Write 8 Done Sent Done false
44162024-04-17T17:13:16.364Z 22 Acked 1021 Read 3 Done Sent Done false
44172024-04-17T17:13:16.364Z 23 Acked 1022 Write 10 Done Sent Done false
44182024-04-17T17:13:16.364Z 24 Acked 1023 Read 4 Done Sent Done false
44192024-04-17T17:13:16.364Z 25 Acked 1024 Write 1 Done Sent Done false
44202024-04-17T17:13:16.365Z 26 Acked 1025 Write 3 Done Sent Done false
44212024-04-17T17:13:16.365Z 27 Acked 1026 Write 5 Done Sent Done false
44222024-04-17T17:13:16.365Z 28 Acked 1027 Write 1 Done Sent Done false
44232024-04-17T17:13:16.366Z 29 Acked 1028 Write 4 Done Sent Done false
44242024-04-17T17:13:16.366Z 30 Acked 1029 Write 4 Sent Sent Sent false
44252024-04-17T17:13:16.366Z STATES DS:0 DS:1 DS:2 TOTAL
44262024-04-17T17:13:16.366Z New 0 0 0 0
44272024-04-17T17:13:16.367Z Sent 1 26 1 28
44282024-04-17T17:13:16.367Z Done 29 4 29 62
44292024-04-17T17:13:16.367Z Skipped 0 0 0 0
44302024-04-17T17:13:16.367Z Error 0 0 0 0
44312024-04-17T17:13:16.367ZLast Flush: 1015 0 1015
44322024-04-17T17:13:16.368ZDownstairs last five completed:
44332024-04-17T17:13:16.368ZUpstairs last five completed: 30 29 28 27 26
44342024-04-17T17:13:16.368Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:13:14.949709148Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1328}
44352024-04-17T17:13:16.368ZWrote out file "/tmp/test_repair_verify.data"
44362024-04-17T17:13:16.369Z
44372024-04-17T17:13:16.381ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
44382024-04-17T17:13:16.381ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
44392024-04-17T17:13:16.381Z 0 000-019 34 34 34 65 65 65 F F F
44402024-04-17T17:13:16.381Z 1 020-039 37 37 37 71 71 71 F F F
44412024-04-17T17:13:16.381Z 2 040-059 40 33 40 75 64 75 F T F <---
44422024-04-17T17:13:16.382Z 3 060-079 34 34 34 66 66 66 T F T <---
44432024-04-17T17:13:16.382Z 4 080-099 36 36 36 70 70 70 T F T <---
44442024-04-17T17:13:16.382Z 5 100-119 36 36 36 70 70 70 F F F
44452024-04-17T17:13:16.382Z 6 120-139 37 37 37 71 71 71 F F F
44462024-04-17T17:13:16.382Z 7 140-159 40 37 40 75 71 75 F T F <---
44472024-04-17T17:13:16.383Z 8 160-179 40 38 40 75 73 75 F T F <---
44482024-04-17T17:13:16.383Z 9 180-199 38 38 38 72 72 72 T F T <---
44492024-04-17T17:13:16.383Z 10 200-219 40 30 40 75 58 75 T F T <---
44502024-04-17T17:13:16.383Z 11 220-239 33 33 33 64 64 64 F F F
44512024-04-17T17:13:16.383Z 12 240-259 30 30 30 60 60 60 F F F
44522024-04-17T17:13:16.384Z 13 260-279 39 39 39 74 74 74 F F F
44532024-04-17T17:13:16.384Z 14 280-299 37 37 37 71 71 71 F F F
44542024-04-17T17:13:16.384Z 15 300-319 38 38 38 73 73 73 T F T <---
44552024-04-17T17:13:16.384Z 16 320-339 33 33 33 64 64 64 F F F
44562024-04-17T17:13:16.384Z 17 340-359 36 36 36 69 69 69 F F F
44572024-04-17T17:13:16.384Z 18 360-379 39 39 39 74 74 74 T F T <---
44582024-04-17T17:13:16.385Z 19 380-399 39 39 39 74 74 74 T F T <---
44592024-04-17T17:13:16.385Z 20 400-419 39 39 39 74 74 74 T F T <---
44602024-04-17T17:13:16.385Z 21 420-439 40 38 40 75 72 75 F F F <---
44612024-04-17T17:13:16.385Z 22 440-459 40 38 40 75 73 75 F F F <---
44622024-04-17T17:13:16.385Z 23 460-479 38 38 38 73 73 73 F F F
44632024-04-17T17:13:16.386Z 24 480-499 34 34 34 65 65 65 T F T <---
44642024-04-17T17:13:16.386Z 25 500-519 38 38 38 72 72 72 F F F
44652024-04-17T17:13:16.386Z 26 520-539 40 38 40 75 72 75 F F F <---
44662024-04-17T17:13:16.386Z 27 540-559 38 38 38 72 72 72 F F F
44672024-04-17T17:13:16.386Z 28 560-579 32 32 32 62 62 62 F F F
44682024-04-17T17:13:16.386Z 29 580-599 40 34 40 75 67 75 T T T <---
44692024-04-17T17:13:16.387ZMax gen: 40, Max flush: 75
44702024-04-17T17:13:16.387ZError: Difference in extent metadata found!
44712024-04-17T17:13:16.387ZA Difference in extent metadata is expected here
44722024-04-17T17:13:16.387ZOn loop 20
44732024-04-17T17:13:16.387Z
44742024-04-17T17:13:16.397ZVerifying data now
44752024-04-17T17:13:31.114ZNo change after verify
44762024-04-17T17:13:31.114ZLoop: 20 Downstairs dump after verify (and repair):
44772024-04-17T17:13:31.135ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
44782024-04-17T17:13:31.136Z 0 000-019 34 34 34 65 65 65 F F F
44792024-04-17T17:13:31.161Z 1 020-039 37 37 37 71 71 71 F F F
44802024-04-17T17:13:31.161Z 2 040-059 40 40 40 75 75 75 F F F
44812024-04-17T17:13:31.161Z 3 060-079 41 41 41 76 76 76 F F F
44822024-04-17T17:13:31.161Z 4 080-099 41 41 41 76 76 76 F F F
44832024-04-17T17:13:31.161Z 5 100-119 36 36 36 70 70 70 F F F
44842024-04-17T17:13:31.161Z 6 120-139 37 37 37 71 71 71 F F F
44852024-04-17T17:13:31.162Z 7 140-159 40 40 40 75 75 75 F F F
44862024-04-17T17:13:31.162Z 8 160-179 40 40 40 75 75 75 F F F
44872024-04-17T17:13:31.162Z 9 180-199 41 41 41 76 76 76 F F F
44882024-04-17T17:13:31.162Z 10 200-219 41 41 41 76 76 76 F F F
44892024-04-17T17:13:31.162Z 11 220-239 33 33 33 64 64 64 F F F
44902024-04-17T17:13:31.162Z 12 240-259 30 30 30 60 60 60 F F F
44912024-04-17T17:13:31.162Z 13 260-279 39 39 39 74 74 74 F F F
44922024-04-17T17:13:31.162Z 14 280-299 37 37 37 71 71 71 F F F
44932024-04-17T17:13:31.162Z 15 300-319 41 41 41 76 76 76 F F F
44942024-04-17T17:13:31.163Z 16 320-339 33 33 33 64 64 64 F F F
44952024-04-17T17:13:31.163Z 17 340-359 36 36 36 69 69 69 F F F
44962024-04-17T17:13:31.163Z 18 360-379 41 41 41 76 76 76 F F F
44972024-04-17T17:13:31.163Z 19 380-399 41 41 41 76 76 76 F F F
44982024-04-17T17:13:31.163Z 20 400-419 41 41 41 76 76 76 F F F
44992024-04-17T17:13:31.163Z 21 420-439 40 40 40 75 75 75 F F F
45002024-04-17T17:13:31.164Z 22 440-459 40 40 40 75 75 75 F F F
45012024-04-17T17:13:31.164Z 23 460-479 38 38 38 73 73 73 F F F
45022024-04-17T17:13:31.164Z 24 480-499 41 41 41 76 76 76 F F F
45032024-04-17T17:13:31.164Z 25 500-519 38 38 38 72 72 72 F F F
45042024-04-17T17:13:31.164Z 26 520-539 40 40 40 75 75 75 F F F
45052024-04-17T17:13:31.164Z 27 540-559 38 38 38 72 72 72 F F F
45062024-04-17T17:13:31.165Z 28 560-579 32 32 32 62 62 62 F F F
45072024-04-17T17:13:31.165Z 29 580-599 41 41 41 76 76 76 F F F
45082024-04-17T17:13:31.165ZMax gen: 41, Max flush: 76
45092024-04-17T17:13:31.165Z
45102024-04-17T17:13:31.165ZBegin loop 21
45112024-04-17T17:13:31.165ZDownstairs to restart: 1
45122024-04-17T17:13:31.202Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.754829192Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336}
45132024-04-17T17:13:31.202Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.75512611Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336}
45142024-04-17T17:13:31.202Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.755153372Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336}
45152024-04-17T17:13:31.202Z{"msg":"Crucible a4e16e24-ebcd-481e-b681-458491fb69c1 has session id: 943ed70c-d099-485a-a5d4-4131096e964a","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.755176173Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45162024-04-17T17:13:31.202ZCrucible runtime is spawned
45172024-04-17T17:13:31.202Z{"msg":"Upstairs opts: Upstairs UUID: a4e16e24-ebcd-481e-b681-458491fb69c1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.755245838Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45182024-04-17T17:13:31.203Z{"msg":"Crucible stats registered with UUID: a4e16e24-ebcd-481e-b681-458491fb69c1","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.75528092Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45192024-04-17T17:13:31.203Z{"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.755319822Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336}
45202024-04-17T17:13:31.203Z{"msg":"a4e16e24-ebcd-481e-b681-458491fb69c1 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.755385626Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45212024-04-17T17:13:31.203Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.755424839Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"client":"0","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45222024-04-17T17:13:31.203Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.75545617Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"client":"0","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45232024-04-17T17:13:31.203Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.755471291Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"client":"1","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45242024-04-17T17:13:31.204Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.755493303Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"client":"1","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45252024-04-17T17:13:31.204Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.755531795Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"client":"2","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45262024-04-17T17:13:31.204Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.755569087Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"client":"2","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45272024-04-17T17:13:31.204Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.755606479Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45282024-04-17T17:13:31.204Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.755639801Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45292024-04-17T17:13:31.204Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.755684584Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45302024-04-17T17:13:31.205Z{"msg":"ds_connection connected from Ok(127.0.0.1:55122)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.755836023Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45312024-04-17T17:13:31.205Z{"msg":"ds_connection connected from Ok(127.0.0.1:40260)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.755892687Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45322024-04-17T17:13:31.205Z{"msg":"ds_connection connected from Ok(127.0.0.1:58759)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.755920129Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45332024-04-17T17:13:31.205Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.756675714Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"client":"0","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45342024-04-17T17:13:31.205Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.756719427Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"client":"0","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45352024-04-17T17:13:31.205Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.756748439Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"client":"1","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45362024-04-17T17:13:31.206Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.756784961Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"client":"1","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45372024-04-17T17:13:31.206Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.756808352Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"client":"2","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45382024-04-17T17:13:31.206Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.756832144Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"client":"2","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45392024-04-17T17:13:31.206Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.757339735Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"client":"1","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45402024-04-17T17:13:31.206Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.757377017Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"client":"2","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45412024-04-17T17:13:31.206Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.757405628Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"client":"0","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45422024-04-17T17:13:31.206Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.757908399Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"client":"0","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45432024-04-17T17:13:31.207Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.757963992Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"client":"0","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45442024-04-17T17:13:31.207Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.758008615Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45452024-04-17T17:13:31.207Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.7580911Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"client":"1","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45462024-04-17T17:13:31.207Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.758118902Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"client":"1","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45472024-04-17T17:13:31.207Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.758134513Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45482024-04-17T17:13:31.207Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.758153974Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"client":"2","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45492024-04-17T17:13:31.208Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.758167615Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"client":"2","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45502024-04-17T17:13:31.208Z{"msg":"[0]R flush_numbers[0..12]: [65, 71, 75, 76, 76, 70, 71, 75, 75, 76, 76, 64]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.758188056Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"":"downstairs","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45512024-04-17T17:13:31.208Z{"msg":"[0]R generation[0..12]: [34, 37, 40, 41, 41, 36, 37, 40, 40, 41, 41, 33]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.758202047Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"":"downstairs","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45522024-04-17T17:13:31.208Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.758222748Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"":"downstairs","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45532024-04-17T17:13:31.208Z{"msg":"[1]R flush_numbers[0..12]: [65, 71, 75, 76, 76, 70, 71, 75, 75, 76, 76, 64]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.758237469Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"":"downstairs","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45542024-04-17T17:13:31.208Z{"msg":"[1]R generation[0..12]: [34, 37, 40, 41, 41, 36, 37, 40, 40, 41, 41, 33]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.75825301Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"":"downstairs","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45552024-04-17T17:13:31.208Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.758275701Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"":"downstairs","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45562024-04-17T17:13:31.208Z{"msg":"[2]R flush_numbers[0..12]: [65, 71, 75, 76, 76, 70, 71, 75, 75, 76, 76, 64]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.758290022Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"":"downstairs","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45572024-04-17T17:13:31.208Z{"msg":"[2]R generation[0..12]: [34, 37, 40, 41, 41, 36, 37, 40, 40, 41, 41, 33]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.758315754Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"":"downstairs","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45582024-04-17T17:13:31.208Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.758337015Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"":"downstairs","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45592024-04-17T17:13:31.209Z{"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.758372747Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO
45602024-04-17T17:13:31.209Zsession_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45612024-04-17T17:13:31.209Z{"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.758399839Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"":"downstairs","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45622024-04-17T17:13:31.209Z{"msg":"Next flush: 77","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.758434651Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"":"downstairs","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45632024-04-17T17:13:31.209Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.758459312Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"":"downstairs","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45642024-04-17T17:13:31.209Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.758474843Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45652024-04-17T17:13:31.209Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.758507305Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"client":"0","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45662024-04-17T17:13:31.209Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.758540737Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"client":"1","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45672024-04-17T17:13:31.209Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.758556718Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"client":"2","":"downstairs","session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45682024-04-17T17:13:31.209Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.75858106Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45692024-04-17T17:13:31.209Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.758614912Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45702024-04-17T17:13:31.209Z{"msg":"a4e16e24-ebcd-481e-b681-458491fb69c1 is now active with session: 943ed70c-d099-485a-a5d4-4131096e964a","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.758645134Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45712024-04-17T17:13:31.209Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.758658675Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336,"session_id":"943ed70c-d099-485a-a5d4-4131096e964a"}
45722024-04-17T17:13:31.209Z{"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:29.758681846Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336}
45732024-04-17T17:13:31.209ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
45742024-04-17T17:13:31.209ZLoading write count information from file "/tmp/test_repair_verify.data"
45752024-04-17T17:13:31.209ZRun Repair workload
45762024-04-17T17:13:31.209Z01/30 Write block 507 len 24576 data: 4 4 3 3 3 4
45772024-04-17T17:13:31.209Z02/30 Write block 483 len 24576 data: 7 7 6 7 5 6
45782024-04-17T17:13:31.209Z03/30 Read block 517 len 20480
45792024-04-17T17:13:31.209Z04/30 Write block 312 len 28672 data: 5 5 6 5 5 5 5
45802024-04-17T17:13:31.209Z05/30 Flush
45812024-04-17T17:13:31.209Z06/30 Write block 24 len 4096 data: 6
45822024-04-17T17:13:31.210Z07/30 Write block 499 len 28672 data: 5 5 5 5 4 4 2
45832024-04-17T17:13:31.210Z08/30 Read block 120 len 12288
45842024-04-17T17:13:31.210Z09/30 Read block 383 len 40960
45852024-04-17T17:13:31.210Z10/30 Read block 307 len 36864
45862024-04-17T17:13:31.210Z11/30 Read block 177 len 28672
45872024-04-17T17:13:31.210Z12/30 Read block 341 len 12288
45882024-04-17T17:13:31.210Z13/30 Write block 436 len 8192 data: 5 5
45892024-04-17T17:13:31.210Z14/30 Write block 301 len 16384 data: 4 3 4 4
45902024-04-17T17:13:31.210Z15/30 Read block 224 len 36864
45912024-04-17T17:13:31.210Z16/30 Write block 123 len 8192 data: 3 3
45922024-04-17T17:13:31.210Z17/30 Write block 45 len 20480 data: 6 5 3 2 3
45932024-04-17T17:13:31.210Z18/30 Write block 534 len 20480 data: 3 3 4 3 3
45942024-04-17T17:13:31.210Z19/30 Write block 252 len 12288 data: 6 5 4
45952024-04-17T17:13:31.210Z20/30 Read block 482 len 4096
45962024-04-17T17:13:31.229Z21/30 Read block 285 len 12288
45972024-04-17T17:13:31.229Z22/30 Write block 233 len 20480 data: 3 3 4 2 2
45982024-04-17T17:13:31.229Z23/30 Read block 40 len 4096
45992024-04-17T17:13:31.229Z24/30 Write block 95 len 40960 data: 6 3 3 3 4 4 4 3 5 5
46002024-04-17T17:13:31.229Z25/30 Write block 548 len 36864 data: 3 3 3 4 5 5 5 5 5
46012024-04-17T17:13:31.229Z26/30 Read block 296 len 32768
46022024-04-17T17:13:31.229Z27/30 Read block 571 len 16384
46032024-04-17T17:13:31.229Z28/30 Read block 363 len 12288
46042024-04-17T17:13:31.229Z29/30 Read block 345 len 40960
46052024-04-17T17:13:31.229Z30/30 Write block 306 len 4096 data: 5
46062024-04-17T17:13:31.229Z----------------------------------------------------------------
46072024-04-17T17:13:31.229Z Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30
46082024-04-17T17:13:31.229ZStates: Active Active Active
46092024-04-17T17:13:31.229ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
46102024-04-17T17:13:31.229Z 1 Acked 1000 Write 6 Done Done Done false
46112024-04-17T17:13:31.229Z 2 Acked 1001 Write 6 Done Done Done false
46122024-04-17T17:13:31.229Z 3 Acked 1002 Read 5 Done Done Done false
46132024-04-17T17:13:31.229Z 4 Acked 1003 Write 7 Done Sent Done false
46142024-04-17T17:13:31.229Z 5 Acked 1004 Flush 0 Done Sent Done false
46152024-04-17T17:13:31.229Z 6 Acked 1005 Write 1 Done Sent Done false
46162024-04-17T17:13:31.229Z 7 Acked 1006 Write 7 Done Sent Done false
46172024-04-17T17:13:31.229Z 8 Acked 1007 Read 3 Done Sent Done false
46182024-04-17T17:13:31.230Z 9 Acked 1008 Read 10 Done Sent Done false
46192024-04-17T17:13:31.230Z 10 Acked 1009 Read 9 Done Sent Done false
46202024-04-17T17:13:31.230Z 11 Acked 1010 Read 7 Done Sent Done false
46212024-04-17T17:13:31.230Z 12 Acked 1011 Read 3 Done Sent Done false
46222024-04-17T17:13:31.230Z 13 Acked 1012 Write 2 Done Sent Done false
46232024-04-17T17:13:31.230Z 14 Acked 1013 Write 4 Done Sent Done false
46242024-04-17T17:13:31.230Z 15 Acked 1014 Read 9 Done Sent Done false
46252024-04-17T17:13:31.230Z 16 Acked 1015 Write 2 Done Sent Done false
46262024-04-17T17:13:31.230Z 17 Acked 1016 Write 5 Done Sent Done false
46272024-04-17T17:13:31.230Z 18 Acked 1017 Write 5 Done Sent Done false
46282024-04-17T17:13:31.230Z 19 Acked 1018 Write 3 Done Sent Done false
46292024-04-17T17:13:31.230Z 20 Acked 1019 Read 1 Done Sent Done false
46302024-04-17T17:13:31.230Z 21 Acked 1020 Read 3 Done Sent Done false
46312024-04-17T17:13:31.230Z 22 Acked 1021 Write 5 Done Sent Done false
46322024-04-17T17:13:31.230Z 23 Acked 1022 Read 1 Done Sent Done false
46332024-04-17T17:13:31.230Z 24 Acked 1023 Write 10 Done Sent Done false
46342024-04-17T17:13:31.230Z 25 Acked 1024 Write 9 Done Sent Done false
46352024-04-17T17:13:31.230Z 26 Acked 1025 Read 8 Done Sent Done false
46362024-04-17T17:13:31.230Z 27 Acked 1026 Read 4 Done Sent Done false
46372024-04-17T17:13:31.230Z 28 Acked 1027 Read 3 Done Sent Done false
46382024-04-17T17:13:31.230Z 29 Acked 1028 Read 10 Done Sent Done false
46392024-04-17T17:13:31.230Z 30 Acked 1029 Write 1 Sent Sent Sent false
46402024-04-17T17:13:31.230Z STATES DS:0 DS:1 DS:2 TOTAL
46412024-04-17T17:13:31.231Z New 0 0 0 0
46422024-04-17T17:13:31.231Z Sent 1 27 1 29
46432024-04-17T17:13:31.231Z Done 29 3 29 61
46442024-04-17T17:13:31.231Z Skipped 0 0 0 0
46452024-04-17T17:13:31.231Z Error 0 0 0 0
46462024-04-17T17:13:31.231ZLast Flush: 1004 0 1004
46472024-04-17T17:13:31.231ZDownstairs last five completed:
46482024-04-17T17:13:31.231ZUpstairs last five completed: 30 29 28 27 26
46492024-04-17T17:13:31.231Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:13:29.78877851Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1336}
46502024-04-17T17:13:31.231ZWrote out file "/tmp/test_repair_verify.data"
46512024-04-17T17:13:31.231Z
46522024-04-17T17:13:31.231ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
46532024-04-17T17:13:31.231ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
46542024-04-17T17:13:31.231Z 0 000-019 34 34 34 65 65 65 F F F
46552024-04-17T17:13:31.231Z 1 020-039 37 37 37 71 71 71 T F T <---
46562024-04-17T17:13:31.231Z 2 040-059 40 40 40 75 75 75 T F T <---
46572024-04-17T17:13:31.231Z 3 060-079 41 41 41 76 76 76 F F F
46582024-04-17T17:13:31.231Z 4 080-099 41 41 41 76 76 76 T F T <---
46592024-04-17T17:13:31.231Z 5 100-119 36 36 36 70 70 70 T F T <---
46602024-04-17T17:13:31.231Z 6 120-139 37 37 37 71 71 71 T F T <---
46612024-04-17T17:13:31.231Z 7 140-159 40 40 40 75 75 75 F F F
46622024-04-17T17:13:31.231Z 8 160-179 40 40 40 75 75 75 F F F
46632024-04-17T17:13:31.231Z 9 180-199 41 41 41 76 76 76 F F F
46642024-04-17T17:13:31.231Z 10 200-219 41 41 41 76 76 76 F F F
46652024-04-17T17:13:31.232Z 11 220-239 33 33 33 64 64 64 T F T <---
46662024-04-17T17:13:31.232Z 12 240-259 30 30 30 60 60 60 T F T <---
46672024-04-17T17:13:31.232Z 13 260-279 39 39 39 74 74 74 F F F
46682024-04-17T17:13:31.232Z 14 280-299 37 37 37 71 71 71 F F F
46692024-04-17T17:13:31.232Z 15 300-319 42 41 42 77 76 77 T F T <---
46702024-04-17T17:13:31.232Z 16 320-339 33 33 33 64 64 64 F F F
46712024-04-17T17:13:31.232Z 17 340-359 36 36 36 69 69 69 F F F
46722024-04-17T17:13:31.232Z 18 360-379 41 41 41 76 76 76 F F F
46732024-04-17T17:13:31.232Z 19 380-399 41 41 41 76 76 76 F F F
46742024-04-17T17:13:31.232Z 20 400-419 41 41 41 76 76 76 F F F
46752024-04-17T17:13:31.232Z 21 420-439 40 40 40 75 75 75 T F T <---
46762024-04-17T17:13:31.232Z 22 440-459 40 40 40 75 75 75 F F F
46772024-04-17T17:13:31.232Z 23 460-479 38 38 38 73 73 73 F F F
46782024-04-17T17:13:31.232Z 24 480-499 42 41 42 77 76 77 T T T <---
46792024-04-17T17:13:31.232Z 25 500-519 42 38 42 77 72 77 T T T <---
46802024-04-17T17:13:31.232Z 26 520-539 40 40 40 75 75 75 T F T <---
46812024-04-17T17:13:31.232Z 27 540-559 38 38 38 72 72 72 T F T <---
46822024-04-17T17:13:31.232Z 28 560-579 32 32 32 62 62 62 F F F
46832024-04-17T17:13:31.232Z 29 580-599 41 41 41 76 76 76 F F F
46842024-04-17T17:13:31.232ZMax gen: 42, Max flush: 77
46852024-04-17T17:13:31.232ZError: Difference in extent metadata found!
46862024-04-17T17:13:31.232ZA Difference in extent metadata is expected here
46872024-04-17T17:13:31.232ZOn loop 21
46882024-04-17T17:13:31.232Z
46892024-04-17T17:13:31.233ZVerifying data now
46902024-04-17T17:13:45.862ZNo change after verify
46912024-04-17T17:13:45.862ZLoop: 21 Downstairs dump after verify (and repair):
46922024-04-17T17:13:45.877ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
46932024-04-17T17:13:45.877Z 0 000-019 34 34 34 65 65 65 F F F
46942024-04-17T17:13:45.891Z 1 020-039 43 43 43 78 78 78 F F F
46952024-04-17T17:13:45.892Z 2 040-059 43 43 43 78 78 78 F F F
46962024-04-17T17:13:45.892Z 3 060-079 41 41 41 76 76 76 F F F
46972024-04-17T17:13:45.892Z 4 080-099 43 43 43 78 78 78 F F F
46982024-04-17T17:13:45.892Z 5 100-119 43 43 43 78 78 78 F F F
46992024-04-17T17:13:45.892Z 6 120-139 43 43 43 78 78 78 F F F
47002024-04-17T17:13:45.892Z 7 140-159 40 40 40 75 75 75 F F F
47012024-04-17T17:13:45.892Z 8 160-179 40 40 40 75 75 75 F F F
47022024-04-17T17:13:45.893Z 9 180-199 41 41 41 76 76 76 F F F
47032024-04-17T17:13:45.893Z 10 200-219 41 41 41 76 76 76 F F F
47042024-04-17T17:13:45.893Z 11 220-239 43 43 43 78 78 78 F F F
47052024-04-17T17:13:45.893Z 12 240-259 43 43 43 78 78 78 F F F
47062024-04-17T17:13:45.893Z 13 260-279 39 39 39 74 74 74 F F F
47072024-04-17T17:13:45.893Z 14 280-299 37 37 37 71 71 71 F F F
47082024-04-17T17:13:45.893Z 15 300-319 43 43 43 78 78 78 F F F
47092024-04-17T17:13:45.894Z 16 320-339 33 33 33 64 64 64 F F F
47102024-04-17T17:13:45.894Z 17 340-359 36 36 36 69 69 69 F F F
47112024-04-17T17:13:45.894Z 18 360-379 41 41 41 76 76 76 F F F
47122024-04-17T17:13:45.894Z 19 380-399 41 41 41 76 76 76 F F F
47132024-04-17T17:13:45.894Z 20 400-419 41 41 41 76 76 76 F F F
47142024-04-17T17:13:45.894Z 21 420-439 43 43 43 78 78 78 F F F
47152024-04-17T17:13:45.895Z 22 440-459 40 40 40 75 75 75 F F F
47162024-04-17T17:13:45.895Z 23 460-479 38 38 38 73 73 73 F F F
47172024-04-17T17:13:45.895Z 24 480-499 43 43 43 78 78 78 F F F
47182024-04-17T17:13:45.895Z 25 500-519 43 43 43 78 78 78 F F F
47192024-04-17T17:13:45.895Z 26 520-539 43 43 43 78 78 78 F F F
47202024-04-17T17:13:45.895Z 27 540-559 43 43 43 78 78 78 F F F
47212024-04-17T17:13:45.896Z 28 560-579 32 32 32 62 62 62 F F F
47222024-04-17T17:13:45.896Z 29 580-599 41 41 41 76 76 76 F F F
47232024-04-17T17:13:45.896ZMax gen: 43, Max flush: 78
47242024-04-17T17:13:45.896Z
47252024-04-17T17:13:45.896ZBegin loop 22
47262024-04-17T17:13:45.896ZDownstairs to restart: 0
47272024-04-17T17:13:45.911Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.504298736Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344}
47282024-04-17T17:13:45.912Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.50453376Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344}
47292024-04-17T17:13:45.912Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.504562541Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344}
47302024-04-17T17:13:45.912Z{"msg":"Crucible 69e699af-bd82-47fe-8d61-41622d42d2ab has session id: effdf3d5-8739-4ee4-9d47-b0083fa22ea1","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.504576312Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47312024-04-17T17:13:45.912Z{Crucible runtime is spawned
47322024-04-17T17:13:45.912Z"msg":"Upstairs opts: Upstairs UUID: 69e699af-bd82-47fe-8d61-41622d42d2ab, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.504718351Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47332024-04-17T17:13:45.913Z{"msg":"Crucible stats registered with UUID: 69e699af-bd82-47fe-8d61-41622d42d2ab","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.504752763Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47342024-04-17T17:13:45.913Z{"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.504783105Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344}
47352024-04-17T17:13:45.913Z{"msg":"69e699af-bd82-47fe-8d61-41622d42d2ab active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.50486303Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47362024-04-17T17:13:45.914Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.504889021Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"client":"0","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47372024-04-17T17:13:45.914Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.504906712Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"client":"0","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47382024-04-17T17:13:45.914Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.504939714Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"client":"1","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47392024-04-17T17:13:45.914Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.504968876Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"client":"1","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47402024-04-17T17:13:45.915Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.504991787Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"client":"2","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47412024-04-17T17:13:45.915Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.50502674Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"client":"2","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47422024-04-17T17:13:45.915Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.505053851Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47432024-04-17T17:13:45.915Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.505096034Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47442024-04-17T17:13:45.915Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.505134606Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47452024-04-17T17:13:45.916Z{"msg":"ds_connection connected from Ok(127.0.0.1:63559)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.505181079Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47462024-04-17T17:13:45.916Z{"msg":"ds_connection connected from Ok(127.0.0.1:37137)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.505219101Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47472024-04-17T17:13:45.916Z{"msg":"ds_connection connected from Ok(127.0.0.1:39156)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.505245563Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47482024-04-17T17:13:45.916Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.505992308Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"client":"1","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47492024-04-17T17:13:45.917Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.506034801Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"client":"1","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47502024-04-17T17:13:45.917Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.506056732Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"client":"2","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47512024-04-17T17:13:45.917Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.506098525Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"client":"2","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47522024-04-17T17:13:45.917Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.506204131Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"client":"0","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47532024-04-17T17:13:45.917Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.506241993Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"client":"0","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47542024-04-17T17:13:45.918Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.506594055Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"client":"1","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47552024-04-17T17:13:45.918Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.506647988Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"client":"2","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47562024-04-17T17:13:45.918Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.507083624Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"client":"0","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47572024-04-17T17:13:45.918Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.507254125Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"client":"1","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47582024-04-17T17:13:45.919Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.50734087Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"client":"1","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47592024-04-17T17:13:45.919Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.507378602Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47602024-04-17T17:13:45.919Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.507394053Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"client":"2","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47612024-04-17T17:13:45.919Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.507426165Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"client":"2","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47622024-04-17T17:13:45.919Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.507442726Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47632024-04-17T17:13:45.920Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.507565414Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"client":"0","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47642024-04-17T17:13:45.920Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.507607496Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"client":"0","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47652024-04-17T17:13:45.920Z{"msg":"[0]R flush_numbers[0..12]: [65, 78, 78, 76, 78, 78, 78, 75, 75, 76, 76, 78]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.507624587Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"":"downstairs","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47662024-04-17T17:13:45.920Z{"msg":"[0]R generation[0..12]: [34, 43, 43, 41, 43, 43, 43, 40, 40, 41, 41, 43]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.507646138Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"":"downstairs","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47672024-04-17T17:13:45.920Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.507660769Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"":"downstairs","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47682024-04-17T17:13:45.921Z{"msg":"[1]R flush_numbers[0..12]: [65, 78, 78, 76, 78, 78, 78, 75, 75, 76, 76, 78]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.50768127Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"":"downstairs","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47692024-04-17T17:13:45.921Z{"msg":"[1]R generation[0..12]: [34, 43, 43, 41, 43, 43, 43, 40, 40, 41, 41, 43]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.507695571Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"":"downstairs","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47702024-04-17T17:13:45.921Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.507709972Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"":"downstairs","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47712024-04-17T17:13:45.921Z{"msg":"[2]R flush_numbers[0..12]: [65, 78, 78, 76, 78, 78, 78, 75, 75, 76, 76, 78]","v":0,"name":"crucible","level":30,"timeWait for a query_work_queue command to finish before sending IO
47722024-04-17T17:13:45.922Z":"2024-04-17T17:13:44.507730733Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"":"downstairs","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47732024-04-17T17:13:45.922Z{"msg":"[2]R generation[0..12]: [34, 43, 43, 41, 43, 43, 43, 40, 40, 41, 41, 43]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.507748014Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"":"downstairs","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47742024-04-17T17:13:45.922Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.507768216Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"":"downstairs","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47752024-04-17T17:13:45.922Z{"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.507781416Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"":"downstairs","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47762024-04-17T17:13:45.922Z{"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.507800828Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"":"downstairs","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47772024-04-17T17:13:45.923Z{"msg":"Next flush: 79","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.507813609Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"":"downstairs","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47782024-04-17T17:13:45.923Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.5078331Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"":"downstairs","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47792024-04-17T17:13:45.923Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.507846831Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47802024-04-17T17:13:45.923Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.507865202Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"client":"0","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47812024-04-17T17:13:45.924Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.507878562Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"client":"1","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47822024-04-17T17:13:45.924Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.507897784Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"client":"2","":"downstairs","session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47832024-04-17T17:13:45.924Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.507911024Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47842024-04-17T17:13:45.924Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.507929796Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47852024-04-17T17:13:45.925Z{"msg":"69e699af-bd82-47fe-8d61-41622d42d2ab is now active with session: effdf3d5-8739-4ee4-9d47-b0083fa22ea1","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.507942636Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47862024-04-17T17:13:45.925Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.507954417Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344,"session_id":"effdf3d5-8739-4ee4-9d47-b0083fa22ea1"}
47872024-04-17T17:13:45.925Z{"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:44.507973028Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344}
47882024-04-17T17:13:45.925ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
47892024-04-17T17:13:45.925ZLoading write count information from file "/tmp/test_repair_verify.data"
47902024-04-17T17:13:45.926ZRun Repair workload
47912024-04-17T17:13:45.955Z01/30 Write block 253 len 28672 data: 6 5 3 5 5 5 5
47922024-04-17T17:13:45.955Z02/30 Write block 478 len 28672 data: 6 7 7 6 6 8 8
47932024-04-17T17:13:45.955Z03/30 Write block 8 len 28672 data: 5 6 6 6 6 5 6
47942024-04-17T17:13:45.955Z04/30 Flush
47952024-04-17T17:13:45.956Z05/30 Write block 82 len 24576 data: 3 3 3 3 3 5
47962024-04-17T17:13:45.956Z06/30 Write block 202 len 24576 data: 6 7 7 7 5 5
47972024-04-17T17:13:45.956Z07/30 Read block 22 len 4096
47982024-04-17T17:13:45.956Z08/30 Write block 241 len 4096 data: 5
47992024-04-17T17:13:45.957Z09/30 Read block 234 len 24576
48002024-04-17T17:13:45.957Z10/30 Read block 342 len 8192
48012024-04-17T17:13:45.958Z11/30 Write block 62 len 16384 data: 4 5 5 4
48022024-04-17T17:13:45.958Z12/30 Read block 213 len 20480
48032024-04-17T17:13:45.958Z13/30 Read block 432 len 36864
48042024-04-17T17:13:45.958Z14/30 Read block 142 len 40960
48052024-04-17T17:13:45.959Z15/30 Read block 427 len 36864
48062024-04-17T17:13:45.959Z16/30 Read block 536 len 40960
48072024-04-17T17:13:45.959Z17/30 Read block 44 len 4096
48082024-04-17T17:13:45.959Z18/30 Flush
48092024-04-17T17:13:45.960Z19/30 Write block 529 len 20480 data: 6 6 5 4 4
48102024-04-17T17:13:45.960Z20/30 Write block 396 len 36864 data: 3 3 4 4 4 4 4 6 5
48112024-04-17T17:13:45.960Z21/30 Read block 522 len 24576
48122024-04-17T17:13:45.960Z22/30 Read block 403 len 12288
48132024-04-17T17:13:45.961Z23/30 Read block 84 len 32768
48142024-04-17T17:13:45.961Z24/30 Flush
48152024-04-17T17:13:45.961Z25/30 Write block 334 len 12288 data: 5 6 5
48162024-04-17T17:13:45.961Z26/30 Write block 107 len 20480 data: 6 6 5 5 7
48172024-04-17T17:13:45.962Z27/30 Read block 508 len 40960
48182024-04-17T17:13:45.962Z28/30 Write block 252 len 4096 data: 7
48192024-04-17T17:13:45.962Z29/30 Write block 558 len 28672 data: 5 5 4 3 3 2 3
48202024-04-17T17:13:45.962Z30/30 Write block 448 len 20480 data: 4 5 5 3 4
48212024-04-17T17:13:45.962Z----------------------------------------------------------------
48222024-04-17T17:13:45.963Z Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:26
48232024-04-17T17:13:45.963ZStates: Active Active Active
48242024-04-17T17:13:45.963ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
48252024-04-17T17:13:45.963Z 5 Acked 1004 Write 6 Sent Done Done false
48262024-04-17T17:13:45.963Z 6 Acked 1005 Write 6 Sent Done Done false
48272024-04-17T17:13:45.964Z 7 Acked 1006 Read 1 Sent Done Done false
48282024-04-17T17:13:45.964Z 8 Acked 1007 Write 1 Sent Done Done false
48292024-04-17T17:13:45.964Z 9 Acked 1008 Read 6 Sent Done Done false
48302024-04-17T17:13:45.964Z 10 Acked 1009 Read 2 Sent Done Done false
48312024-04-17T17:13:45.964Z 11 Acked 1010 Write 4 Sent Done Done false
48322024-04-17T17:13:45.965Z 12 Acked 1011 Read 5 Sent Done Done false
48332024-04-17T17:13:45.965Z 13 Acked 1012 Read 9 Sent Done Done false
48342024-04-17T17:13:45.965Z 14 Acked 1013 Read 10 Sent Done Done false
48352024-04-17T17:13:45.965Z 15 Acked 1014 Read 9 Sent Done Done false
48362024-04-17T17:13:45.965Z 16 Acked 1015 Read 10 Sent Done Done false
48372024-04-17T17:13:45.966Z 17 Acked 1016 Read 1 Sent Done Done false
48382024-04-17T17:13:45.966Z 18 Acked 1017 Flush 0 Sent Done Done false
48392024-04-17T17:13:45.966Z 19 Acked 1018 Write 5 Sent Done Done false
48402024-04-17T17:13:45.966Z 20 Acked 1019 Write 9 Sent Done Done false
48412024-04-17T17:13:45.966Z 21 Acked 1020 Read 6 Sent Done Done false
48422024-04-17T17:13:45.967Z 22 Acked 1021 Read 3 Sent Done Done false
48432024-04-17T17:13:45.967Z 23 Acked 1022 Read 8 Sent Done Done false
48442024-04-17T17:13:45.967Z 24 Acked 1023 Flush 0 Sent Done Done false
48452024-04-17T17:13:45.967Z 25 Acked 1024 Write 3 Sent Done Done false
48462024-04-17T17:13:45.968Z 26 Acked 1025 Write 5 Sent Done Done false
48472024-04-17T17:13:45.968Z 27 Acked 1026 Read 10 Sent Done Done false
48482024-04-17T17:13:45.968Z 28 Acked 1027 Write 1 Sent Done Done false
48492024-04-17T17:13:45.968Z 29 Acked 1028 Write 7 Sent Done Done false
48502024-04-17T17:13:45.968Z 30 Acked 1029 Write 5 Sent Sent Sent false
48512024-04-17T17:13:45.968Z STATES DS:0 DS:1 DS:2 TOTAL
48522024-04-17T17:13:45.968Z New 0 0 0 0
48532024-04-17T17:13:45.968Z Sent 26 1 1 28
48542024-04-17T17:13:45.968Z Done 0 25 25 50
48552024-04-17T17:13:45.968Z Skipped 0 0 0 0
48562024-04-17T17:13:45.968Z Error 0 0 0 0
48572024-04-17T17:13:45.968ZLast Flush: 1003 1023 1023
48582024-04-17T17:13:45.969ZDownstairs last five completed: 1003 1002 1001 1000
48592024-04-17T17:13:45.969ZUpstairs last five completed: 30 29 28 27 26
48602024-04-17T17:13:45.969Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:13:44.541260646Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1344}
48612024-04-17T17:13:45.969ZWrote out file "/tmp/test_repair_verify.data"
48622024-04-17T17:13:45.969Z
48632024-04-17T17:13:46.389ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
48642024-04-17T17:13:46.390ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
48652024-04-17T17:13:46.390Z 0 000-019 44 44 44 79 79 79 F F F
48662024-04-17T17:13:46.390Z 1 020-039 43 43 43 78 78 78 F F F
48672024-04-17T17:13:46.390Z 2 040-059 43 43 43 78 78 78 F F F
48682024-04-17T17:13:46.390Z 3 060-079 41 44 44 76 80 80 F F F <---
48692024-04-17T17:13:46.391Z 4 080-099 43 44 44 78 80 80 F F F <---
48702024-04-17T17:13:46.391Z 5 100-119 43 43 43 78 78 78 F T T <---
48712024-04-17T17:13:46.391Z 6 120-139 43 43 43 78 78 78 F F F
48722024-04-17T17:13:46.391Z 7 140-159 40 40 40 75 75 75 F F F
48732024-04-17T17:13:46.391Z 8 160-179 40 40 40 75 75 75 F F F
48742024-04-17T17:13:46.391Z 9 180-199 41 41 41 76 76 76 F F F
48752024-04-17T17:13:46.392Z 10 200-219 41 44 44 76 80 80 F F F <---
48762024-04-17T17:13:46.392Z 11 220-239 43 43 43 78 78 78 F F F
48772024-04-17T17:13:46.392Z 12 240-259 44 44 44 79 80 80 F T T <---
48782024-04-17T17:13:46.392Z 13 260-279 39 39 39 74 74 74 F F F
48792024-04-17T17:13:46.392Z 14 280-299 37 37 37 71 71 71 F F F
48802024-04-17T17:13:46.393Z 15 300-319 43 43 43 78 78 78 F F F
48812024-04-17T17:13:46.393Z 16 320-339 33 33 33 64 64 64 F T T <---
48822024-04-17T17:13:46.393Z 17 340-359 36 36 36 69 69 69 F F F
48832024-04-17T17:13:46.393Z 18 360-379 41 41 41 76 76 76 F F F
48842024-04-17T17:13:46.394Z 19 380-399 41 44 44 76 81 81 F F F <---
48852024-04-17T17:13:46.394Z 20 400-419 41 44 44 76 81 81 F F F <---
48862024-04-17T17:13:46.394Z 21 420-439 43 43 43 78 78 78 F F F
48872024-04-17T17:13:46.394Z 22 440-459 40 40 40 75 75 75 F T T <---
48882024-04-17T17:13:46.394Z 23 460-479 44 44 44 79 79 79 F F F
48892024-04-17T17:13:46.395Z 24 480-499 44 44 44 79 79 79 F F F
48902024-04-17T17:13:46.395Z 25 500-519 43 43 43 78 78 78 F F F
48912024-04-17T17:13:46.395Z 26 520-539 43 44 44 78 81 81 F F F <---
48922024-04-17T17:13:46.395Z 27 540-559 43 43 43 78 78 78 F T T <---
48932024-04-17T17:13:46.396Z 28 560-579 32 32 32 62 62 62 F T T <---
48942024-04-17T17:13:46.396Z 29 580-599 41 41 41 76 76 76 F F F
48952024-04-17T17:13:46.396ZMax gen: 44, Max flush: 81
48962024-04-17T17:13:46.396ZError: Difference in extent metadata found!
48972024-04-17T17:13:46.396ZA Difference in extent metadata is expected here
48982024-04-17T17:13:46.397ZOn loop 22
48992024-04-17T17:13:46.397Z
49002024-04-17T17:13:46.397ZVerifying data now
49012024-04-17T17:13:50.541ZNo change after verify
49022024-04-17T17:13:50.541ZLoop: 22 Downstairs dump after verify (and repair):
49032024-04-17T17:13:50.556ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
49042024-04-17T17:13:50.556Z 0 000-019 44 44 44 79 79 79 F F F
49052024-04-17T17:13:50.556Z 1 020-039 43 43 43 78 78 78 F F F
49062024-04-17T17:13:50.556Z 2 040-059 43 43 43 78 78 78 F F F
49072024-04-17T17:13:50.560Z 3 060-079 44 44 44 80 80 80 F F F
49082024-04-17T17:13:50.560Z 4 080-099 44 44 44 80 80 80 F F F
49092024-04-17T17:13:50.560Z 5 100-119 45 45 45 82 82 82 F F F
49102024-04-17T17:13:50.560Z 6 120-139 43 43 43 78 78 78 F F F
49112024-04-17T17:13:50.560Z 7 140-159 40 40 40 75 75 75 F F F
49122024-04-17T17:13:50.560Z 8 160-179 40 40 40 75 75 75 F F F
49132024-04-17T17:13:50.560Z 9 180-199 41 41 41 76 76 76 F F F
49142024-04-17T17:13:50.560Z 10 200-219 44 44 44 80 80 80 F F F
49152024-04-17T17:13:50.560Z 11 220-239 43 43 43 78 78 78 F F F
49162024-04-17T17:13:50.560Z 12 240-259 45 45 45 82 82 82 F F F
49172024-04-17T17:13:50.561Z 13 260-279 39 39 39 74 74 74 F F F
49182024-04-17T17:13:50.561Z 14 280-299 37 37 37 71 71 71 F F F
49192024-04-17T17:13:50.561Z 15 300-319 43 43 43 78 78 78 F F F
49202024-04-17T17:13:50.561Z 16 320-339 45 45 45 82 82 82 F F F
49212024-04-17T17:13:50.561Z 17 340-359 36 36 36 69 69 69 F F F
49222024-04-17T17:13:50.561Z 18 360-379 41 41 41 76 76 76 F F F
49232024-04-17T17:13:50.561Z 19 380-399 44 44 44 81 81 81 F F F
49242024-04-17T17:13:50.561Z 20 400-419 44 44 44 81 81 81 F F F
49252024-04-17T17:13:50.561Z 21 420-439 43 43 43 78 78 78 F F F
49262024-04-17T17:13:50.561Z 22 440-459 45 45 45 82 82 82 F F F
49272024-04-17T17:13:50.561Z 23 460-479 44 44 44 79 79 79 F F F
49282024-04-17T17:13:50.561Z 24 480-499 44 44 44 79 79 79 F F F
49292024-04-17T17:13:50.562Z 25 500-519 43 43 43 78 78 78 F F F
49302024-04-17T17:13:50.562Z 26 520-539 44 44 44 81 81 81 F F F
49312024-04-17T17:13:50.562Z 27 540-559 45 45 45 82 82 82 F F F
49322024-04-17T17:13:50.562Z 28 560-579 45 45 45 82 82 82 F F F
49332024-04-17T17:13:50.562Z 29 580-599 41 41 41 76 76 76 F F F
49342024-04-17T17:13:50.562ZMax gen: 45, Max flush: 82
49352024-04-17T17:13:50.562Z
49362024-04-17T17:13:50.562ZBegin loop 23
49372024-04-17T17:13:50.562ZDownstairs to restart: 0
49382024-04-17T17:13:50.578Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.185120758Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352}
49392024-04-17T17:13:50.586Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.185376214Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352}
49402024-04-17T17:13:50.586Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30Crucible runtime is spawned
49412024-04-17T17:13:50.586Z,"time":"2024-04-17T17:13:49.185414726Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352}
49422024-04-17T17:13:50.586Z{"msg":"Crucible 28eea101-b598-43d2-9d17-db61d0532868 has session id: b52a8930-611e-4371-a1b9-ffc602f44cfb","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.185456979Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49432024-04-17T17:13:50.586Z{"msg":"Upstairs opts: Upstairs UUID: 28eea101-b598-43d2-9d17-db61d0532868, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.185516042Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49442024-04-17T17:13:50.586Z{"msg":"Crucible stats registered with UUID: 28eea101-b598-43d2-9d17-db61d0532868","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.185554374Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49452024-04-17T17:13:50.586Z{"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.185577696Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352}
49462024-04-17T17:13:50.586Z{"msg":"28eea101-b598-43d2-9d17-db61d0532868 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.185609508Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49472024-04-17T17:13:50.586Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.185633289Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"client":"0","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49482024-04-17T17:13:50.586Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.185675992Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"client":"0","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49492024-04-17T17:13:50.586Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.185701003Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"client":"1","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49502024-04-17T17:13:50.586Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.185726515Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"client":"1","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49512024-04-17T17:13:50.586Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.185768567Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"client":"2","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49522024-04-17T17:13:50.586Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.185793449Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"client":"2","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49532024-04-17T17:13:50.586Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.185829521Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49542024-04-17T17:13:50.586Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.185860673Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49552024-04-17T17:13:50.586Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.185885105Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49562024-04-17T17:13:50.587Z{"msg":"ds_connection connected from Ok(127.0.0.1:63325)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.186102398Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49572024-04-17T17:13:50.587Z{"msg":"ds_connection connected from Ok(127.0.0.1:39610)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.186149421Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49582024-04-17T17:13:50.587Z{"msg":"ds_connection connected from Ok(127.0.0.1:39653)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.186194913Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49592024-04-17T17:13:50.587Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.186982221Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"client":"1","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49602024-04-17T17:13:50.587Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.187023704Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"client":"1","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49612024-04-17T17:13:50.587Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.187039455Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"client":"2","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49622024-04-17T17:13:50.587Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.187053845Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"client":"2","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49632024-04-17T17:13:50.587Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.187084157Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"client":"0","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49642024-04-17T17:13:50.587Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.187138581Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"client":"0","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49652024-04-17T17:13:50.587Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.187631231Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"client":"1","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49662024-04-17T17:13:50.587Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.187755908Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"client":"0","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49672024-04-17T17:13:50.587Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.187884446Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"client":"2","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49682024-04-17T17:13:50.587Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.188188024Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"client":"1","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49692024-04-17T17:13:50.587Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.188230477Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"client":"1","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49702024-04-17T17:13:50.587Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.188262159Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49712024-04-17T17:13:50.587Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.188293961Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"client":"0","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49722024-04-17T17:13:50.587Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.188310662Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"client":"0","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49732024-04-17T17:13:50.588Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.188324262Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49742024-04-17T17:13:50.588Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.188417798Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"client":"2","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49752024-04-17T17:13:50.588Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.18844837Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"client":"2","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49762024-04-17T17:13:50.588Z{"msg":"[0]R flush_numbers[0..12]: [79, 78, 78, 80, 80, 82, 78, 75, 75, 76, 80, 78]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.188463451Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"":"downstairs","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49772024-04-17T17:13:50.588Z{"msg":"[0]R generation[0..12]: [44, 43, 43, 44, 44, 45, 43, 40, 40, 41, 44, 43]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.188484152Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"":"downstairs","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49782024-04-17T17:13:50.588Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.188498893Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"":"downstairs","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49792024-04-17T17:13:50.588Z{"msg":"[1]R flush_numbers[0..12]: [79, 78, 78, 80, 80, 82, 78, 75, 75, 76, 80, 78]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.188519114Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"":"downstairs","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49802024-04-17T17:13:50.588Z{"msg":"[1]R generation[0..12]: [44, 43, 43, 44, 44, 45, 43, 40, 40, 41, 44, 43]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.188533515Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"":"downstairs","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49812024-04-17T17:13:50.588Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.188554296Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"":"downstairs","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49822024-04-17T17:13:50.588Z{"msg":"[2]R flush_numbers[0..12]: [79, 78, 78, 80, 80, 82, 78, 75, 75, 76, 80, 78]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.188567997Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"":"downstairs","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49832024-04-17T17:13:50.588Z{"msg":"[2]R generation[0..12]: [44, 43, 43, 44, 44, 45, 43, 40, 40, 41, 44, 43]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
49842024-04-17T17:13:50.588Z,"time":"2024-04-17T17:13:49.188582048Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"":"downstairs","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49852024-04-17T17:13:50.588Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.188596089Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"":"downstairs","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49862024-04-17T17:13:50.588Z{"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.18861548Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"":"downstairs","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49872024-04-17T17:13:50.588Z{"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.188628881Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"":"downstairs","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49882024-04-17T17:13:50.588Z{"msg":"Next flush: 83","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.188648152Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"":"downstairs","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49892024-04-17T17:13:50.588Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.188661023Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"":"downstairs","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49902024-04-17T17:13:50.588Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.188673754Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49912024-04-17T17:13:50.588Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.188692325Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"client":"0","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49922024-04-17T17:13:50.588Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.188705185Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"client":"1","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49932024-04-17T17:13:50.588Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.188717956Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"client":"2","":"downstairs","session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49942024-04-17T17:13:50.588Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.188731437Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49952024-04-17T17:13:50.588Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.188743318Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49962024-04-17T17:13:50.589Z{"msg":"28eea101-b598-43d2-9d17-db61d0532868 is now active with session: b52a8930-611e-4371-a1b9-ffc602f44cfb","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.188756259Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49972024-04-17T17:13:50.589Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.188768329Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352,"session_id":"b52a8930-611e-4371-a1b9-ffc602f44cfb"}
49982024-04-17T17:13:50.589Z{"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:49.18878413Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352}
49992024-04-17T17:13:50.589ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
50002024-04-17T17:13:50.593ZLoading write count information from file "/tmp/test_repair_verify.data"
50012024-04-17T17:13:50.593ZRun Repair workload
50022024-04-17T17:13:50.593Z01/30 Write block 357 len 12288 data: 5 5 5
50032024-04-17T17:13:50.593Z02/30 Read block 506 len 24576
50042024-04-17T17:13:50.593Z03/30 Write block 309 len 20480 data: 3 4 4 6 6
50052024-04-17T17:13:50.593Z04/30 Write block 64 len 28672 data: 6 5 5 5 4 4 4
50062024-04-17T17:13:50.593Z05/30 Write block 424 len 24576 data: 4 4 4 4 5 6
50072024-04-17T17:13:50.593Z06/30 Write block 173 len 20480 data: 4 2 4 4 3
50082024-04-17T17:13:50.593Z07/30 Write block 459 len 28672 data: 6 6 5 5 6 7 6
50092024-04-17T17:13:50.593Z08/30 Read block 118 len 40960
50102024-04-17T17:13:50.593Z09/30 Write block 134 len 28672 data: 7 5 5 4 4 4 3
50112024-04-17T17:13:50.594Z10/30 Read block 258 len 16384
50122024-04-17T17:13:50.604Z11/30 Read block 33 len 16384
50132024-04-17T17:13:50.604Z12/30 Read block 218 len 12288
50142024-04-17T17:13:50.604Z13/30 Write block 87 len 20480 data: 6 5 7 7 7
50152024-04-17T17:13:50.604Z14/30 Write block 187 len 12288 data: 3 3 3
50162024-04-17T17:13:50.604Z15/30 Read block 308 len 32768
50172024-04-17T17:13:50.604Z16/30 Read block 217 len 24576
50182024-04-17T17:13:50.604Z17/30 Write block 471 len 16384 data: 5 5 4 4
50192024-04-17T17:13:50.605Z18/30 Read block 541 len 36864
50202024-04-17T17:13:50.605Z19/30 Read block 581 len 20480
50212024-04-17T17:13:50.605Z20/30 Write block 80 len 40960 data: 2 2 4 4 4 4 4 7 6 8
50222024-04-17T17:13:50.605Z21/30 Write block 312 len 16384 data: 7 7 7 6
50232024-04-17T17:13:50.605Z22/30 Read block 340 len 36864
50242024-04-17T17:13:50.605Z23/30 Flush
50252024-04-17T17:13:50.608Z24/30 Write block 477 len 40960 data: 5 7 8 8 7 7 9 9 7 8
50262024-04-17T17:13:50.624Z25/30 Flush
50272024-04-17T17:13:50.624Z26/30 Write block 162 len 12288 data: 4 4 3
50282024-04-17T17:13:50.629Z27/30 Write block 389 len 28672 data: 4 4 3 4 4 5 3
50292024-04-17T17:13:50.629Z28/30 Read block 70 len 4096
50302024-04-17T17:13:50.629Z29/30 Write block 368 len 16384 data: 4 4 4 6
50312024-04-17T17:13:50.629Z30/30 Write block 3 len 4096 data: 4
50322024-04-17T17:13:50.629Z----------------------------------------------------------------
50332024-04-17T17:13:50.629Z Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:30
50342024-04-17T17:13:50.629ZStates: Active Active Active
50352024-04-17T17:13:50.629ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
50362024-04-17T17:13:50.629Z 1 Acked 1000 Write 3 Sent Done Done false
50372024-04-17T17:13:50.629Z 2 Acked 1001 Read 6 Sent Done Done false
50382024-04-17T17:13:50.629Z 3 Acked 1002 Write 5 Sent Done Done false
50392024-04-17T17:13:50.629Z 4 Acked 1003 Write 7 Sent Done Done false
50402024-04-17T17:13:50.629Z 5 Acked 1004 Write 6 Sent Done Done false
50412024-04-17T17:13:50.629Z 6 Acked 1005 Write 5 Sent Done Done false
50422024-04-17T17:13:50.629Z 7 Acked 1006 Write 7 Sent Done Done false
50432024-04-17T17:13:50.629Z 8 Acked 1007 Read 10 Sent Done Done false
50442024-04-17T17:13:50.629Z 9 Acked 1008 Write 7 Sent Done Done false
50452024-04-17T17:13:50.629Z 10 Acked 1009 Read 4 Sent Done Done false
50462024-04-17T17:13:50.629Z 11 Acked 1010 Read 4 Sent Done Done false
50472024-04-17T17:13:50.629Z 12 Acked 1011 Read 3 Sent Done Done false
50482024-04-17T17:13:50.629Z 13 Acked 1012 Write 5 Sent Done Done false
50492024-04-17T17:13:50.630Z 14 Acked 1013 Write 3 Sent Done Done false
50502024-04-17T17:13:50.630Z 15 Acked 1014 Read 8 Sent Done Done false
50512024-04-17T17:13:50.630Z 16 Acked 1015 Read 6 Sent Done Done false
50522024-04-17T17:13:50.630Z 17 Acked 1016 Write 4 Sent Done Done false
50532024-04-17T17:13:50.630Z 18 Acked 1017 Read 9 Sent Done Done false
50542024-04-17T17:13:50.630Z 19 Acked 1018 Read 5 Sent Done Done false
50552024-04-17T17:13:50.630Z 20 Acked 1019 Write 10 Sent Done Done false
50562024-04-17T17:13:50.630Z 21 Acked 1020 Write 4 Sent Done Done false
50572024-04-17T17:13:50.630Z 22 Acked 1021 Read 9 Sent Done Done false
50582024-04-17T17:13:50.630Z 23 Acked 1022 Flush 0 Sent Done Done false
50592024-04-17T17:13:50.630Z 24 Acked 1023 Write 10 Sent Done Done false
50602024-04-17T17:13:50.630Z 25 Acked 1024 Flush 0 Sent Done Done false
50612024-04-17T17:13:50.630Z 26 Acked 1025 Write 3 Sent Done Done false
50622024-04-17T17:13:50.630Z 27 Acked 1026 Write 7 Sent Done Done false
50632024-04-17T17:13:50.630Z 28 Acked 1027 Read 1 Sent Done Done false
50642024-04-17T17:13:50.630Z 29 Acked 1028 Write 4 Sent Done Done false
50652024-04-17T17:13:50.630Z 30 Acked 1029 Write 1 Sent Sent Sent false
50662024-04-17T17:13:50.630Z STATES DS:0 DS:1 DS:2 TOTAL
50672024-04-17T17:13:50.630Z New 0 0 0 0
50682024-04-17T17:13:50.630Z Sent 30 1 1 32
50692024-04-17T17:13:50.630Z Done 0 29 29 58
50702024-04-17T17:13:50.630Z Skipped 0 0 0 0
50712024-04-17T17:13:50.630Z Error 0 0 0 0
50722024-04-17T17:13:50.630ZLast Flush: 0 1024 1024
50732024-04-17T17:13:50.631ZDownstairs last five completed:
50742024-04-17T17:13:50.631ZUpstairs last five completed: 30 29 28 27 26
50752024-04-17T17:13:50.631Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:13:49.223576699Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1352}
50762024-04-17T17:13:50.631ZWrote out file "/tmp/test_repair_verify.data"
50772024-04-17T17:13:50.631Z
50782024-04-17T17:13:50.631ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
50792024-04-17T17:13:50.639ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
50802024-04-17T17:13:50.639Z 0 000-019 44 44 44 79 79 79 F T T <---
50812024-04-17T17:13:50.644Z 1 020-039 43 43 43 78 78 78 F F F
50822024-04-17T17:13:50.644Z 2 040-059 43 43 43 78 78 78 F F F
50832024-04-17T17:13:50.644Z 3 060-079 44 46 46 80 83 83 F F F <---
50842024-04-17T17:13:50.644Z 4 080-099 44 46 46 80 83 83 F F F <---
50852024-04-17T17:13:50.644Z 5 100-119 45 45 45 82 82 82 F F F
50862024-04-17T17:13:50.644Z 6 120-139 43 46 46 78 83 83 F F F <---
50872024-04-17T17:13:50.644Z 7 140-159 40 46 46 75 83 83 F F F <---
50882024-04-17T17:13:50.644Z 8 160-179 40 46 46 75 83 83 F T T <---
50892024-04-17T17:13:50.644Z 9 180-199 41 46 46 76 83 83 F F F <---
50902024-04-17T17:13:50.644Z 10 200-219 44 44 44 80 80 80 F F F
50912024-04-17T17:13:50.644Z 11 220-239 43 43 43 78 78 78 F F F
50922024-04-17T17:13:50.644Z 12 240-259 45 45 45 82 82 82 F F F
50932024-04-17T17:13:50.644Z 13 260-279 39 39 39 74 74 74 F F F
50942024-04-17T17:13:50.644Z 14 280-299 37 37 37 71 71 71 F F F
50952024-04-17T17:13:50.644Z 15 300-319 43 46 46 78 83 83 F F F <---
50962024-04-17T17:13:50.644Z 16 320-339 45 45 45 82 82 82 F F F
50972024-04-17T17:13:50.645Z 17 340-359 36 46 46 69 83 83 F F F <---
50982024-04-17T17:13:50.645Z 18 360-379 41 41 41 76 76 76 F T T <---
50992024-04-17T17:13:50.645Z 19 380-399 44 44 44 81 81 81 F T T <---
51002024-04-17T17:13:50.645Z 20 400-419 44 44 44 81 81 81 F F F
51012024-04-17T17:13:50.645Z 21 420-439 43 46 46 78 83 83 F F F <---
51022024-04-17T17:13:50.645Z 22 440-459 45 46 46 82 83 83 F F F <---
51032024-04-17T17:13:50.645Z 23 460-479 44 46 46 79 84 84 F F F <---
51042024-04-17T17:13:50.645Z 24 480-499 44 46 46 79 84 84 F F F <---
51052024-04-17T17:13:50.645Z 25 500-519 43 43 43 78 78 78 F F F
51062024-04-17T17:13:50.645Z 26 520-539 44 44 44 81 81 81 F F F
51072024-04-17T17:13:50.645Z 27 540-559 45 45 45 82 82 82 F F F
51082024-04-17T17:13:50.645Z 28 560-579 45 45 45 82 82 82 F F F
51092024-04-17T17:13:50.645Z 29 580-599 41 41 41 76 76 76 F F F
51102024-04-17T17:13:50.645ZMax gen: 46, Max flush: 84
51112024-04-17T17:13:50.645ZError: Difference in extent metadata found!
51122024-04-17T17:13:50.645ZA Difference in extent metadata is expected here
51132024-04-17T17:13:50.645ZOn loop 23
51142024-04-17T17:13:50.645Z
51152024-04-17T17:13:50.649ZVerifying data now
51162024-04-17T17:13:55.329ZNo change after verify
51172024-04-17T17:13:55.329ZLoop: 23 Downstairs dump after verify (and repair):
51182024-04-17T17:13:55.344ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51192024-04-17T17:13:55.344Z 0 000-019 47 47 47 85 85 85 F F F
51202024-04-17T17:13:55.348Z 1 020-039 43 43 43 78 78 78 F F F
51212024-04-17T17:13:55.348Z 2 040-059 43 43 43 78 78 78 F F F
51222024-04-17T17:13:55.348Z 3 060-079 46 46 46 83 83 83 F F F
51232024-04-17T17:13:55.348Z 4 080-099 46 46 46 83 83 83 F F F
51242024-04-17T17:13:55.348Z 5 100-119 45 45 45 82 82 82 F F F
51252024-04-17T17:13:55.348Z 6 120-139 46 46 46 83 83 83 F F F
51262024-04-17T17:13:55.348Z 7 140-159 46 46 46 83 83 83 F F F
51272024-04-17T17:13:55.348Z 8 160-179 47 47 47 85 85 85 F F F
51282024-04-17T17:13:55.348Z 9 180-199 46 46 46 83 83 83 F F F
51292024-04-17T17:13:55.348Z 10 200-219 44 44 44 80 80 80 F F F
51302024-04-17T17:13:55.348Z 11 220-239 43 43 43 78 78 78 F F F
51312024-04-17T17:13:55.348Z 12 240-259 45 45 45 82 82 82 F F F
51322024-04-17T17:13:55.348Z 13 260-279 39 39 39 74 74 74 F F F
51332024-04-17T17:13:55.348Z 14 280-299 37 37 37 71 71 71 F F F
51342024-04-17T17:13:55.348Z 15 300-319 46 46 46 83 83 83 F F F
51352024-04-17T17:13:55.348Z 16 320-339 45 45 45 82 82 82 F F F
51362024-04-17T17:13:55.348Z 17 340-359 46 46 46 83 83 83 F F F
51372024-04-17T17:13:55.348Z 18 360-379 47 47 47 85 85 85 F F F
51382024-04-17T17:13:55.348Z 19 380-399 47 47 47 85 85 85 F F F
51392024-04-17T17:13:55.348Z 20 400-419 44 44 44 81 81 81 F F F
51402024-04-17T17:13:55.348Z 21 420-439 46 46 46 83 83 83 F F F
51412024-04-17T17:13:55.349Z 22 440-459 46 46 46 83 83 83 F F F
51422024-04-17T17:13:55.349Z 23 460-479 46 46 46 84 84 84 F F F
51432024-04-17T17:13:55.349Z 24 480-499 46 46 46 84 84 84 F F F
51442024-04-17T17:13:55.349Z 25 500-519 43 43 43 78 78 78 F F F
51452024-04-17T17:13:55.349Z 26 520-539 44 44 44 81 81 81 F F F
51462024-04-17T17:13:55.349Z 27 540-559 45 45 45 82 82 82 F F F
51472024-04-17T17:13:55.349Z 28 560-579 45 45 45 82 82 82 F F F
51482024-04-17T17:13:55.349Z 29 580-599 41 41 41 76 76 76 F F F
51492024-04-17T17:13:55.349ZMax gen: 47, Max flush: 85
51502024-04-17T17:13:55.349Z
51512024-04-17T17:13:55.349ZBegin loop 24
51522024-04-17T17:13:55.349ZDownstairs to restart: 1
51532024-04-17T17:13:55.368Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.975055003Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360}
51542024-04-17T17:13:55.368Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.975313039Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360}
51552024-04-17T17:13:55.375Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.975344891Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360}
51562024-04-17T17:13:55.375Z{"msg":"Crucible 9ca35ec8-8340-4912-9a6d-ac2b5fda8d82 has session id: 95ea3c58-41e4-486a-99af-79c06abf1717","v":0,"name":"crucible","level":30Crucible runtime is spawned
51572024-04-17T17:13:55.375Z,"time":"2024-04-17T17:13:53.975376313Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51582024-04-17T17:13:55.375Z{"msg":"Upstairs opts: Upstairs UUID: 9ca35ec8-8340-4912-9a6d-ac2b5fda8d82, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.975485959Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51592024-04-17T17:13:55.375Z{"msg":"Crucible stats registered with UUID: 9ca35ec8-8340-4912-9a6d-ac2b5fda8d82","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.975523471Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51602024-04-17T17:13:55.375Z{"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.975546803Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360}
51612024-04-17T17:13:55.375Z{"msg":"9ca35ec8-8340-4912-9a6d-ac2b5fda8d82 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.975573865Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51622024-04-17T17:13:55.375Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.975612157Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"client":"0","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51632024-04-17T17:13:55.375Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.975637148Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"client":"0","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51642024-04-17T17:13:55.375Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.97566804Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"client":"1","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51652024-04-17T17:13:55.375Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.975703012Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"client":"1","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51662024-04-17T17:13:55.375Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.975726754Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"client":"2","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51672024-04-17T17:13:55.375Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.975751885Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"client":"2","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51682024-04-17T17:13:55.376Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.975776157Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51692024-04-17T17:13:55.376Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.975820089Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51702024-04-17T17:13:55.376Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.975848351Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51712024-04-17T17:13:55.376Z{"msg":"ds_connection connected from Ok(127.0.0.1:37999)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.975886233Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51722024-04-17T17:13:55.376Z{"msg":"ds_connection connected from Ok(127.0.0.1:50258)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.975912835Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51732024-04-17T17:13:55.376Z{"msg":"ds_connection connected from Ok(127.0.0.1:50352)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.975949417Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51742024-04-17T17:13:55.376Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.976613488Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"client":"0","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51752024-04-17T17:13:55.376Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.97665066Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"client":"0","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51762024-04-17T17:13:55.376Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.976667291Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"client":"2","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51772024-04-17T17:13:55.376Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.976693342Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"client":"2","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51782024-04-17T17:13:55.376Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.976711794Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"client":"1","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51792024-04-17T17:13:55.376Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.976733055Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"client":"1","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51802024-04-17T17:13:55.376Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.977259727Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"client":"0","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51812024-04-17T17:13:55.376Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.977302099Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"client":"2","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51822024-04-17T17:13:55.376Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.977336921Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"client":"1","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51832024-04-17T17:13:55.376Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.97780951Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"client":"0","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51842024-04-17T17:13:55.376Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.977865684Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"client":"0","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51852024-04-17T17:13:55.376Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.977903546Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51862024-04-17T17:13:55.376Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.977927077Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"client":"1","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51872024-04-17T17:13:55.376Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.977955349Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"client":"1","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51882024-04-17T17:13:55.376Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.97797911Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51892024-04-17T17:13:55.376Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.978004482Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"client":"2","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51902024-04-17T17:13:55.376Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.978028573Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"client":"2","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51912024-04-17T17:13:55.377Z{"msg":"[0]R flush_numbers[0..12]: [85, 78, 78, 83, 83, 82, 83, 83, 85, 83, 80, 78]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.978056385Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"":"downstairs","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51922024-04-17T17:13:55.377Z{"msg":"[0]R generation[0..12]: [47, 43, 43, 46, 46, 45, 46, 46, 47, 46, 44, 43]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.978080746Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"":"downstairs","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51932024-04-17T17:13:55.377Z{"msg":"Wait for a query_work_queue command to finish before sending IO
51942024-04-17T17:13:55.377Z[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.978111738Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"":"downstairs","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51952024-04-17T17:13:55.377Z{"msg":"[1]R flush_numbers[0..12]: [85, 78, 78, 83, 83, 82, 83, 83, 85, 83, 80, 78]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.97813799Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"":"downstairs","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51962024-04-17T17:13:55.377Z{"msg":"[1]R generation[0..12]: [47, 43, 43, 46, 46, 45, 46, 46, 47, 46, 44, 43]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.978169502Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"":"downstairs","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51972024-04-17T17:13:55.377Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.978195053Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"":"downstairs","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51982024-04-17T17:13:55.377Z{"msg":"[2]R flush_numbers[0..12]: [85, 78, 78, 83, 83, 82, 83, 83, 85, 83, 80, 78]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.978222385Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"":"downstairs","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
51992024-04-17T17:13:55.377Z{"msg":"[2]R generation[0..12]: [47, 43, 43, 46, 46, 45, 46, 46, 47, 46, 44, 43]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.978246376Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"":"downstairs","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
52002024-04-17T17:13:55.377Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.978274308Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"":"downstairs","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
52012024-04-17T17:13:55.377Z{"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.97829757Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"":"downstairs","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
52022024-04-17T17:13:55.377Z{"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.978321391Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"":"downstairs","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
52032024-04-17T17:13:55.377Z{"msg":"Next flush: 86","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.978351883Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"":"downstairs","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
52042024-04-17T17:13:55.377Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.978375064Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"":"downstairs","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
52052024-04-17T17:13:55.377Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.978402166Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
52062024-04-17T17:13:55.377ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
52072024-04-17T17:13:55.377Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.978426147Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"client":"0","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
52082024-04-17T17:13:55.377Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.978453339Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"client":"1","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
52092024-04-17T17:13:55.377Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.97847605Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"client":"2","":"downstairs","session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
52102024-04-17T17:13:55.377Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.978502842Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
52112024-04-17T17:13:55.377Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.978524443Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
52122024-04-17T17:13:55.377Z{"msg":"9ca35ec8-8340-4912-9a6d-ac2b5fda8d82 is now active with session: 95ea3c58-41e4-486a-99af-79c06abf1717","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.978552075Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
52132024-04-17T17:13:55.378Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.978573266Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360,"session_id":"95ea3c58-41e4-486a-99af-79c06abf1717"}
52142024-04-17T17:13:55.378Z{"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:53.978598988Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360}
52152024-04-17T17:13:55.378ZLoading write count information from file "/tmp/test_repair_verify.data"
52162024-04-17T17:13:55.378ZRun Repair workload
52172024-04-17T17:13:55.382Z01/30 Write block 514 len 36864 data: 6 6 4 4 5 4 4 4 5
52182024-04-17T17:13:55.382Z02/30 Read block 520 len 20480
52192024-04-17T17:13:55.382Z03/30 Read block 546 len 12288
52202024-04-17T17:13:55.382Z04/30 Read block 491 len 4096
52212024-04-17T17:13:55.382Z05/30 Read block 586 len 16384
52222024-04-17T17:13:55.382Z06/30 Write block 219 len 8192 data: 4 4
52232024-04-17T17:13:55.382Z07/30 Read block 18 len 20480
52242024-04-17T17:13:55.382Z08/30 Read block 90 len 28672
52252024-04-17T17:13:55.382Z09/30 Read block 268 len 20480
52262024-04-17T17:13:55.382Z10/30 Read block 19 len 16384
52272024-04-17T17:13:55.382Z11/30 Flush
52282024-04-17T17:13:55.386Z12/30 Flush
52292024-04-17T17:13:55.386Z13/30 Write block 20 len 20480 data: 3 5 6 6 7
52302024-04-17T17:13:55.386Z14/30 Read block 162 len 4096
52312024-04-17T17:13:55.386Z15/30 Read block 112 len 24576
52322024-04-17T17:13:55.386Z16/30 Read block 100 len 32768
52332024-04-17T17:13:55.389Z17/30 Write block 423 len 36864 data: 4 5 5 5 5 6 7 4 3
52342024-04-17T17:13:55.389Z18/30 Read block 361 len 12288
52352024-04-17T17:13:55.389Z19/30 Write block 547 len 16384 data: 3 4 4 4
52362024-04-17T17:13:55.389Z20/30 Read block 253 len 4096
52372024-04-17T17:13:55.393Z21/30 Write block 478 len 24576 data: 8 9 9 8 8 10
52382024-04-17T17:13:55.393Z22/30 Write block 144 len 12288 data: 4 4 4
52392024-04-17T17:13:55.393Z23/30 Write block 588 len 20480 data: 3 3 4 3 5
52402024-04-17T17:13:55.393Z24/30 Read block 59 len 8192
52412024-04-17T17:13:55.396Z25/30 Write block 119 len 36864 data: 5 5 4 3 4 4 3 3 3
52422024-04-17T17:13:55.397Z26/30 Read block 367 len 16384
52432024-04-17T17:13:55.397Z27/30 Read block 112 len 12288
52442024-04-17T17:13:55.397Z28/30 Write block 186 len 32768 data: 2 4 4 4 3 3 4 4
52452024-04-17T17:13:55.401Z29/30 Write block 243 len 24576 data: 4 5 5 6 6 5
52462024-04-17T17:13:55.401Z30/30 Write block 235 len 20480 data: 5 3 3 2 4
52472024-04-17T17:13:55.401Z----------------------------------------------------------------
52482024-04-17T17:13:55.401Z Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30
52492024-04-17T17:13:55.401ZStates: Active Active Active
52502024-04-17T17:13:55.401ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
52512024-04-17T17:13:55.401Z 1 Acked 1000 Write 9 Done Sent Done false
52522024-04-17T17:13:55.401Z 2 Acked 1001 Read 5 Done Sent Done false
52532024-04-17T17:13:55.401Z 3 Acked 1002 Read 3 Done Sent Done false
52542024-04-17T17:13:55.401Z 4 Acked 1003 Read 1 Done Sent Done false
52552024-04-17T17:13:55.401Z 5 Acked 1004 Read 4 Done Sent Done false
52562024-04-17T17:13:55.401Z 6 Acked 1005 Write 2 Done Sent Done false
52572024-04-17T17:13:55.401Z 7 Acked 1006 Read 5 Done Sent Done false
52582024-04-17T17:13:55.401Z 8 Acked 1007 Read 7 Done Sent Done false
52592024-04-17T17:13:55.401Z 9 Acked 1008 Read 5 Done Sent Done false
52602024-04-17T17:13:55.401Z 10 Acked 1009 Read 4 Done Sent Done false
52612024-04-17T17:13:55.401Z 11 Acked 1010 Flush 0 Done Sent Done false
52622024-04-17T17:13:55.401Z 12 Acked 1011 Flush 0 Done Sent Done false
52632024-04-17T17:13:55.401Z 13 Acked 1012 Write 5 Done Sent Done false
52642024-04-17T17:13:55.401Z 14 Acked 1013 Read 1 Done Sent Done false
52652024-04-17T17:13:55.401Z 15 Acked 1014 Read 6 Done Sent Done false
52662024-04-17T17:13:55.401Z 16 Acked 1015 Read 8 Done Sent Done false
52672024-04-17T17:13:55.402Z 17 Acked 1016 Write 9 Done Sent Done false
52682024-04-17T17:13:55.402Z 18 Acked 1017 Read 3 Done Sent Done false
52692024-04-17T17:13:55.402Z 19 Acked 1018 Write 4 Done Sent Done false
52702024-04-17T17:13:55.402Z 20 Acked 1019 Read 1 Done Sent Done false
52712024-04-17T17:13:55.402Z 21 Acked 1020 Write 6 Done Sent Done false
52722024-04-17T17:13:55.402Z 22 Acked 1021 Write 3 Done Sent Done false
52732024-04-17T17:13:55.402Z 23 Acked 1022 Write 5 Done Sent Done false
52742024-04-17T17:13:55.402Z 24 Acked 1023 Read 2 Done Sent Done false
52752024-04-17T17:13:55.402Z 25 Acked 1024 Write 9 Done Sent Done false
52762024-04-17T17:13:55.402Z 26 Acked 1025 Read 4 Done Sent Done false
52772024-04-17T17:13:55.402Z 27 Acked 1026 Read 3 Done Sent Done false
52782024-04-17T17:13:55.402Z 28 Acked 1027 Write 8 Done Sent Done false
52792024-04-17T17:13:55.402Z 29 Acked 1028 Write 6 Done Sent Done false
52802024-04-17T17:13:55.402Z 30 Acked 1029 Write 5 Sent Sent Sent false
52812024-04-17T17:13:55.402Z STATES DS:0 DS:1 DS:2 TOTAL
52822024-04-17T17:13:55.402Z New 0 0 0 0
52832024-04-17T17:13:55.402Z Sent 1 30 1 32
52842024-04-17T17:13:55.402Z Done 29 0 29 58
52852024-04-17T17:13:55.402Z Skipped 0 0 0 0
52862024-04-17T17:13:55.402Z Error 0 0 0 0
52872024-04-17T17:13:55.402ZLast Flush: 1011 0 1011
52882024-04-17T17:13:55.402ZDownstairs last five completed:
52892024-04-17T17:13:55.402ZUpstairs last five completed: 30 29 28 27 26
52902024-04-17T17:13:55.402Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:13:54.006865091Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1360}
52912024-04-17T17:13:55.403ZWrote out file "/tmp/test_repair_verify.data"
52922024-04-17T17:13:55.406Z
52932024-04-17T17:13:55.412ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
52942024-04-17T17:13:55.425ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
52952024-04-17T17:13:55.425Z 0 000-019 47 47 47 85 85 85 F F F
52962024-04-17T17:13:55.425Z 1 020-039 43 43 43 78 78 78 T F T <---
52972024-04-17T17:13:55.425Z 2 040-059 43 43 43 78 78 78 F F F
52982024-04-17T17:13:55.425Z 3 060-079 46 46 46 83 83 83 F F F
52992024-04-17T17:13:55.425Z 4 080-099 46 46 46 83 83 83 F F F
53002024-04-17T17:13:55.425Z 5 100-119 45 45 45 82 82 82 T F T <---
53012024-04-17T17:13:55.425Z 6 120-139 46 46 46 83 83 83 T F T <---
53022024-04-17T17:13:55.425Z 7 140-159 46 46 46 83 83 83 T F T <---
53032024-04-17T17:13:55.425Z 8 160-179 47 47 47 85 85 85 F F F
53042024-04-17T17:13:55.426Z 9 180-199 46 46 46 83 83 83 T F T <---
53052024-04-17T17:13:55.426Z 10 200-219 48 44 48 86 80 86 F F F <---
53062024-04-17T17:13:55.426Z 11 220-239 48 43 48 86 78 86 T F T <---
53072024-04-17T17:13:55.426Z 12 240-259 45 45 45 82 82 82 T F T <---
53082024-04-17T17:13:55.429Z 13 260-279 39 39 39 74 74 74 F F F
53092024-04-17T17:13:55.429Z 14 280-299 37 37 37 71 71 71 F F F
53102024-04-17T17:13:55.429Z 15 300-319 46 46 46 83 83 83 F F F
53112024-04-17T17:13:55.429Z 16 320-339 45 45 45 82 82 82 F F F
53122024-04-17T17:13:55.430Z 17 340-359 46 46 46 83 83 83 F F F
53132024-04-17T17:13:55.430Z 18 360-379 47 47 47 85 85 85 F F F
53142024-04-17T17:13:55.430Z 19 380-399 47 47 47 85 85 85 F F F
53152024-04-17T17:13:55.430Z 20 400-419 44 44 44 81 81 81 F F F
53162024-04-17T17:13:55.430Z 21 420-439 46 46 46 83 83 83 T F T <---
53172024-04-17T17:13:55.430Z 22 440-459 46 46 46 83 83 83 F F F
53182024-04-17T17:13:55.430Z 23 460-479 46 46 46 84 84 84 T F T <---
53192024-04-17T17:13:55.430Z 24 480-499 46 46 46 84 84 84 T F T <---
53202024-04-17T17:13:55.430Z 25 500-519 48 43 48 86 78 86 F F F <---
53212024-04-17T17:13:55.430Z 26 520-539 48 44 48 86 81 86 F F F <---
53222024-04-17T17:13:55.430Z 27 540-559 45 45 45 82 82 82 T F T <---
53232024-04-17T17:13:55.430Z 28 560-579 45 45 45 82 82 82 F F F
53242024-04-17T17:13:55.430Z 29 580-599 41 41 41 76 76 76 T F T <---
53252024-04-17T17:13:55.430ZMax gen: 48, Max flush: 86
53262024-04-17T17:13:55.430ZError: Difference in extent metadata found!
53272024-04-17T17:13:55.430ZA Difference in extent metadata is expected here
53282024-04-17T17:13:55.430ZOn loop 24
53292024-04-17T17:13:55.430Z
53302024-04-17T17:13:55.434ZVerifying data now
53312024-04-17T17:14:00.138ZNo change after verify
53322024-04-17T17:14:00.138ZLoop: 24 Downstairs dump after verify (and repair):
53332024-04-17T17:14:00.153ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53342024-04-17T17:14:00.153Z 0 000-019 47 47 47 85 85 85 F F F
53352024-04-17T17:14:00.157Z 1 020-039 49 49 49 87 87 87 F F F
53362024-04-17T17:14:00.157Z 2 040-059 43 43 43 78 78 78 F F F
53372024-04-17T17:14:00.157Z 3 060-079 46 46 46 83 83 83 F F F
53382024-04-17T17:14:00.157Z 4 080-099 46 46 46 83 83 83 F F F
53392024-04-17T17:14:00.157Z 5 100-119 49 49 49 87 87 87 F F F
53402024-04-17T17:14:00.157Z 6 120-139 49 49 49 87 87 87 F F F
53412024-04-17T17:14:00.157Z 7 140-159 49 49 49 87 87 87 F F F
53422024-04-17T17:14:00.157Z 8 160-179 47 47 47 85 85 85 F F F
53432024-04-17T17:14:00.157Z 9 180-199 49 49 49 87 87 87 F F F
53442024-04-17T17:14:00.157Z 10 200-219 48 48 48 86 86 86 F F F
53452024-04-17T17:14:00.157Z 11 220-239 49 49 49 87 87 87 F F F
53462024-04-17T17:14:00.157Z 12 240-259 49 49 49 87 87 87 F F F
53472024-04-17T17:14:00.157Z 13 260-279 39 39 39 74 74 74 F F F
53482024-04-17T17:14:00.157Z 14 280-299 37 37 37 71 71 71 F F F
53492024-04-17T17:14:00.157Z 15 300-319 46 46 46 83 83 83 F F F
53502024-04-17T17:14:00.157Z 16 320-339 45 45 45 82 82 82 F F F
53512024-04-17T17:14:00.157Z 17 340-359 46 46 46 83 83 83 F F F
53522024-04-17T17:14:00.157Z 18 360-379 47 47 47 85 85 85 F F F
53532024-04-17T17:14:00.157Z 19 380-399 47 47 47 85 85 85 F F F
53542024-04-17T17:14:00.158Z 20 400-419 44 44 44 81 81 81 F F F
53552024-04-17T17:14:00.158Z 21 420-439 49 49 49 87 87 87 F F F
53562024-04-17T17:14:00.158Z 22 440-459 46 46 46 83 83 83 F F F
53572024-04-17T17:14:00.158Z 23 460-479 49 49 49 87 87 87 F F F
53582024-04-17T17:14:00.158Z 24 480-499 49 49 49 87 87 87 F F F
53592024-04-17T17:14:00.158Z 25 500-519 48 48 48 86 86 86 F F F
53602024-04-17T17:14:00.158Z 26 520-539 48 48 48 86 86 86 F F F
53612024-04-17T17:14:00.158Z 27 540-559 49 49 49 87 87 87 F F F
53622024-04-17T17:14:00.158Z 28 560-579 45 45 45 82 82 82 F F F
53632024-04-17T17:14:00.158Z 29 580-599 49 49 49 87 87 87 F F F
53642024-04-17T17:14:00.158ZMax gen: 49, Max flush: 87
53652024-04-17T17:14:00.158Z
53662024-04-17T17:14:00.158ZBegin loop 25
53672024-04-17T17:14:00.158ZDownstairs to restart: 0
53682024-04-17T17:14:00.175Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.782724743Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368}
53692024-04-17T17:14:00.175Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.78300438Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368}
53702024-04-17T17:14:00.183Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.783043312Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368}
53712024-04-17T17:14:00.183Z{"msg":"Crucible 3a937fbd-a3ad-433f-a875-4c7f5a6190f2 has session id: ea2273cf-21ff-49e4-9cc2-42d0d8832f48","v":0,"name":"crucible","level":30Crucible runtime is spawned
53722024-04-17T17:14:00.183Z,"time":"2024-04-17T17:13:58.783068844Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
53732024-04-17T17:14:00.183Z{"msg":"Upstairs opts: Upstairs UUID: 3a937fbd-a3ad-433f-a875-4c7f5a6190f2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.783114416Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
53742024-04-17T17:14:00.183Z{"msg":"Crucible stats registered with UUID: 3a937fbd-a3ad-433f-a875-4c7f5a6190f2","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.783149259Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
53752024-04-17T17:14:00.183Z{"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.78316403Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368}
53762024-04-17T17:14:00.183Z{"msg":"3a937fbd-a3ad-433f-a875-4c7f5a6190f2 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.783246275Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
53772024-04-17T17:14:00.183Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.783299378Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"client":"0","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
53782024-04-17T17:14:00.183Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.783357831Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"client":"0","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
53792024-04-17T17:14:00.183Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.783444917Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"client":"1","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
53802024-04-17T17:14:00.183Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.783508731Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"client":"1","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
53812024-04-17T17:14:00.183Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.783540482Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"client":"2","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
53822024-04-17T17:14:00.183Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.783567584Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"client":"2","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
53832024-04-17T17:14:00.183Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.783607706Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
53842024-04-17T17:14:00.183Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.783640008Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
53852024-04-17T17:14:00.183Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.78366796Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
53862024-04-17T17:14:00.183Z{"msg":"ds_connection connected from Ok(127.0.0.1:36657)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.783693612Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
53872024-04-17T17:14:00.184Z{"msg":"ds_connection connected from Ok(127.0.0.1:38642)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.783729614Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
53882024-04-17T17:14:00.184Z{"msg":"ds_connection connected from Ok(127.0.0.1:64297)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.783763826Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
53892024-04-17T17:14:00.184Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.78432236Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"client":"2","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
53902024-04-17T17:14:00.184Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.784361242Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"client":"2","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
53912024-04-17T17:14:00.184Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.784414335Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"client":"1","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
53922024-04-17T17:14:00.184Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.784451538Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"client":"1","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
53932024-04-17T17:14:00.184Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.784476069Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"client":"0","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
53942024-04-17T17:14:00.184Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.784501151Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"client":"0","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
53952024-04-17T17:14:00.184Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.784934607Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"client":"1","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
53962024-04-17T17:14:00.184Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.785007381Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"client":"2","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
53972024-04-17T17:14:00.184Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.785040193Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"client":"0","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
53982024-04-17T17:14:00.184Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.78548061Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"client":"1","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
53992024-04-17T17:14:00.184Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.785539254Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"client":"1","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
54002024-04-17T17:14:00.184Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.785596227Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
54012024-04-17T17:14:00.184Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.785632279Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"client":"2","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
54022024-04-17T17:14:00.184Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.78565014Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"client":"2","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
54032024-04-17T17:14:00.184Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.785671622Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
54042024-04-17T17:14:00.184Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.785684212Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"client":"0","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
54052024-04-17T17:14:00.184Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.785704964Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"client":"0","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
54062024-04-17T17:14:00.184Z{"msg":"[0]R flush_numbers[0..12]: [85, 87, 78, 83, 83, 87, 87, 87, 85, 87, 86, 87]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.785719014Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"":"downstairs","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
54072024-04-17T17:14:00.184Z{"msg":"[0]R generation[0..12]: [47, 49, 43, 46, 46, 49, 49, 49, 47, 49, 48, 49]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.785732885Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"":"downstairs","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
54082024-04-17T17:14:00.184Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.785754067Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"":"downstairs","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
54092024-04-17T17:14:00.184Z{"msg":"[1]R flush_numbers[0..12]: [85, 87, 78, 83, 83, 87, 87, 87, 85, 87, 86, 87]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.785767817Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"":"downstairs","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
54102024-04-17T17:14:00.184Z{"msg":"[1]R generation[0..12]: [47, 49, 43, 46, 46, 49, 49, 49, 47, 49, 48, 49]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.785788039Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"":"downstairs","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
54112024-04-17T17:14:00.185Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.785802319Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"":"downstairs","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
54122024-04-17T17:14:00.185Z{"msg":"Wait for a query_work_queue command to finish before sending IO
54132024-04-17T17:14:00.185Z[2]R flush_numbers[0..12]: [85, 87, 78, 83, 83, 87, 87, 87, 85, 87, 86, 87]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.785817951Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"":"downstairs","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
54142024-04-17T17:14:00.185Z{"msg":"[2]R generation[0..12]: [47, 49, 43, 46, 46, 49, 49, 49, 47, 49, 48, 49]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.785832661Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"":"downstairs","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
54152024-04-17T17:14:00.185Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.785853593Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"":"downstairs","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
54162024-04-17T17:14:00.185Z{"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.785866563Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"":"downstairs","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
54172024-04-17T17:14:00.185Z{"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.785879384Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"":"downstairs","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
54182024-04-17T17:14:00.185Z{"msg":"Next flush: 88","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.785904126Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"":"downstairs","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
54192024-04-17T17:14:00.185Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.785935187Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"":"downstairs","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
54202024-04-17T17:14:00.185Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.785950568Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
54212024-04-17T17:14:00.185Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.78597033Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"client":"0","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
54222024-04-17T17:14:00.185Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.785993031Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"client":"1","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
54232024-04-17T17:14:00.185Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.786020163Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"client":"2","":"downstairs","session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
54242024-04-17T17:14:00.185Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.786053375Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
54252024-04-17T17:14:00.185Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.786068346Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
54262024-04-17T17:14:00.185Z{"msg":"3a937fbd-a3ad-433f-a875-4c7f5a6190f2 is now active with session: ea2273cf-21ff-49e4-9cc2-42d0d8832f48","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.786093407Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
54272024-04-17T17:14:00.185ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
54282024-04-17T17:14:00.185Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.786123909Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368,"session_id":"ea2273cf-21ff-49e4-9cc2-42d0d8832f48"}
54292024-04-17T17:14:00.185Z{"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:13:58.78613803Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368}
54302024-04-17T17:14:00.189ZLoading write count information from file "/tmp/test_repair_verify.data"
54312024-04-17T17:14:00.189ZRun Repair workload
54322024-04-17T17:14:00.189Z01/30 Write block 460 len 16384 data: 7 6 6 7
54332024-04-17T17:14:00.189Z02/30 Flush
54342024-04-17T17:14:00.189Z03/30 Write block 573 len 16384 data: 4 4 5 5
54352024-04-17T17:14:00.189Z04/30 Write block 348 len 12288 data: 6 6 7
54362024-04-17T17:14:00.190Z05/30 Read block 526 len 4096
54372024-04-17T17:14:00.190Z06/30 Read block 304 len 4096
54382024-04-17T17:14:00.190Z07/30 Flush
54392024-04-17T17:14:00.190Z08/30 Write block 28 len 28672 data: 4 6 5 7 10 11 11
54402024-04-17T17:14:00.190Z09/30 Write block 230 len 28672 data: 2 2 3 4 4 6 4
54412024-04-17T17:14:00.193Z10/30 Read block 359 len 8192
54422024-04-17T17:14:00.193Z11/30 Write block 264 len 4096 data: 4
54432024-04-17T17:14:00.193Z12/30 Write block 41 len 24576 data: 7 6 5 5 7 6
54442024-04-17T17:14:00.193Z13/30 Write block 0 len 8192 data: 4 4
54452024-04-17T17:14:00.193Z14/30 Read block 229 len 36864
54462024-04-17T17:14:00.200Z15/30 Write block 337 len 32768 data: 7 7 7 6 6 7 7 7
54472024-04-17T17:14:00.200Z16/30 Read block 389 len 20480
54482024-04-17T17:14:00.200Z17/30 Write block 175 len 20480 data: 5 5 4 5 5
54492024-04-17T17:14:00.203Z18/30 Write block 99 len 36864 data: 5 5 5 4 6 6 6 7 7
54502024-04-17T17:14:00.203Z19/30 Write block 568 len 36864 data: 4 5 4 4 5 5 5 6 6
54512024-04-17T17:14:00.203Z20/30 Write block 154 len 40960 data: 4 5 6 6 4 4 4 5 5 5
54522024-04-17T17:14:00.204Z21/30 Write block 236 len 24576 data: 5 4 3 5 5 6
54532024-04-17T17:14:00.204Z22/30 Read block 485 len 4096
54542024-04-17T17:14:00.204Z23/30 Read block 338 len 20480
54552024-04-17T17:14:00.204Z24/30 Read block 408 len 4096
54562024-04-17T17:14:00.208Z25/30 Read block 244 len 36864
54572024-04-17T17:14:00.208Z26/30 Write block 70 len 32768 data: 5 5 4 3 4 3 3 3
54582024-04-17T17:14:00.208Z27/30 Write block 104 len 32768 data: 7 7 8 8 7 6 6 8
54592024-04-17T17:14:00.208Z28/30 Read block 253 len 28672
54602024-04-17T17:14:00.212Z29/30 Read block 322 len 16384
54612024-04-17T17:14:00.213Z30/30 Read block 143 len 20480
54622024-04-17T17:14:00.213Z----------------------------------------------------------------
54632024-04-17T17:14:00.213Z Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:28
54642024-04-17T17:14:00.213ZStates: Active Active Active
54652024-04-17T17:14:00.213ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
54662024-04-17T17:14:00.213Z 3 Acked 1002 Write 4 Sent Done Done false
54672024-04-17T17:14:00.213Z 4 Acked 1003 Write 3 Sent Done Done false
54682024-04-17T17:14:00.213Z 5 Acked 1004 Read 1 Sent Done Done false
54692024-04-17T17:14:00.213Z 6 Acked 1005 Read 1 Sent Done Done false
54702024-04-17T17:14:00.213Z 7 Acked 1006 Flush 0 Sent Done Done false
54712024-04-17T17:14:00.213Z 8 Acked 1007 Write 7 Sent Done Done false
54722024-04-17T17:14:00.213Z 9 Acked 1008 Write 7 Sent Done Done false
54732024-04-17T17:14:00.213Z 10 Acked 1009 Read 2 Sent Done Done false
54742024-04-17T17:14:00.213Z 11 Acked 1010 Write 1 Sent Done Done false
54752024-04-17T17:14:00.213Z 12 Acked 1011 Write 6 Sent Done Done false
54762024-04-17T17:14:00.213Z 13 Acked 1012 Write 2 Sent Done Done false
54772024-04-17T17:14:00.213Z 14 Acked 1013 Read 9 Sent Done Done false
54782024-04-17T17:14:00.213Z 15 Acked 1014 Write 8 Sent Done Done false
54792024-04-17T17:14:00.213Z 16 Acked 1015 Read 5 Sent Done Done false
54802024-04-17T17:14:00.213Z 17 Acked 1016 Write 5 Sent Done Done false
54812024-04-17T17:14:00.213Z 18 Acked 1017 Write 9 Sent Done Done false
54822024-04-17T17:14:00.214Z 19 Acked 1018 Write 9 Sent Done Done false
54832024-04-17T17:14:00.214Z 20 Acked 1019 Write 10 Sent Done Done false
54842024-04-17T17:14:00.214Z 21 Acked 1020 Write 6 Sent Done Done false
54852024-04-17T17:14:00.214Z 22 Acked 1021 Read 1 Sent Done Done false
54862024-04-17T17:14:00.214Z 23 Acked 1022 Read 5 Sent Done Done false
54872024-04-17T17:14:00.214Z 24 Acked 1023 Read 1 Sent Done Done false
54882024-04-17T17:14:00.214Z 25 Acked 1024 Read 9 Sent Done Done false
54892024-04-17T17:14:00.214Z 26 Acked 1025 Write 8 Sent Done Done false
54902024-04-17T17:14:00.214Z 27 Acked 1026 Write 8 Sent Done Done false
54912024-04-17T17:14:00.214Z 28 Acked 1027 Read 7 Sent Done Done false
54922024-04-17T17:14:00.214Z 29 Acked 1028 Read 4 Sent Done Done false
54932024-04-17T17:14:00.214Z 30 Acked 1029 Read 5 Sent Done Done false
54942024-04-17T17:14:00.214Z STATES DS:0 DS:1 DS:2 TOTAL
54952024-04-17T17:14:00.214Z New 0 0 0 0
54962024-04-17T17:14:00.214Z Sent 28 0 0 28
54972024-04-17T17:14:00.214Z Done 0 28 28 56
54982024-04-17T17:14:00.214Z Skipped 0 0 0 0
54992024-04-17T17:14:00.214Z Error 0 0 0 0
55002024-04-17T17:14:00.214ZLast Flush: 1001 1006 1006
55012024-04-17T17:14:00.214ZDownstairs last five completed: 1001 1000
55022024-04-17T17:14:00.214ZUpstairs last five completed: 30 29 28 27 26
55032024-04-17T17:14:00.214Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:13:58.817579995Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1368}
55042024-04-17T17:14:00.214ZWrote out file "/tmp/test_repair_verify.data"
55052024-04-17T17:14:00.218Z
55062024-04-17T17:14:00.218ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
55072024-04-17T17:14:00.237ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
55082024-04-17T17:14:00.237Z 0 000-019 47 47 47 85 85 85 F T T <---
55092024-04-17T17:14:00.237Z 1 020-039 49 49 49 87 87 87 F T T <---
55102024-04-17T17:14:00.237Z 2 040-059 43 43 43 78 78 78 F T T <---
55112024-04-17T17:14:00.237Z 3 060-079 46 46 46 83 83 83 F T T <---
55122024-04-17T17:14:00.237Z 4 080-099 46 46 46 83 83 83 F T T <---
55132024-04-17T17:14:00.237Z 5 100-119 49 49 49 87 87 87 F T T <---
55142024-04-17T17:14:00.237Z 6 120-139 49 49 49 87 87 87 F F F
55152024-04-17T17:14:00.241Z 7 140-159 49 49 49 87 87 87 F T T <---
55162024-04-17T17:14:00.241Z 8 160-179 47 47 47 85 85 85 F T T <---
55172024-04-17T17:14:00.241Z 9 180-199 49 49 49 87 87 87 F F F
55182024-04-17T17:14:00.241Z 10 200-219 48 48 48 86 86 86 F F F
55192024-04-17T17:14:00.241Z 11 220-239 49 49 49 87 87 87 F T T <---
55202024-04-17T17:14:00.241Z 12 240-259 49 49 49 87 87 87 F T T <---
55212024-04-17T17:14:00.241Z 13 260-279 39 39 39 74 74 74 F T T <---
55222024-04-17T17:14:00.241Z 14 280-299 37 37 37 71 71 71 F F F
55232024-04-17T17:14:00.241Z 15 300-319 46 46 46 83 83 83 F F F
55242024-04-17T17:14:00.241Z 16 320-339 45 45 45 82 82 82 F T T <---
55252024-04-17T17:14:00.241Z 17 340-359 46 50 50 83 89 89 F T T <---
55262024-04-17T17:14:00.241Z 18 360-379 47 47 47 85 85 85 F F F
55272024-04-17T17:14:00.241Z 19 380-399 47 47 47 85 85 85 F F F
55282024-04-17T17:14:00.241Z 20 400-419 44 44 44 81 81 81 F F F
55292024-04-17T17:14:00.241Z 21 420-439 49 49 49 87 87 87 F F F
55302024-04-17T17:14:00.241Z 22 440-459 46 46 46 83 83 83 F F F
55312024-04-17T17:14:00.241Z 23 460-479 50 50 50 88 88 88 F F F
55322024-04-17T17:14:00.242Z 24 480-499 49 49 49 87 87 87 F F F
55332024-04-17T17:14:00.242Z 25 500-519 48 48 48 86 86 86 F F F
55342024-04-17T17:14:00.242Z 26 520-539 48 48 48 86 86 86 F F F
55352024-04-17T17:14:00.242Z 27 540-559 49 49 49 87 87 87 F F F
55362024-04-17T17:14:00.242Z 28 560-579 45 50 50 82 89 89 F T T <---
55372024-04-17T17:14:00.242Z 29 580-599 49 49 49 87 87 87 F F F
55382024-04-17T17:14:00.242ZMax gen: 50, Max flush: 89
55392024-04-17T17:14:00.242ZError: Difference in extent metadata found!
55402024-04-17T17:14:00.242ZA Difference in extent metadata is expected here
55412024-04-17T17:14:00.242ZOn loop 25
55422024-04-17T17:14:00.242Z
55432024-04-17T17:14:00.246ZVerifying data now
55442024-04-17T17:14:04.903ZNo change after verify
55452024-04-17T17:14:04.903ZLoop: 25 Downstairs dump after verify (and repair):
55462024-04-17T17:14:04.918ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
55472024-04-17T17:14:04.918Z 0 000-019 51 51 51 90 90 90 F F F
55482024-04-17T17:14:04.918Z 1 020-039 51 51 51 90 90 90 F F F
55492024-04-17T17:14:04.918Z 2 040-059 51 51 51 90 90 90 F F F
55502024-04-17T17:14:04.918Z 3 060-079 51 51 51 90 90 90 F F F
55512024-04-17T17:14:04.922Z 4 080-099 51 51 51 90 90 90 F F F
55522024-04-17T17:14:04.922Z 5 100-119 51 51 51 90 90 90 F F F
55532024-04-17T17:14:04.922Z 6 120-139 49 49 49 87 87 87 F F F
55542024-04-17T17:14:04.922Z 7 140-159 51 51 51 90 90 90 F F F
55552024-04-17T17:14:04.922Z 8 160-179 51 51 51 90 90 90 F F F
55562024-04-17T17:14:04.922Z 9 180-199 49 49 49 87 87 87 F F F
55572024-04-17T17:14:04.922Z 10 200-219 48 48 48 86 86 86 F F F
55582024-04-17T17:14:04.922Z 11 220-239 51 51 51 90 90 90 F F F
55592024-04-17T17:14:04.922Z 12 240-259 51 51 51 90 90 90 F F F
55602024-04-17T17:14:04.922Z 13 260-279 51 51 51 90 90 90 F F F
55612024-04-17T17:14:04.922Z 14 280-299 37 37 37 71 71 71 F F F
55622024-04-17T17:14:04.922Z 15 300-319 46 46 46 83 83 83 F F F
55632024-04-17T17:14:04.922Z 16 320-339 51 51 51 90 90 90 F F F
55642024-04-17T17:14:04.922Z 17 340-359 51 51 51 90 90 90 F F F
55652024-04-17T17:14:04.922Z 18 360-379 47 47 47 85 85 85 F F F
55662024-04-17T17:14:04.923Z 19 380-399 47 47 47 85 85 85 F F F
55672024-04-17T17:14:04.923Z 20 400-419 44 44 44 81 81 81 F F F
55682024-04-17T17:14:04.923Z 21 420-439 49 49 49 87 87 87 F F F
55692024-04-17T17:14:04.923Z 22 440-459 46 46 46 83 83 83 F F F
55702024-04-17T17:14:04.923Z 23 460-479 50 50 50 88 88 88 F F F
55712024-04-17T17:14:04.923Z 24 480-499 49 49 49 87 87 87 F F F
55722024-04-17T17:14:04.923Z 25 500-519 48 48 48 86 86 86 F F F
55732024-04-17T17:14:04.923Z 26 520-539 48 48 48 86 86 86 F F F
55742024-04-17T17:14:04.923Z 27 540-559 49 49 49 87 87 87 F F F
55752024-04-17T17:14:04.923Z 28 560-579 51 51 51 90 90 90 F F F
55762024-04-17T17:14:04.923Z 29 580-599 49 49 49 87 87 87 F F F
55772024-04-17T17:14:04.923ZMax gen: 51, Max flush: 90
55782024-04-17T17:14:04.923Z
55792024-04-17T17:14:04.923ZBegin loop 26
55802024-04-17T17:14:04.923ZDownstairs to restart: 2
55812024-04-17T17:14:04.941Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.548205956Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376}
55822024-04-17T17:14:04.941Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.548486923Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376}
55832024-04-17T17:14:04.942Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.548524916Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376}
55842024-04-17T17:14:04.950Z{"msg":"Crucible 26d22709-c3e2-415a-8f78-024cef97ddaa has session id: bd6769c5-3e94-4940-99d0-6c407677a63a","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.548547927Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
55852024-04-17T17:14:04.950Z{Crucible runtime is spawned
55862024-04-17T17:14:04.950Z"msg":"Upstairs opts: Upstairs UUID: 26d22709-c3e2-415a-8f78-024cef97ddaa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.54859317Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
55872024-04-17T17:14:04.950Z{"msg":"Crucible stats registered with UUID: 26d22709-c3e2-415a-8f78-024cef97ddaa","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.548616961Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
55882024-04-17T17:14:04.950Z{"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.548652203Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376}
55892024-04-17T17:14:04.950Z{"msg":"26d22709-c3e2-415a-8f78-024cef97ddaa active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.548772111Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
55902024-04-17T17:14:04.950Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.548824974Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"client":"0","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
55912024-04-17T17:14:04.950Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.548859826Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"client":"0","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
55922024-04-17T17:14:04.950Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.548906469Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"client":"1","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
55932024-04-17T17:14:04.951Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.548956942Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"client":"1","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
55942024-04-17T17:14:04.951Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.549024896Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"client":"2","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
55952024-04-17T17:14:04.951Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.549072539Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"client":"2","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
55962024-04-17T17:14:04.951Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.549120002Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
55972024-04-17T17:14:04.951Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.549162364Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
55982024-04-17T17:14:04.951Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.549215818Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
55992024-04-17T17:14:04.951Z{"msg":"ds_connection connected from Ok(127.0.0.1:53818)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.549267841Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56002024-04-17T17:14:04.951Z{"msg":"ds_connection connected from Ok(127.0.0.1:45832)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.549327354Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56012024-04-17T17:14:04.951Z{"msg":"ds_connection connected from Ok(127.0.0.1:64239)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.549368617Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56022024-04-17T17:14:04.951Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.549882368Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"client":"0","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56032024-04-17T17:14:04.951Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.549932101Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"client":"0","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56042024-04-17T17:14:04.951Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.549963243Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"client":"1","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56052024-04-17T17:14:04.951Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.549990885Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"client":"1","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56062024-04-17T17:14:04.951Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.550101121Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"client":"2","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56072024-04-17T17:14:04.951Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.550151464Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"client":"2","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56082024-04-17T17:14:04.951Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.550538768Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"client":"1","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56092024-04-17T17:14:04.951Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.55057683Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"client":"0","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56102024-04-17T17:14:04.951Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.550838446Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"client":"2","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56112024-04-17T17:14:04.951Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.55108058Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"client":"1","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56122024-04-17T17:14:04.951Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.551132174Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"client":"1","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56132024-04-17T17:14:04.951Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.551176366Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56142024-04-17T17:14:04.951Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.551209738Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"client":"0","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56152024-04-17T17:14:04.951Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.551226059Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"client":"0","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56162024-04-17T17:14:04.952Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.551247781Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56172024-04-17T17:14:04.952Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.551365988Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"client":"2","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56182024-04-17T17:14:04.952Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.551414271Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"client":"2","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56192024-04-17T17:14:04.952Z{"msg":"[0]R flush_numbers[0..12]: [90, 90, 90, 90, 90, 90, 87, 90, 90, 87, 86, 90]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.551432512Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"":"downstairs","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56202024-04-17T17:14:04.952Z{"msg":"[0]R generation[0..12]: [51, 51, 51, 51, 51, 51, 49, 51, 51, 49, 48, 51]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.551447363Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"":"downstairs","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56212024-04-17T17:14:04.952Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.551462104Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"":"downstairs","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56222024-04-17T17:14:04.952Z{"msg":"[1]R flush_numbers[0..12]: [90, 90, 90, 90, 90, 90, 87, 90, 90, 87, 86, 90]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.551485145Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"":"downstairs","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56232024-04-17T17:14:04.952Z{"msg":"[1]R generation[0..12]: [51, 51, 51, 51, 51, 51, 49, 51, 51, 49, 48, 51]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.551499446Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"":"downstairs","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56242024-04-17T17:14:04.952Z{"msg"Wait for a query_work_queue command to finish before sending IO
56252024-04-17T17:14:04.952Z:"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.551533788Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"":"downstairs","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56262024-04-17T17:14:04.952Z{"msg":"[2]R flush_numbers[0..12]: [90, 90, 90, 90, 90, 90, 87, 90, 90, 87, 86, 90]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.551548479Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"":"downstairs","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56272024-04-17T17:14:04.952Z{"msg":"[2]R generation[0..12]: [51, 51, 51, 51, 51, 51, 49, 51, 51, 49, 48, 51]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.55156384Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"":"downstairs","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56282024-04-17T17:14:04.952Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.551578071Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"":"downstairs","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56292024-04-17T17:14:04.952Z{"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.551598512Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"":"downstairs","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56302024-04-17T17:14:04.952Z{"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.551612033Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"":"downstairs","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56312024-04-17T17:14:04.952Z{"msg":"Next flush: 91","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.551624924Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"":"downstairs","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56322024-04-17T17:14:04.952Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.551645015Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"":"downstairs","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56332024-04-17T17:14:04.952Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.551658646Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56342024-04-17T17:14:04.952Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.551671106Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"client":"0","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56352024-04-17T17:14:04.952Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.551691037Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"client":"1","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56362024-04-17T17:14:04.952Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.551704198Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"client":"2","":"downstairs","session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56372024-04-17T17:14:04.952Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.551717079Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56382024-04-17T17:14:04.952Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.55173597Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56392024-04-17T17:14:04.953Z{"msg":"26d22709-c3e2-415a-8f78-024cef97ddaa is now active with session: bd6769c5-3e94-4940-99d0-6c407677a63a","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.551748871Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56402024-04-17T17:14:04.953Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:03.551760722Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376,"session_id":"bd6769c5-3e94-4940-99d0-6c407677a63a"}
56412024-04-17T17:14:04.953Z{"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
56422024-04-17T17:14:04.953Z,"time":"2024-04-17T17:14:03.551783253Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376}
56432024-04-17T17:14:04.953ZLoading write count information from file "/tmp/test_repair_verify.data"
56442024-04-17T17:14:04.953ZRun Repair workload
56452024-04-17T17:14:04.960Z01/30 Write block 500 len 28672 data: 6 6 6 5 5 3 2
56462024-04-17T17:14:04.960Z02/30 Read block 157 len 16384
56472024-04-17T17:14:04.960Z03/30 Write block 531 len 12288 data: 6 5 5
56482024-04-17T17:14:04.960Z04/30 Write block 36 len 36864 data: 7 6 5 5 6 8 7 6 6
56492024-04-17T17:14:04.960Z05/30 Read block 188 len 40960
56502024-04-17T17:14:04.960Z06/30 Flush
56512024-04-17T17:14:04.960Z07/30 Write block 47 len 32768 data: 4 3 4 2 3 3 3 4
56522024-04-17T17:14:04.960Z08/30 Read block 18 len 16384
56532024-04-17T17:14:04.960Z09/30 Write block 241 len 20480 data: 7 5 5 6 6
56542024-04-17T17:14:04.964Z10/30 Write block 342 len 32768 data: 8 8 8 9 9 8 7 7
56552024-04-17T17:14:04.964Z11/30 Read block 409 len 32768
56562024-04-17T17:14:04.964Z12/30 Read block 387 len 40960
56572024-04-17T17:14:04.964Z13/30 Write block 504 len 24576 data: 6 4 3 5 5 4
56582024-04-17T17:14:04.964Z14/30 Write block 29 len 20480 data: 7 6 8 11 12
56592024-04-17T17:14:04.964Z15/30 Flush
56602024-04-17T17:14:04.968Z16/30 Flush
56612024-04-17T17:14:04.968Z17/30 Write block 438 len 24576 data: 6 7 7 7 8 8
56622024-04-17T17:14:04.968Z18/30 Write block 505 len 4096 data: 5
56632024-04-17T17:14:04.968Z19/30 Write block 250 len 32768 data: 5 6 8 7 6 4 6 6
56642024-04-17T17:14:04.971Z20/30 Flush
56652024-04-17T17:14:04.971Z21/30 Flush
56662024-04-17T17:14:04.975Z22/30 Write block 560 len 4096 data: 5
56672024-04-17T17:14:04.975Z23/30 Write block 557 len 20480 data: 5 6 6 6 4
56682024-04-17T17:14:04.975Z24/30 Write block 413 len 20480 data: 9 9 8 7 6
56692024-04-17T17:14:04.975Z25/30 Read block 304 len 28672
56702024-04-17T17:14:04.975Z26/30 Read block 308 len 16384
56712024-04-17T17:14:04.975Z27/30 Read block 210 len 40960
56722024-04-17T17:14:04.980Z28/30 Read block 237 len 4096
56732024-04-17T17:14:04.980Z29/30 Read block 66 len 16384
56742024-04-17T17:14:04.980Z30/30 Write block 111 len 32768 data: 9 7 5 4 4 4 4 3
56752024-04-17T17:14:04.980Z----------------------------------------------------------------
56762024-04-17T17:14:04.980Z Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30
56772024-04-17T17:14:04.980ZStates: Active Active Active
56782024-04-17T17:14:04.980ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
56792024-04-17T17:14:04.980Z 1 Acked 1000 Write 7 Done Done Done false
56802024-04-17T17:14:04.980Z 2 Acked 1001 Read 4 Done Done Done false
56812024-04-17T17:14:04.980Z 3 Acked 1002 Write 3 Done Done Done false
56822024-04-17T17:14:04.980Z 4 Acked 1003 Write 9 Done Done Sent false
56832024-04-17T17:14:04.980Z 5 Acked 1004 Read 10 Done Done Sent false
56842024-04-17T17:14:04.980Z 6 Acked 1005 Flush 0 Done Done Sent false
56852024-04-17T17:14:04.980Z 7 Acked 1006 Write 8 Done Done Sent false
56862024-04-17T17:14:04.980Z 8 Acked 1007 Read 4 Done Done Sent false
56872024-04-17T17:14:04.980Z 9 Acked 1008 Write 5 Done Done Sent false
56882024-04-17T17:14:04.980Z 10 Acked 1009 Write 8 Done Done Sent false
56892024-04-17T17:14:04.980Z 11 Acked 1010 Read 8 Done Done Sent false
56902024-04-17T17:14:04.980Z 12 Acked 1011 Read 10 Done Done Sent false
56912024-04-17T17:14:04.981Z 13 Acked 1012 Write 6 Done Done Sent false
56922024-04-17T17:14:04.981Z 14 Acked 1013 Write 5 Done Done Sent false
56932024-04-17T17:14:04.981Z 15 Acked 1014 Flush 0 Done Done Sent false
56942024-04-17T17:14:04.981Z 16 Acked 1015 Flush 0 Done Done Sent false
56952024-04-17T17:14:04.981Z 17 Acked 1016 Write 6 Done Done Sent false
56962024-04-17T17:14:04.981Z 18 Acked 1017 Write 1 Done Done Sent false
56972024-04-17T17:14:04.981Z 19 Acked 1018 Write 8 Done Done Sent false
56982024-04-17T17:14:04.981Z 20 Acked 1019 Flush 0 Done Done Sent false
56992024-04-17T17:14:04.981Z 21 Acked 1020 Flush 0 Done Done Sent false
57002024-04-17T17:14:04.981Z 22 Acked 1021 Write 1 Done Done Sent false
57012024-04-17T17:14:04.981Z 23 Acked 1022 Write 5 Done Done Sent false
57022024-04-17T17:14:04.981Z 24 Acked 1023 Write 5 Done Done Sent false
57032024-04-17T17:14:04.981Z 25 Acked 1024 Read 7 Done Done Sent false
57042024-04-17T17:14:04.981Z 26 Acked 1025 Read 4 Done Done Sent false
57052024-04-17T17:14:04.981Z 27 Acked 1026 Read 10 Done Done Sent false
57062024-04-17T17:14:04.981Z 28 Acked 1027 Read 1 Done Done Sent false
57072024-04-17T17:14:04.981Z 29 Acked 1028 Read 4 Done Done Sent false
57082024-04-17T17:14:04.981Z 30 Acked 1029 Write 8 Sent Sent Sent false
57092024-04-17T17:14:04.981Z STATES DS:0 DS:1 DS:2 TOTAL
57102024-04-17T17:14:04.981Z New 0 0 0 0
57112024-04-17T17:14:04.981Z Sent 1 1 27 29
57122024-04-17T17:14:04.981Z Done 29 29 3 61
57132024-04-17T17:14:04.981Z Skipped 0 0 0 0
57142024-04-17T17:14:04.981Z Error 0 0 0 0
57152024-04-17T17:14:04.982ZLast Flush: 1020 1020 0
57162024-04-17T17:14:04.982ZDownstairs last five completed:
57172024-04-17T17:14:04.982ZUpstairs last five completed: 30 29 28 27 26
57182024-04-17T17:14:04.982Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:14:03.586138865Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1376}
57192024-04-17T17:14:04.985ZWrote out file "/tmp/test_repair_verify.data"
57202024-04-17T17:14:04.985Z
57212024-04-17T17:14:04.989ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
57222024-04-17T17:14:05.002ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
57232024-04-17T17:14:05.002Z 0 000-019 51 51 51 90 90 90 F F F
57242024-04-17T17:14:05.007Z 1 020-039 52 52 51 92 92 90 F F F <---
57252024-04-17T17:14:05.007Z 2 040-059 52 52 51 92 92 90 F F F <---
57262024-04-17T17:14:05.007Z 3 060-079 51 51 51 90 90 90 F F F
57272024-04-17T17:14:05.007Z 4 080-099 51 51 51 90 90 90 F F F
57282024-04-17T17:14:05.007Z 5 100-119 51 51 51 90 90 90 T T F <---
57292024-04-17T17:14:05.007Z 6 120-139 49 49 49 87 87 87 F F F
57302024-04-17T17:14:05.007Z 7 140-159 51 51 51 90 90 90 F F F
57312024-04-17T17:14:05.007Z 8 160-179 51 51 51 90 90 90 F F F
57322024-04-17T17:14:05.007Z 9 180-199 49 49 49 87 87 87 F F F
57332024-04-17T17:14:05.007Z 10 200-219 48 48 48 86 86 86 F F F
57342024-04-17T17:14:05.007Z 11 220-239 51 51 51 90 90 90 F F F
57352024-04-17T17:14:05.007Z 12 240-259 52 52 51 94 94 90 F F F <---
57362024-04-17T17:14:05.007Z 13 260-279 51 51 51 90 90 90 F F F
57372024-04-17T17:14:05.007Z 14 280-299 37 37 37 71 71 71 F F F
57382024-04-17T17:14:05.008Z 15 300-319 46 46 46 83 83 83 F F F
57392024-04-17T17:14:05.008Z 16 320-339 51 51 51 90 90 90 F F F
57402024-04-17T17:14:05.008Z 17 340-359 52 52 51 92 92 90 F F F <---
57412024-04-17T17:14:05.008Z 18 360-379 47 47 47 85 85 85 F F F
57422024-04-17T17:14:05.008Z 19 380-399 47 47 47 85 85 85 F F F
57432024-04-17T17:14:05.008Z 20 400-419 44 44 44 81 81 81 T T F <---
57442024-04-17T17:14:05.008Z 21 420-439 52 52 49 94 94 87 F F F <---
57452024-04-17T17:14:05.008Z 22 440-459 52 52 46 94 94 83 F F F <---
57462024-04-17T17:14:05.008Z 23 460-479 50 50 50 88 88 88 F F F
57472024-04-17T17:14:05.008Z 24 480-499 49 49 49 87 87 87 F F F
57482024-04-17T17:14:05.008Z 25 500-519 52 52 48 94 94 86 F F T <---
57492024-04-17T17:14:05.008Z 26 520-539 52 52 48 91 91 86 F F T <---
57502024-04-17T17:14:05.008Z 27 540-559 49 49 49 87 87 87 T T F <---
57512024-04-17T17:14:05.008Z 28 560-579 51 51 51 90 90 90 T T F <---
57522024-04-17T17:14:05.008Z 29 580-599 49 49 49 87 87 87 F F F
57532024-04-17T17:14:05.008ZMax gen: 52, Max flush: 94
57542024-04-17T17:14:05.008ZError: Difference in extent metadata found!
57552024-04-17T17:14:05.008ZA Difference in extent metadata is expected here
57562024-04-17T17:14:05.008ZOn loop 26
57572024-04-17T17:14:05.008Z
57582024-04-17T17:14:05.016ZVerifying data now
57592024-04-17T17:14:15.569ZNo change after verify
57602024-04-17T17:14:15.569ZLoop: 26 Downstairs dump after verify (and repair):
57612024-04-17T17:14:15.584ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
57622024-04-17T17:14:15.584Z 0 000-019 51 51 51 90 90 90 F F F
57632024-04-17T17:14:15.584Z 1 020-039 52 52 52 92 92 92 F F F
57642024-04-17T17:14:15.584Z 2 040-059 52 52 52 92 92 92 F F F
57652024-04-17T17:14:15.584Z 3 060-079 51 51 51 90 90 90 F F F
57662024-04-17T17:14:15.584Z 4 080-099 51 51 51 90 90 90 F F F
57672024-04-17T17:14:15.584Z 5 100-119 53 53 53 95 95 95 F F F
57682024-04-17T17:14:15.585Z 6 120-139 49 49 49 87 87 87 F F F
57692024-04-17T17:14:15.585Z 7 140-159 51 51 51 90 90 90 F F F
57702024-04-17T17:14:15.585Z 8 160-179 51 51 51 90 90 90 F F F
57712024-04-17T17:14:15.585Z 9 180-199 49 49 49 87 87 87 F F F
57722024-04-17T17:14:15.585Z 10 200-219 48 48 48 86 86 86 F F F
57732024-04-17T17:14:15.585Z 11 220-239 51 51 51 90 90 90 F F F
57742024-04-17T17:14:15.585Z 12 240-259 52 52 52 94 94 94 F F F
57752024-04-17T17:14:15.585Z 13 260-279 51 51 51 90 90 90 F F F
57762024-04-17T17:14:15.585Z 14 280-299 37 37 37 71 71 71 F F F
57772024-04-17T17:14:15.588Z 15 300-319 46 46 46 83 83 83 F F F
57782024-04-17T17:14:15.589Z 16 320-339 51 51 51 90 90 90 F F F
57792024-04-17T17:14:15.589Z 17 340-359 52 52 52 92 92 92 F F F
57802024-04-17T17:14:15.589Z 18 360-379 47 47 47 85 85 85 F F F
57812024-04-17T17:14:15.589Z 19 380-399 47 47 47 85 85 85 F F F
57822024-04-17T17:14:15.589Z 20 400-419 53 53 53 95 95 95 F F F
57832024-04-17T17:14:15.589Z 21 420-439 52 52 52 94 94 94 F F F
57842024-04-17T17:14:15.589Z 22 440-459 52 52 52 94 94 94 F F F
57852024-04-17T17:14:15.589Z 23 460-479 50 50 50 88 88 88 F F F
57862024-04-17T17:14:15.589Z 24 480-499 49 49 49 87 87 87 F F F
57872024-04-17T17:14:15.589Z 25 500-519 52 52 52 94 94 94 F F F
57882024-04-17T17:14:15.589Z 26 520-539 52 52 52 91 91 91 F F F
57892024-04-17T17:14:15.589Z 27 540-559 53 53 53 95 95 95 F F F
57902024-04-17T17:14:15.589Z 28 560-579 53 53 53 95 95 95 F F F
57912024-04-17T17:14:15.589Z 29 580-599 49 49 49 87 87 87 F F F
57922024-04-17T17:14:15.589ZMax gen: 53, Max flush: 95
57932024-04-17T17:14:15.589Z
57942024-04-17T17:14:15.589ZBegin loop 27
57952024-04-17T17:14:15.589ZDownstairs to restart: 0
57962024-04-17T17:14:15.612Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.2131854Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384}
57972024-04-17T17:14:15.625Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.21351489ZCrucible runtime is spawned
57982024-04-17T17:14:15.625Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384}
57992024-04-17T17:14:15.625Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.213565573Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384}
58002024-04-17T17:14:15.625Z{"msg":"Crucible 4bb72701-b7b4-4ba6-b1ed-78065126c696 has session id: 804658d8-b3ff-4f86-80bd-a7126f239f2a","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.213604976Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58012024-04-17T17:14:15.625Z{"msg":"Upstairs opts: Upstairs UUID: 4bb72701-b7b4-4ba6-b1ed-78065126c696, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.213652459Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58022024-04-17T17:14:15.625Z{"msg":"Crucible stats registered with UUID: 4bb72701-b7b4-4ba6-b1ed-78065126c696","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.21368095Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58032024-04-17T17:14:15.625Z{"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.213698301Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384}
58042024-04-17T17:14:15.625Z{"msg":"4bb72701-b7b4-4ba6-b1ed-78065126c696 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.213730183Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58052024-04-17T17:14:15.625Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.213760245Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"client":"0","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58062024-04-17T17:14:15.625Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.213818549Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"client":"0","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58072024-04-17T17:14:15.625Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.213931115Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"client":"1","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58082024-04-17T17:14:15.625Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.213976808Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"client":"1","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58092024-04-17T17:14:15.625Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.21401479Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"client":"2","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58102024-04-17T17:14:15.625Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.214058013Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"client":"2","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58112024-04-17T17:14:15.626Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.214092965Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58122024-04-17T17:14:15.626Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.214140598Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58132024-04-17T17:14:15.626Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.2141713Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58142024-04-17T17:14:15.626Z{"msg":"ds_connection connected from Ok(127.0.0.1:33881)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.214198972Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58152024-04-17T17:14:15.626Z{"msg":"ds_connection connected from Ok(127.0.0.1:47235)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.214237534Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58162024-04-17T17:14:15.626Z{"msg":"ds_connection connected from Ok(127.0.0.1:46399)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.214262265Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58172024-04-17T17:14:15.626Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.214866152Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"client":"2","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58182024-04-17T17:14:15.626Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.214902114Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"client":"2","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58192024-04-17T17:14:15.626Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.214925486Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"client":"1","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58202024-04-17T17:14:15.626Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.214964848Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"client":"1","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58212024-04-17T17:14:15.626Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.215008221Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"client":"0","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58222024-04-17T17:14:15.626Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.215037612Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"client":"0","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58232024-04-17T17:14:15.626Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.215454128Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"client":"2","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58242024-04-17T17:14:15.626Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.21549189Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"client":"1","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58252024-04-17T17:14:15.626Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.215510451Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"client":"0","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58262024-04-17T17:14:15.626Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.216036243Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"client":"2","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58272024-04-17T17:14:15.626Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.216073515Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"client":"2","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58282024-04-17T17:14:15.626Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.216089186Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58292024-04-17T17:14:15.626Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.216101327Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"client":"0","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58302024-04-17T17:14:15.626Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.216115338Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"client":"0","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58312024-04-17T17:14:15.626Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.216128358Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58322024-04-17T17:14:15.626Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.21614758Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"client":"1","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58332024-04-17T17:14:15.627Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.216161251Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"client":"1","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58342024-04-17T17:14:15.627Z{"msg":"[0]R flush_numbers[0..12]: [90, 92, 92, 90, 90, 95, 87, 90, 90, 87, 86, 90]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.216182282Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"":"downstairs","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58352024-04-17T17:14:15.627Z{"msg":"[0]R generation[0..12]: [51, 52, 52, 51, 51, 53, 49, 51, 51, 49, 48, 51]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.216196663Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"":"downstairs","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58362024-04-17T17:14:15.627Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.216210743Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"":"downstairs","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58372024-04-17T17:14:15.627Z{"msg":"[1]R flush_numbers[0..12]: [90, 92, 92, 90, 90, 95, 87, 90, 90, 87, 86, 90]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.216224274Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"":"downstairs","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58382024-04-17T17:14:15.627Z{"msg":"[1]R generation[0..12]: [51, 52, 52, 51, 51, 53, 49, 51, 51, 49, 48, 51]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.216241585Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"":"downstairs","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58392024-04-17T17:14:15.627ZWait for a query_work_queue command to finish before sending IO
58402024-04-17T17:14:15.627Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.216273187Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"":"downstairs","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58412024-04-17T17:14:15.627Z{"msg":"[2]R flush_numbers[0..12]: [90, 92, 92, 90, 90, 95, 87, 90, 90, 87, 86, 90]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.216288388Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"":"downstairs","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58422024-04-17T17:14:15.627Z{"msg":"[2]R generation[0..12]: [51, 52, 52, 51, 51, 53, 49, 51, 51, 49, 48, 51]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.216302039Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"":"downstairs","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58432024-04-17T17:14:15.627Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.21632303Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"":"downstairs","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58442024-04-17T17:14:15.627Z{"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.216337021Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"":"downstairs","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58452024-04-17T17:14:15.627Z{"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.216350492Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"":"downstairs","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58462024-04-17T17:14:15.627Z{"msg":"Next flush: 96","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.216363163Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"":"downstairs","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58472024-04-17T17:14:15.627Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.216375824Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"":"downstairs","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58482024-04-17T17:14:15.627Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.216399555Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58492024-04-17T17:14:15.627Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.216418166Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"client":"0","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58502024-04-17T17:14:15.627Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.216450818Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"client":"1","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58512024-04-17T17:14:15.627Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.21648597Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"client":"2","":"downstairs","session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58522024-04-17T17:14:15.627Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.216522073Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58532024-04-17T17:14:15.627Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.216557344Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58542024-04-17T17:14:15.627Z{"msg":"4bb72701-b7b4-4ba6-b1ed-78065126c696 is now active with session: 804658d8-b3ff-4f86-80bd-a7126f239f2a","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.216593927Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
58552024-04-17T17:14:15.627Z:1384,"session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58562024-04-17T17:14:15.628Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.216634729Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384,"session_id":"804658d8-b3ff-4f86-80bd-a7126f239f2a"}
58572024-04-17T17:14:15.628Z{"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:14.216660871Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384}
58582024-04-17T17:14:15.628ZLoading write count information from file "/tmp/test_repair_verify.data"
58592024-04-17T17:14:15.628ZRun Repair workload
58602024-04-17T17:14:15.628Z01/30 Write block 127 len 32768 data: 4 3 3 5 6 8 6 8
58612024-04-17T17:14:15.628Z02/30 Write block 214 len 12288 data: 3 4 3
58622024-04-17T17:14:15.628Z03/30 Flush
58632024-04-17T17:14:15.628Z04/30 Write block 498 len 32768 data: 5 6 7 7 7 6 7 6
58642024-04-17T17:14:15.628Z05/30 Read block 153 len 40960
58652024-04-17T17:14:15.628Z06/30 Write block 297 len 24576 data: 2 2 3 3 5 4
58662024-04-17T17:14:15.628Z07/30 Read block 68 len 4096
58672024-04-17T17:14:15.628Z08/30 Flush
58682024-04-17T17:14:15.632Z09/30 Write block 82 len 8192 data: 5 5
58692024-04-17T17:14:15.632Z10/30 Read block 589 len 32768
58702024-04-17T17:14:15.632Z11/30 Read block 344 len 28672
58712024-04-17T17:14:15.632Z12/30 Read block 278 len 8192
58722024-04-17T17:14:15.632Z13/30 Read block 108 len 40960
58732024-04-17T17:14:15.632Z14/30 Read block 199 len 36864
58742024-04-17T17:14:15.632Z15/30 Write block 580 len 28672 data: 6 6 5 5 5 6 6
58752024-04-17T17:14:15.632Z16/30 Read block 458 len 40960
58762024-04-17T17:14:15.633Z17/30 Read block 509 len 32768
58772024-04-17T17:14:15.633Z18/30 Write block 243 len 4096 data: 6
58782024-04-17T17:14:15.633Z19/30 Read block 139 len 24576
58792024-04-17T17:14:15.633Z20/30 Write block 156 len 16384 data: 7 7 5 5
58802024-04-17T17:14:15.636Z21/30 Write block 60 len 20480 data: 5 5 5 6 7
58812024-04-17T17:14:15.636Z22/30 Read block 583 len 4096
58822024-04-17T17:14:15.636Z23/30 Read block 33 len 20480
58832024-04-17T17:14:15.636Z24/30 Write block 317 len 12288 data: 6 6 5
58842024-04-17T17:14:15.636Z25/30 Write block 422 len 32768 data: 5 5 6 6 6 6 7 8
58852024-04-17T17:14:15.643Z26/30 Write block 577 len 12288 data: 5 5 7
58862024-04-17T17:14:15.643Z27/30 Read block 321 len 8192
58872024-04-17T17:14:15.643Z28/30 Read block 470 len 40960
58882024-04-17T17:14:15.643Z29/30 Read block 473 len 4096
58892024-04-17T17:14:15.643Z30/30 Read block 250 len 12288
58902024-04-17T17:14:15.643Z----------------------------------------------------------------
58912024-04-17T17:14:15.644Z Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:27
58922024-04-17T17:14:15.644ZStates: Active Active Active
58932024-04-17T17:14:15.644ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
58942024-04-17T17:14:15.644Z 4 Acked 1003 Write 8 Done Done Done false
58952024-04-17T17:14:15.644Z 5 Acked 1004 Read 10 Done Done Done false
58962024-04-17T17:14:15.644Z 6 Acked 1005 Write 6 Sent Done Done false
58972024-04-17T17:14:15.644Z 7 Acked 1006 Read 1 Sent Done Done false
58982024-04-17T17:14:15.644Z 8 Acked 1007 Flush 0 Sent Done Done false
58992024-04-17T17:14:15.644Z 9 Acked 1008 Write 2 Sent Done Done false
59002024-04-17T17:14:15.644Z 10 Acked 1009 Read 8 Sent Done Done false
59012024-04-17T17:14:15.644Z 11 Acked 1010 Read 7 Sent Done Done false
59022024-04-17T17:14:15.644Z 12 Acked 1011 Read 2 Sent Done Done false
59032024-04-17T17:14:15.644Z 13 Acked 1012 Read 10 Sent Done Done false
59042024-04-17T17:14:15.644Z 14 Acked 1013 Read 9 Sent Done Done false
59052024-04-17T17:14:15.644Z 15 Acked 1014 Write 7 Sent Done Done false
59062024-04-17T17:14:15.644Z 16 Acked 1015 Read 10 Sent Done Done false
59072024-04-17T17:14:15.644Z 17 Acked 1016 Read 8 Sent Done Done false
59082024-04-17T17:14:15.644Z 18 Acked 1017 Write 1 Sent Done Done false
59092024-04-17T17:14:15.644Z 19 Acked 1018 Read 6 Sent Done Done false
59102024-04-17T17:14:15.644Z 20 Acked 1019 Write 4 Sent Done Done false
59112024-04-17T17:14:15.644Z 21 Acked 1020 Write 5 Sent Done Done false
59122024-04-17T17:14:15.645Z 22 Acked 1021 Read 1 Sent Done Done false
59132024-04-17T17:14:15.645Z 23 Acked 1022 Read 5 Sent Done Done false
59142024-04-17T17:14:15.645Z 24 Acked 1023 Write 3 Sent Done Done false
59152024-04-17T17:14:15.645Z 25 Acked 1024 Write 8 Sent Done Done false
59162024-04-17T17:14:15.645Z 26 Acked 1025 Write 3 Sent Done Done false
59172024-04-17T17:14:15.645Z 27 Acked 1026 Read 2 Sent Done Done false
59182024-04-17T17:14:15.645Z 28 Acked 1027 Read 10 Sent Done Done false
59192024-04-17T17:14:15.645Z 29 Acked 1028 Read 1 Sent Done Done false
59202024-04-17T17:14:15.645Z 30 Acked 1029 Read 3 Sent Done Done false
59212024-04-17T17:14:15.645Z STATES DS:0 DS:1 DS:2 TOTAL
59222024-04-17T17:14:15.645Z New 0 0 0 0
59232024-04-17T17:14:15.645Z Sent 25 0 0 25
59242024-04-17T17:14:15.645Z Done 2 27 27 56
59252024-04-17T17:14:15.645Z Skipped 0 0 0 0
59262024-04-17T17:14:15.645Z Error 0 0 0 0
59272024-04-17T17:14:15.645ZLast Flush: 1002 1007 1007
59282024-04-17T17:14:15.645ZDownstairs last five completed: 1002 1001 1000
59292024-04-17T17:14:15.645ZUpstairs last five completed: 30 29 28 27 26
59302024-04-17T17:14:15.645Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:14:14.247422255Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1384}
59312024-04-17T17:14:15.649ZWrote out file "/tmp/test_repair_verify.data"
59322024-04-17T17:14:15.649Z
59332024-04-17T17:14:15.649ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
59342024-04-17T17:14:15.665ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
59352024-04-17T17:14:15.665Z 0 000-019 51 51 51 90 90 90 F F F
59362024-04-17T17:14:15.665Z 1 020-039 52 52 52 92 92 92 F F F
59372024-04-17T17:14:15.665Z 2 040-059 52 52 52 92 92 92 F F F
59382024-04-17T17:14:15.665Z 3 060-079 51 51 51 90 90 90 F T T <---
59392024-04-17T17:14:15.665Z 4 080-099 51 51 51 90 90 90 F T T <---
59402024-04-17T17:14:15.669Z 5 100-119 53 53 53 95 95 95 F F F
59412024-04-17T17:14:15.669Z 6 120-139 54 54 54 96 96 96 F F F
59422024-04-17T17:14:15.669Z 7 140-159 51 51 51 90 90 90 F T T <---
59432024-04-17T17:14:15.669Z 8 160-179 51 51 51 90 90 90 F F F
59442024-04-17T17:14:15.669Z 9 180-199 49 49 49 87 87 87 F F F
59452024-04-17T17:14:15.669Z 10 200-219 54 54 54 96 96 96 F F F
59462024-04-17T17:14:15.669Z 11 220-239 51 51 51 90 90 90 F F F
59472024-04-17T17:14:15.669Z 12 240-259 52 52 52 94 94 94 F T T <---
59482024-04-17T17:14:15.669Z 13 260-279 51 51 51 90 90 90 F F F
59492024-04-17T17:14:15.670Z 14 280-299 37 54 54 71 97 97 F F F <---
59502024-04-17T17:14:15.670Z 15 300-319 46 54 54 83 97 97 F T T <---
59512024-04-17T17:14:15.670Z 16 320-339 51 51 51 90 90 90 F F F
59522024-04-17T17:14:15.670Z 17 340-359 52 52 52 92 92 92 F F F
59532024-04-17T17:14:15.670Z 18 360-379 47 47 47 85 85 85 F F F
59542024-04-17T17:14:15.670Z 19 380-399 47 47 47 85 85 85 F F F
59552024-04-17T17:14:15.670Z 20 400-419 53 53 53 95 95 95 F F F
59562024-04-17T17:14:15.670Z 21 420-439 52 52 52 94 94 94 F T T <---
59572024-04-17T17:14:15.670Z 22 440-459 52 52 52 94 94 94 F F F
59582024-04-17T17:14:15.670Z 23 460-479 50 50 50 88 88 88 F F F
59592024-04-17T17:14:15.670Z 24 480-499 49 54 54 87 97 97 T F F <---
59602024-04-17T17:14:15.670Z 25 500-519 52 54 54 94 97 97 T F F <---
59612024-04-17T17:14:15.670Z 26 520-539 52 52 52 91 91 91 F F F
59622024-04-17T17:14:15.670Z 27 540-559 53 53 53 95 95 95 F F F
59632024-04-17T17:14:15.670Z 28 560-579 53 53 53 95 95 95 F T T <---
59642024-04-17T17:14:15.670Z 29 580-599 49 49 49 87 87 87 F T T <---
59652024-04-17T17:14:15.670ZMax gen: 54, Max flush: 97
59662024-04-17T17:14:15.670ZError: Difference in extent metadata found!
59672024-04-17T17:14:15.670ZA Difference in extent metadata is expected here
59682024-04-17T17:14:15.670ZOn loop 27
59692024-04-17T17:14:15.670Z
59702024-04-17T17:14:15.674ZVerifying data now
59712024-04-17T17:14:30.194ZNo change after verify
59722024-04-17T17:14:30.194ZLoop: 27 Downstairs dump after verify (and repair):
59732024-04-17T17:14:30.208ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
59742024-04-17T17:14:30.208Z 0 000-019 51 51 51 90 90 90 F F F
59752024-04-17T17:14:30.212Z 1 020-039 52 52 52 92 92 92 F F F
59762024-04-17T17:14:30.212Z 2 040-059 52 52 52 92 92 92 F F F
59772024-04-17T17:14:30.212Z 3 060-079 55 55 55 98 98 98 F F F
59782024-04-17T17:14:30.212Z 4 080-099 55 55 55 98 98 98 F F F
59792024-04-17T17:14:30.212Z 5 100-119 53 53 53 95 95 95 F F F
59802024-04-17T17:14:30.212Z 6 120-139 54 54 54 96 96 96 F F F
59812024-04-17T17:14:30.212Z 7 140-159 55 55 55 98 98 98 F F F
59822024-04-17T17:14:30.212Z 8 160-179 51 51 51 90 90 90 F F F
59832024-04-17T17:14:30.212Z 9 180-199 49 49 49 87 87 87 F F F
59842024-04-17T17:14:30.212Z 10 200-219 54 54 54 96 96 96 F F F
59852024-04-17T17:14:30.212Z 11 220-239 51 51 51 90 90 90 F F F
59862024-04-17T17:14:30.212Z 12 240-259 55 55 55 98 98 98 F F F
59872024-04-17T17:14:30.212Z 13 260-279 51 51 51 90 90 90 F F F
59882024-04-17T17:14:30.212Z 14 280-299 54 54 54 97 97 97 F F F
59892024-04-17T17:14:30.212Z 15 300-319 55 55 55 98 98 98 F F F
59902024-04-17T17:14:30.212Z 16 320-339 51 51 51 90 90 90 F F F
59912024-04-17T17:14:30.212Z 17 340-359 52 52 52 92 92 92 F F F
59922024-04-17T17:14:30.212Z 18 360-379 47 47 47 85 85 85 F F F
59932024-04-17T17:14:30.212Z 19 380-399 47 47 47 85 85 85 F F F
59942024-04-17T17:14:30.212Z 20 400-419 53 53 53 95 95 95 F F F
59952024-04-17T17:14:30.212Z 21 420-439 55 55 55 98 98 98 F F F
59962024-04-17T17:14:30.212Z 22 440-459 52 52 52 94 94 94 F F F
59972024-04-17T17:14:30.213Z 23 460-479 50 50 50 88 88 88 F F F
59982024-04-17T17:14:30.213Z 24 480-499 54 54 54 97 97 97 F F F
59992024-04-17T17:14:30.213Z 25 500-519 54 54 54 97 97 97 F F F
60002024-04-17T17:14:30.213Z 26 520-539 52 52 52 91 91 91 F F F
60012024-04-17T17:14:30.213Z 27 540-559 53 53 53 95 95 95 F F F
60022024-04-17T17:14:30.213Z 28 560-579 55 55 55 98 98 98 F F F
60032024-04-17T17:14:30.213Z 29 580-599 55 55 55 98 98 98 F F F
60042024-04-17T17:14:30.213ZMax gen: 55, Max flush: 98
60052024-04-17T17:14:30.217Z
60062024-04-17T17:14:30.217ZBegin loop 28
60072024-04-17T17:14:30.217ZDownstairs to restart: 2
60082024-04-17T17:14:30.231Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.838871592Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392}
60092024-04-17T17:14:30.238Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.83917056Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392}
60102024-04-17T17:14:30.238Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.839212533Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392}
60112024-04-17T17:14:30.238Z{"msg":"Crucible ae1fc25c-acb6-40c6-9789-1650f72af785 has session id: 647d6b28-173b-41f3-82b0-aa9aae7e29a1","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.839239514Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60122024-04-17T17:14:30.238ZCrucible runtime is spawned
60132024-04-17T17:14:30.238Z{"msg":"Upstairs opts: Upstairs UUID: ae1fc25c-acb6-40c6-9789-1650f72af785, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.839299848Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60142024-04-17T17:14:30.238Z{"msg":"Crucible stats registered with UUID: ae1fc25c-acb6-40c6-9789-1650f72af785","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.839341801Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60152024-04-17T17:14:30.238Z{"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.839370332Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392}
60162024-04-17T17:14:30.238Z{"msg":"ae1fc25c-acb6-40c6-9789-1650f72af785 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.839407355Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60172024-04-17T17:14:30.238Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.839435526Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"client":"0","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60182024-04-17T17:14:30.238Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.839474629Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"client":"0","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60192024-04-17T17:14:30.239Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.839506181Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"client":"1","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60202024-04-17T17:14:30.239Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.839540393Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"client":"1","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60212024-04-17T17:14:30.239Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.839573485Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"client":"2","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60222024-04-17T17:14:30.239Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.839604267Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"client":"2","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60232024-04-17T17:14:30.239Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.839636269Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60242024-04-17T17:14:30.239Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.839701793Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60252024-04-17T17:14:30.239Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.839745995Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60262024-04-17T17:14:30.239Z{"msg":"ds_connection connected from Ok(127.0.0.1:37252)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.839788148Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60272024-04-17T17:14:30.239Z{"msg":"ds_connection connected from Ok(127.0.0.1:45652)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.839819409Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60282024-04-17T17:14:30.239Z{"msg":"ds_connection connected from Ok(127.0.0.1:40141)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.839865072Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60292024-04-17T17:14:30.239Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.840558724Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"client":"1","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60302024-04-17T17:14:30.239Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.840603867Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"client":"1","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60312024-04-17T17:14:30.239Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.840630509Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"client":"0","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60322024-04-17T17:14:30.239Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.840678602Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"client":"0","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60332024-04-17T17:14:30.239Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.840742325Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"client":"2","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60342024-04-17T17:14:30.239Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.840769557Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"client":"2","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60352024-04-17T17:14:30.239Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.841200643Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"client":"0","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60362024-04-17T17:14:30.239Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.841243556Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"client":"1","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60372024-04-17T17:14:30.239Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.841363783Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"client":"2","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60382024-04-17T17:14:30.239Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.841759917Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"client":"0","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60392024-04-17T17:14:30.239Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.84180824Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"client":"0","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60402024-04-17T17:14:30.239Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.841883255Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60412024-04-17T17:14:30.239Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.841908566Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"client":"2","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60422024-04-17T17:14:30.240Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.841937988Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"client":"2","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60432024-04-17T17:14:30.240Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.841959429Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60442024-04-17T17:14:30.240Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.841993481Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"client":"1","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60452024-04-17T17:14:30.240Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.842030774Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"client":"1","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60462024-04-17T17:14:30.240Z{"msg":"[0]R flush_numbers[0..12]: [90, 92, 92, 98, 98, 95, 96, 98, 90, 87, 96, 90]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.842056325Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"":"downstairs","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60472024-04-17T17:14:30.240Z{"msg":"[0]R generation[0..12]: [51, 52, 52, 55, 55, 53, 54, 55, 51, 49, 54, 51]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.842090807Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"":"downstairs","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60482024-04-17T17:14:30.240Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.842128029Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"":"downstairs","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60492024-04-17T17:14:30.240Z{"msg":"[1]R flush_numbers[0..12]: [90, 92, 92, 98, 98, 95, 96, 98, 90, 87, 96, 90]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.842163012Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"":"downstairs","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60502024-04-17T17:14:30.240Z{"msg":"[1]R generation[0..12]: [51, 52, 52, 55, 55, 53, 54, 55, 51, 49, 54, 51]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.842196904Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"":"downstairs","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60512024-04-17T17:14:30.240ZWait for a query_work_queue command to finish before sending IO
60522024-04-17T17:14:30.240Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.842224935Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"":"downstairs","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60532024-04-17T17:14:30.240Z{"msg":"[2]R flush_numbers[0..12]: [90, 92, 92, 98, 98, 95, 96, 98, 90, 87, 96, 90]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.842248467Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"":"downstairs","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60542024-04-17T17:14:30.240Z{"msg":"[2]R generation[0..12]: [51, 52, 52, 55, 55, 53, 54, 55, 51, 49, 54, 51]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.842287619Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"":"downstairs","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60552024-04-17T17:14:30.240Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.842328002Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"":"downstairs","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60562024-04-17T17:14:30.240Z{"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.842366924Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"":"downstairs","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60572024-04-17T17:14:30.240Z{"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.842406866Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"":"downstairs","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60582024-04-17T17:14:30.240Z{"msg":"Next flush: 99","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.842443629Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"":"downstairs","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60592024-04-17T17:14:30.240Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.842480491Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"":"downstairs","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60602024-04-17T17:14:30.245Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.842518013Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60612024-04-17T17:14:30.245Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.842557646Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"client":"0","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60622024-04-17T17:14:30.245Z{"msg":"Transition from WaitQuorum to Active","v"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
60632024-04-17T17:14:30.245Z:0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.842599108Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"client":"1","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60642024-04-17T17:14:30.245Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.84262669Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"client":"2","":"downstairs","session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60652024-04-17T17:14:30.245Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.842659862Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60662024-04-17T17:14:30.245Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.842683043Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60672024-04-17T17:14:30.245Z{"msg":"ae1fc25c-acb6-40c6-9789-1650f72af785 is now active with session: 647d6b28-173b-41f3-82b0-aa9aae7e29a1","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.842711105Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60682024-04-17T17:14:30.245Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.842733146Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392,"session_id":"647d6b28-173b-41f3-82b0-aa9aae7e29a1"}
60692024-04-17T17:14:30.245Z{"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:28.842759398Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392}
60702024-04-17T17:14:30.245ZLoading write count information from file "/tmp/test_repair_verify.data"
60712024-04-17T17:14:30.245ZRun Repair workload
60722024-04-17T17:14:30.246Z01/30 Write block 261 len 12288 data: 4 4 4
60732024-04-17T17:14:30.246Z02/30 Read block 243 len 4096
60742024-04-17T17:14:30.246Z03/30 Read block 273 len 8192
60752024-04-17T17:14:30.246Z04/30 Flush
60762024-04-17T17:14:30.246Z05/30 Write block 573 len 8192 data: 6 6
60772024-04-17T17:14:30.246Z06/30 Read block 320 len 12288
60782024-04-17T17:14:30.246Z07/30 Flush
60792024-04-17T17:14:30.246Z08/30 Write block 191 len 20480 data: 4 5 5 3 2
60802024-04-17T17:14:30.250Z09/30 Write block 96 len 40960 data: 4 4 4 6 6 6 5 7 8 8
60812024-04-17T17:14:30.250Z10/30 Read block 582 len 36864
60822024-04-17T17:14:30.250Z11/30 Read block 512 len 8192
60832024-04-17T17:14:30.250Z12/30 Read block 39 len 20480
60842024-04-17T17:14:30.250Z13/30 Read block 388 len 8192
60852024-04-17T17:14:30.250Z14/30 Flush
60862024-04-17T17:14:30.253Z15/30 Write block 560 len 36864 data: 7 5 4 3 4 3 4 3 5
60872024-04-17T17:14:30.254Z16/30 Read block 330 len 32768
60882024-04-17T17:14:30.254Z17/30 Flush
60892024-04-17T17:14:30.257Z18/30 Write block 483 len 20480 data: 11 10 8 9 6
60902024-04-17T17:14:30.257Z19/30 Read block 155 len 40960
60912024-04-17T17:14:30.257Z20/30 Read block 418 len 20480
60922024-04-17T17:14:30.261Z21/30 Write block 125 len 28672 data: 4 4 5 4 4 6 7
60932024-04-17T17:14:30.261Z22/30 Write block 158 len 40960 data: 6 6 5 6 6 6 4 3 4 5
60942024-04-17T17:14:30.261Z23/30 Write block 41 len 4096 data: 9
60952024-04-17T17:14:30.264Z24/30 Write block 432 len 32768 data: 2 5 5 6 6 6 7 8
60962024-04-17T17:14:30.264Z25/30 Read block 461 len 36864
60972024-04-17T17:14:30.264Z26/30 Read block 581 len 8192
60982024-04-17T17:14:30.264Z27/30 Write block 404 len 12288 data: 6 5 5
60992024-04-17T17:14:30.268Z28/30 Write block 477 len 24576 data: 6 9 10 10 9 9
61002024-04-17T17:14:30.268Z29/30 Write block 296 len 4096 data: 2
61012024-04-17T17:14:30.269Z30/30 Write block 567 len 12288 data: 4 6 6
61022024-04-17T17:14:30.269Z----------------------------------------------------------------
61032024-04-17T17:14:30.269Z Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30
61042024-04-17T17:14:30.269ZStates: Active Active Active
61052024-04-17T17:14:30.269ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
61062024-04-17T17:14:30.269Z 1 Acked 1000 Write 3 Done Done Done false
61072024-04-17T17:14:30.269Z 2 Acked 1001 Read 1 Done Done Done false
61082024-04-17T17:14:30.269Z 3 Acked 1002 Read 2 Done Done Sent false
61092024-04-17T17:14:30.269Z 4 Acked 1003 Flush 0 Done Done Sent false
61102024-04-17T17:14:30.269Z 5 Acked 1004 Write 2 Done Done Sent false
61112024-04-17T17:14:30.269Z 6 Acked 1005 Read 3 Done Done Sent false
61122024-04-17T17:14:30.269Z 7 Acked 1006 Flush 0 Done Done Sent false
61132024-04-17T17:14:30.269Z 8 Acked 1007 Write 5 Done Done Sent false
61142024-04-17T17:14:30.269Z 9 Acked 1008 Write 10 Done Done Sent false
61152024-04-17T17:14:30.269Z 10 Acked 1009 Read 9 Done Done Sent false
61162024-04-17T17:14:30.269Z 11 Acked 1010 Read 2 Done Done Sent false
61172024-04-17T17:14:30.269Z 12 Acked 1011 Read 5 Done Done Sent false
61182024-04-17T17:14:30.269Z 13 Acked 1012 Read 2 Done Done Sent false
61192024-04-17T17:14:30.269Z 14 Acked 1013 Flush 0 Done Done Sent false
61202024-04-17T17:14:30.269Z 15 Acked 1014 Write 9 Done Done Sent false
61212024-04-17T17:14:30.270Z 16 Acked 1015 Read 8 Done Done Sent false
61222024-04-17T17:14:30.270Z 17 Acked 1016 Flush 0 Done Done Sent false
61232024-04-17T17:14:30.270Z 18 Acked 1017 Write 5 Done Done Sent false
61242024-04-17T17:14:30.270Z 19 Acked 1018 Read 10 Done Done Sent false
61252024-04-17T17:14:30.270Z 20 Acked 1019 Read 5 Done Done Sent false
61262024-04-17T17:14:30.270Z 21 Acked 1020 Write 7 Done Done Sent false
61272024-04-17T17:14:30.270Z 22 Acked 1021 Write 10 Done Done Sent false
61282024-04-17T17:14:30.270Z 23 Acked 1022 Write 1 Done Done Sent false
61292024-04-17T17:14:30.270Z 24 Acked 1023 Write 8 Done Done Sent false
61302024-04-17T17:14:30.270Z 25 Acked 1024 Read 9 Done Done Sent false
61312024-04-17T17:14:30.270Z 26 Acked 1025 Read 2 Done Done Sent false
61322024-04-17T17:14:30.270Z 27 Acked 1026 Write 3 Done Done Sent false
61332024-04-17T17:14:30.270Z 28 Acked 1027 Write 6 Done Done Sent false
61342024-04-17T17:14:30.270Z 29 Acked 1028 Write 1 Done Done Sent false
61352024-04-17T17:14:30.270Z 30 Acked 1029 Write 3 Sent Sent Sent false
61362024-04-17T17:14:30.270Z STATES DS:0 DS:1 DS:2 TOTAL
61372024-04-17T17:14:30.270Z New 0 0 0 0
61382024-04-17T17:14:30.270Z Sent 1 1 28 30
61392024-04-17T17:14:30.270Z Done 29 29 2 60
61402024-04-17T17:14:30.270Z Skipped 0 0 0 0
61412024-04-17T17:14:30.270Z Error 0 0 0 0
61422024-04-17T17:14:30.271ZLast Flush: 1016 1016 0
61432024-04-17T17:14:30.271ZDownstairs last five completed:
61442024-04-17T17:14:30.271ZUpstairs last five completed: 30 29 28 27 26
61452024-04-17T17:14:30.271Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:14:28.874856483Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1392}
61462024-04-17T17:14:30.271ZWrote out file "/tmp/test_repair_verify.data"
61472024-04-17T17:14:30.274Z
61482024-04-17T17:14:30.274ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
61492024-04-17T17:14:30.293ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
61502024-04-17T17:14:30.293Z 0 000-019 51 51 51 90 90 90 F F F
61512024-04-17T17:14:30.297Z 1 020-039 52 52 52 92 92 92 F F F
61522024-04-17T17:14:30.297Z 2 040-059 52 52 52 92 92 92 T T F <---
61532024-04-17T17:14:30.297Z 3 060-079 55 55 55 98 98 98 F F F
61542024-04-17T17:14:30.297Z 4 080-099 56 56 55 101 101 98 F F F <---
61552024-04-17T17:14:30.297Z 5 100-119 56 56 53 101 101 95 F F F <---
61562024-04-17T17:14:30.297Z 6 120-139 54 54 54 96 96 96 T T F <---
61572024-04-17T17:14:30.297Z 7 140-159 55 55 55 98 98 98 T T F <---
61582024-04-17T17:14:30.297Z 8 160-179 51 51 51 90 90 90 T T F <---
61592024-04-17T17:14:30.297Z 9 180-199 56 56 49 101 101 87 F F F <---
61602024-04-17T17:14:30.297Z 10 200-219 54 54 54 96 96 96 F F F
61612024-04-17T17:14:30.297Z 11 220-239 51 51 51 90 90 90 F F F
61622024-04-17T17:14:30.297Z 12 240-259 55 55 55 98 98 98 F F F
61632024-04-17T17:14:30.297Z 13 260-279 56 56 51 99 99 90 F F T <---
61642024-04-17T17:14:30.297Z 14 280-299 54 54 54 97 97 97 T T F <---
61652024-04-17T17:14:30.297Z 15 300-319 55 55 55 98 98 98 F F F
61662024-04-17T17:14:30.297Z 16 320-339 51 51 51 90 90 90 F F F
61672024-04-17T17:14:30.297Z 17 340-359 52 52 52 92 92 92 F F F
61682024-04-17T17:14:30.297Z 18 360-379 47 47 47 85 85 85 F F F
61692024-04-17T17:14:30.297Z 19 380-399 47 47 47 85 85 85 F F F
61702024-04-17T17:14:30.297Z 20 400-419 53 53 53 95 95 95 T T F <---
61712024-04-17T17:14:30.298Z 21 420-439 55 55 55 98 98 98 T T F <---
61722024-04-17T17:14:30.298Z 22 440-459 52 52 52 94 94 94 F F F
61732024-04-17T17:14:30.298Z 23 460-479 50 50 50 88 88 88 T T F <---
61742024-04-17T17:14:30.298Z 24 480-499 54 54 54 97 97 97 T T F <---
61752024-04-17T17:14:30.298Z 25 500-519 54 54 54 97 97 97 F F F
61762024-04-17T17:14:30.298Z 26 520-539 52 52 52 91 91 91 F F F
61772024-04-17T17:14:30.298Z 27 540-559 53 53 53 95 95 95 F F F
61782024-04-17T17:14:30.298Z 28 560-579 56 56 55 102 102 98 T T F <---
61792024-04-17T17:14:30.298Z 29 580-599 55 55 55 98 98 98 F F F
61802024-04-17T17:14:30.298ZMax gen: 56, Max flush: 102
61812024-04-17T17:14:30.298ZError: Difference in extent metadata found!
61822024-04-17T17:14:30.302ZA Difference in extent metadata is expected here
61832024-04-17T17:14:30.302ZOn loop 28
61842024-04-17T17:14:30.302Z
61852024-04-17T17:14:30.302ZVerifying data now
61862024-04-17T17:14:34.951ZNo change after verify
61872024-04-17T17:14:34.951ZLoop: 28 Downstairs dump after verify (and repair):
61882024-04-17T17:14:34.966ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
61892024-04-17T17:14:34.966Z 0 000-019 51 51 51 90 90 90 F F F
61902024-04-17T17:14:34.971Z 1 020-039 52 52 52 92 92 92 F F F
61912024-04-17T17:14:34.971Z 2 040-059 57 57 57 103 103 103 F F F
61922024-04-17T17:14:34.971Z 3 060-079 55 55 55 98 98 98 F F F
61932024-04-17T17:14:34.971Z 4 080-099 56 56 56 101 101 101 F F F
61942024-04-17T17:14:34.971Z 5 100-119 56 56 56 101 101 101 F F F
61952024-04-17T17:14:34.971Z 6 120-139 57 57 57 103 103 103 F F F
61962024-04-17T17:14:34.971Z 7 140-159 57 57 57 103 103 103 F F F
61972024-04-17T17:14:34.971Z 8 160-179 57 57 57 103 103 103 F F F
61982024-04-17T17:14:34.971Z 9 180-199 56 56 56 101 101 101 F F F
61992024-04-17T17:14:34.971Z 10 200-219 54 54 54 96 96 96 F F F
62002024-04-17T17:14:34.971Z 11 220-239 51 51 51 90 90 90 F F F
62012024-04-17T17:14:34.971Z 12 240-259 55 55 55 98 98 98 F F F
62022024-04-17T17:14:34.971Z 13 260-279 56 56 56 99 99 99 F F F
62032024-04-17T17:14:34.971Z 14 280-299 57 57 57 103 103 103 F F F
62042024-04-17T17:14:34.971Z 15 300-319 55 55 55 98 98 98 F F F
62052024-04-17T17:14:34.971Z 16 320-339 51 51 51 90 90 90 F F F
62062024-04-17T17:14:34.971Z 17 340-359 52 52 52 92 92 92 F F F
62072024-04-17T17:14:34.971Z 18 360-379 47 47 47 85 85 85 F F F
62082024-04-17T17:14:34.971Z 19 380-399 47 47 47 85 85 85 F F F
62092024-04-17T17:14:34.971Z 20 400-419 57 57 57 103 103 103 F F F
62102024-04-17T17:14:34.972Z 21 420-439 57 57 57 103 103 103 F F F
62112024-04-17T17:14:34.972Z 22 440-459 52 52 52 94 94 94 F F F
62122024-04-17T17:14:34.972Z 23 460-479 57 57 57 103 103 103 F F F
62132024-04-17T17:14:34.972Z 24 480-499 57 57 57 103 103 103 F F F
62142024-04-17T17:14:34.972Z 25 500-519 54 54 54 97 97 97 F F F
62152024-04-17T17:14:34.972Z 26 520-539 52 52 52 91 91 91 F F F
62162024-04-17T17:14:34.972Z 27 540-559 53 53 53 95 95 95 F F F
62172024-04-17T17:14:34.972Z 28 560-579 57 57 57 103 103 103 F F F
62182024-04-17T17:14:34.972Z 29 580-599 55 55 55 98 98 98 F F F
62192024-04-17T17:14:34.972ZMax gen: 57, Max flush: 103
62202024-04-17T17:14:34.972Z
62212024-04-17T17:14:34.972ZBegin loop 29
62222024-04-17T17:14:34.972ZDownstairs to restart: 1
62232024-04-17T17:14:34.990Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.59717797Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400}
62242024-04-17T17:14:34.997Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.597396313Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400}
62252024-04-17T17:14:34.997Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.597460137Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400}
62262024-04-17T17:14:34.997Z{"msg":"Crucible 61422757-904f-4e2f-bb55-aa90f0cd61ff has session id: e6e6ab47-d8b2-47ff-8094-281aca805c89","v":0,"name":"crucible","level":30Crucible runtime is spawned
62272024-04-17T17:14:34.998Z,"time":"2024-04-17T17:14:33.597493349Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62282024-04-17T17:14:34.998Z{"msg":"Upstairs opts: Upstairs UUID: 61422757-904f-4e2f-bb55-aa90f0cd61ff, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.597546392Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62292024-04-17T17:14:34.998Z{"msg":"Crucible stats registered with UUID: 61422757-904f-4e2f-bb55-aa90f0cd61ff","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.597580274Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62302024-04-17T17:14:34.998Z{"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.597594425Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400}
62312024-04-17T17:14:34.998Z{"msg":"61422757-904f-4e2f-bb55-aa90f0cd61ff active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.597717713Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62322024-04-17T17:14:34.998Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.597747875Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"client":"0","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62332024-04-17T17:14:34.998Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.597797988Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"client":"0","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62342024-04-17T17:14:34.998Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.59784633Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"client":"1","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62352024-04-17T17:14:34.998Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.597874712Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"client":"1","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62362024-04-17T17:14:34.998Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.597898224Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"client":"2","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62372024-04-17T17:14:34.998Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.597933396Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"client":"2","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62382024-04-17T17:14:34.998Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.597961157Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62392024-04-17T17:14:34.998Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.59800306Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62402024-04-17T17:14:34.998Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.598030172Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62412024-04-17T17:14:34.998Z{"msg":"ds_connection connected from Ok(127.0.0.1:63438)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.598065464Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62422024-04-17T17:14:34.998Z{"msg":"ds_connection connected from Ok(127.0.0.1:51095)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.598089895Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62432024-04-17T17:14:34.998Z{"msg":"ds_connection connected from Ok(127.0.0.1:50152)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.598146029Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62442024-04-17T17:14:34.998Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.598795578Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"client":"0","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62452024-04-17T17:14:34.998Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.59883295Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"client":"0","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62462024-04-17T17:14:34.998Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.598849141Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"client":"2","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62472024-04-17T17:14:34.998Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.598871063Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"client":"2","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62482024-04-17T17:14:34.998Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.599024302Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"client":"1","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62492024-04-17T17:14:34.999Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.599061424Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"client":"1","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62502024-04-17T17:14:34.999Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.59948967Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"client":"0","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62512024-04-17T17:14:34.999Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.599536873Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"client":"1","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62522024-04-17T17:14:34.999Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.599570885Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"client":"2","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62532024-04-17T17:14:34.999Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.5999909Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"client":"0","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62542024-04-17T17:14:34.999Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.600029203Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"client":"0","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62552024-04-17T17:14:34.999Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.600081376Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62562024-04-17T17:14:34.999Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.600096537Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"client":"2","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62572024-04-17T17:14:34.999Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.600128749Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"client":"2","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62582024-04-17T17:14:34.999Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.600159531Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62592024-04-17T17:14:34.999Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.600190573Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"client":"1","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62602024-04-17T17:14:34.999Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.600208484Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"client":"1","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62612024-04-17T17:14:34.999Z{"msg":"[0]R flush_numbers[0..12]: [90, 92, 103, 98, 101, 101, 103, 103, 103, 101, 96, 90]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.600238266Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"":"downstairs","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62622024-04-17T17:14:34.999Z{"msg":"[0]R generation[0..12]: [51, 52, 57, 55, 56, 56, 57, 57, 57, 56, 54, 51]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.600259177Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"":"downstairs","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62632024-04-17T17:14:34.999ZWait for a query_work_queue command to finish before sending IO
62642024-04-17T17:14:34.999Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.600293539Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"":"downstairs","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62652024-04-17T17:14:34.999Z{"msg":"[1]R flush_numbers[0..12]: [90, 92, 103, 98, 101, 101, 103, 103, 103, 101, 96, 90]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.60030933Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"":"downstairs","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62662024-04-17T17:14:34.999Z{"msg":"[1]R generation[0..12]: [51, 52, 57, 55, 56, 56, 57, 57, 57, 56, 54, 51]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.600328901Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"":"downstairs","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62672024-04-17T17:14:34.999Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.600358803Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"":"downstairs","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62682024-04-17T17:14:34.999Z{"msg":"[2]R flush_numbers[0..12]: [90, 92, 103, 98, 101, 101, 103, 103, 103, 101, 96, 90]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.600374414Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"":"downstairs","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62692024-04-17T17:14:34.999Z{"msg":"[2]R generation[0..12]: [51, 52, 57, 55, 56, 56, 57, 57, 57, 56, 54, 51]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.600406166Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"":"downstairs","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62702024-04-17T17:14:34.999Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.600422927Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"":"downstairs","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62712024-04-17T17:14:35.000Z{"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.600452419Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"":"downstairs","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62722024-04-17T17:14:35.000Z{"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.60046998Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"":"downstairs","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62732024-04-17T17:14:35.000Z{"msg":"Next flush: 104","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.600497861Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"":"downstairs","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62742024-04-17T17:14:35.000Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.600512012Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"":"downstairs","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62752024-04-17T17:14:35.000Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.600541054Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62762024-04-17T17:14:35.000Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.600561175Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"client":"0","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62772024-04-17T17:14:35.000Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.600588937Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"client":"1","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62782024-04-17T17:14:35.000Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.600602808Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"client":"2","":"downstairs","session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62792024-04-17T17:14:35.000Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.600631869Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62802024-04-17T17:14:35.000Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
62812024-04-17T17:14:35.000Z,"time":"2024-04-17T17:14:33.600660711Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62822024-04-17T17:14:35.000Z{"msg":"61422757-904f-4e2f-bb55-aa90f0cd61ff is now active with session: e6e6ab47-d8b2-47ff-8094-281aca805c89","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.600682942Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62832024-04-17T17:14:35.001Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.600709634Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400,"session_id":"e6e6ab47-d8b2-47ff-8094-281aca805c89"}
62842024-04-17T17:14:35.001Z{"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:33.600723315Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400}
62852024-04-17T17:14:35.001ZLoading write count information from file "/tmp/test_repair_verify.data"
62862024-04-17T17:14:35.001ZRun Repair workload
62872024-04-17T17:14:35.005Z01/30 Write block 514 len 32768 data: 7 7 5 5 6 5 5 5
62882024-04-17T17:14:35.005Z02/30 Flush
62892024-04-17T17:14:35.005Z03/30 Write block 113 len 4096 data: 6
62902024-04-17T17:14:35.005Z04/30 Write block 135 len 40960 data: 6 6 5 5 5 4 4 4 5 5
62912024-04-17T17:14:35.005Z05/30 Read block 29 len 8192
62922024-04-17T17:14:35.005Z06/30 Flush
62932024-04-17T17:14:35.005Z07/30 Write block 388 len 20480 data: 4 5 5 4 5
62942024-04-17T17:14:35.006Z08/30 Read block 535 len 24576
62952024-04-17T17:14:35.011Z09/30 Write block 551 len 20480 data: 5 6 6 6 6
62962024-04-17T17:14:35.011Z10/30 Write block 157 len 20480 data: 8 7 7 6 7
62972024-04-17T17:14:35.011Z11/30 Read block 293 len 20480
62982024-04-17T17:14:35.011Z12/30 Read block 330 len 32768
62992024-04-17T17:14:35.011Z13/30 Write block 314 len 32768 data: 8 7 6 7 7 6 5 4
63002024-04-17T17:14:35.011Z14/30 Write block 91 len 28672 data: 8 8 8 7 7 5 5
63012024-04-17T17:14:35.015Z15/30 Read block 181 len 36864
63022024-04-17T17:14:35.015Z16/30 Read block 200 len 8192
63032024-04-17T17:14:35.015Z17/30 Read block 377 len 8192
63042024-04-17T17:14:35.015Z18/30 Write block 484 len 28672 data: 11 9 10 7 7 6 6
63052024-04-17T17:14:35.015Z19/30 Read block 257 len 16384
63062024-04-17T17:14:35.019Z20/30 Write block 559 len 24576 data: 7 8 6 5 4 5
63072024-04-17T17:14:35.019Z21/30 Read block 339 len 40960
63082024-04-17T17:14:35.019Z22/30 Read block 443 len 28672
63092024-04-17T17:14:35.019Z23/30 Read block 133 len 40960
63102024-04-17T17:14:35.022Z24/30 Read block 455 len 20480
63112024-04-17T17:14:35.022Z25/30 Read block 375 len 36864
63122024-04-17T17:14:35.022Z26/30 Read block 333 len 28672
63132024-04-17T17:14:35.022Z27/30 Write block 136 len 40960 data: 7 6 6 6 5 5 5 6 6 5
63142024-04-17T17:14:35.026Z28/30 Read block 421 len 24576
63152024-04-17T17:14:35.027Z29/30 Write block 54 len 24576 data: 5 4 4 6 5 4
63162024-04-17T17:14:35.027Z30/30 Write block 150 len 32768 data: 3 3 4 3 5 6 8 9
63172024-04-17T17:14:35.031Z----------------------------------------------------------------
63182024-04-17T17:14:35.031Z Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:24
63192024-04-17T17:14:35.031ZStates: Active Active Active
63202024-04-17T17:14:35.031ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
63212024-04-17T17:14:35.031Z 7 Acked 1006 Write 5 Done Done Done false
63222024-04-17T17:14:35.031Z 8 Acked 1007 Read 6 Done Done Done false
63232024-04-17T17:14:35.031Z 9 Acked 1008 Write 5 Done Done Done false
63242024-04-17T17:14:35.031Z 10 Acked 1009 Write 5 Done Done Done false
63252024-04-17T17:14:35.031Z 11 Acked 1010 Read 5 Done Done Done false
63262024-04-17T17:14:35.031Z 12 Acked 1011 Read 8 Done Done Done false
63272024-04-17T17:14:35.031Z 13 Acked 1012 Write 8 Done Done Done false
63282024-04-17T17:14:35.031Z 14 Acked 1013 Write 7 Done Done Done false
63292024-04-17T17:14:35.031Z 15 Acked 1014 Read 9 Done Done Done false
63302024-04-17T17:14:35.031Z 16 Acked 1015 Read 2 Done Sent Done false
63312024-04-17T17:14:35.031Z 17 Acked 1016 Read 2 Done Sent Done false
63322024-04-17T17:14:35.032Z 18 Acked 1017 Write 7 Done Sent Done false
63332024-04-17T17:14:35.032Z 19 Acked 1018 Read 4 Done Sent Done false
63342024-04-17T17:14:35.032Z 20 Acked 1019 Write 6 Done Sent Done false
63352024-04-17T17:14:35.032Z 21 Acked 1020 Read 10 Done Sent Done false
63362024-04-17T17:14:35.032Z 22 Acked 1021 Read 7 Done Sent Done false
63372024-04-17T17:14:35.032Z 23 Acked 1022 Read 10 Done Sent Done false
63382024-04-17T17:14:35.032Z 24 Acked 1023 Read 5 Done Sent Done false
63392024-04-17T17:14:35.032Z 25 Acked 1024 Read 9 Done Sent Done false
63402024-04-17T17:14:35.032Z 26 Acked 1025 Read 7 Done Sent Done false
63412024-04-17T17:14:35.032Z 27 Acked 1026 Write 10 Done Sent Done false
63422024-04-17T17:14:35.032Z 28 Acked 1027 Read 6 Done Sent Done false
63432024-04-17T17:14:35.032Z 29 Acked 1028 Write 6 Done Sent Done false
63442024-04-17T17:14:35.032Z 30 Acked 1029 Write 8 Sent Sent Sent false
63452024-04-17T17:14:35.032Z STATES DS:0 DS:1 DS:2 TOTAL
63462024-04-17T17:14:35.032Z New 0 0 0 0
63472024-04-17T17:14:35.032Z Sent 1 15 1 17
63482024-04-17T17:14:35.032Z Done 23 9 23 55
63492024-04-17T17:14:35.032Z Skipped 0 0 0 0
63502024-04-17T17:14:35.032Z Error 0 0 0 0
63512024-04-17T17:14:35.032ZLast Flush: 1005 1005 1005
63522024-04-17T17:14:35.032ZDownstairs last five completed: 1005 1004 1003 1002 1001
63532024-04-17T17:14:35.032ZUpstairs last five completed: 30 29 28 27 26
63542024-04-17T17:14:35.032Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:14:33.634340672Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1400}
63552024-04-17T17:14:35.032ZWrote out file "/tmp/test_repair_verify.data"
63562024-04-17T17:14:35.033Z
63572024-04-17T17:14:35.037ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
63582024-04-17T17:14:35.054ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
63592024-04-17T17:14:35.054Z 0 000-019 51 51 51 90 90 90 F F F
63602024-04-17T17:14:35.054Z 1 020-039 52 52 52 92 92 92 F F F
63612024-04-17T17:14:35.054Z 2 040-059 57 57 57 103 103 103 T F T <---
63622024-04-17T17:14:35.054Z 3 060-079 55 55 55 98 98 98 F F F
63632024-04-17T17:14:35.054Z 4 080-099 56 56 56 101 101 101 T T T
63642024-04-17T17:14:35.054Z 5 100-119 58 58 58 105 105 105 F F F
63652024-04-17T17:14:35.055Z 6 120-139 58 58 58 105 105 105 T F T <---
63662024-04-17T17:14:35.058Z 7 140-159 58 58 58 105 105 105 T T T
63672024-04-17T17:14:35.058Z 8 160-179 57 57 57 103 103 103 T T T
63682024-04-17T17:14:35.058Z 9 180-199 56 56 56 101 101 101 F F F
63692024-04-17T17:14:35.058Z 10 200-219 54 54 54 96 96 96 F F F
63702024-04-17T17:14:35.058Z 11 220-239 51 51 51 90 90 90 F F F
63712024-04-17T17:14:35.058Z 12 240-259 55 55 55 98 98 98 F F F
63722024-04-17T17:14:35.058Z 13 260-279 56 56 56 99 99 99 F F F
63732024-04-17T17:14:35.059Z 14 280-299 57 57 57 103 103 103 F F F
63742024-04-17T17:14:35.059Z 15 300-319 55 55 55 98 98 98 T T T
63752024-04-17T17:14:35.059Z 16 320-339 51 51 51 90 90 90 T T T
63762024-04-17T17:14:35.059Z 17 340-359 52 52 52 92 92 92 F F F
63772024-04-17T17:14:35.059Z 18 360-379 47 47 47 85 85 85 F F F
63782024-04-17T17:14:35.059Z 19 380-399 47 47 47 85 85 85 T T T
63792024-04-17T17:14:35.059Z 20 400-419 57 57 57 103 103 103 F F F
63802024-04-17T17:14:35.059Z 21 420-439 57 57 57 103 103 103 F F F
63812024-04-17T17:14:35.059Z 22 440-459 52 52 52 94 94 94 F F F
63822024-04-17T17:14:35.059Z 23 460-479 57 57 57 103 103 103 F F F
63832024-04-17T17:14:35.059Z 24 480-499 57 57 57 103 103 103 T F T <---
63842024-04-17T17:14:35.059Z 25 500-519 58 58 58 104 104 104 F F F
63852024-04-17T17:14:35.059Z 26 520-539 58 58 58 104 104 104 F F F
63862024-04-17T17:14:35.059Z 27 540-559 53 53 53 95 95 95 T T T
63872024-04-17T17:14:35.059Z 28 560-579 57 57 57 103 103 103 T F T <---
63882024-04-17T17:14:35.059Z 29 580-599 55 55 55 98 98 98 F F F
63892024-04-17T17:14:35.059ZMax gen: 58, Max flush: 105
63902024-04-17T17:14:35.059ZError: Difference in extent metadata found!
63912024-04-17T17:14:35.059ZA Difference in extent metadata is expected here
63922024-04-17T17:14:35.059ZOn loop 29
63932024-04-17T17:14:35.059Z
63942024-04-17T17:14:35.063ZVerifying data now
63952024-04-17T17:14:49.601ZNo change after verify
63962024-04-17T17:14:49.601ZLoop: 29 Downstairs dump after verify (and repair):
63972024-04-17T17:14:49.615ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
63982024-04-17T17:14:49.615Z 0 000-019 51 51 51 90 90 90 F F F
63992024-04-17T17:14:49.615Z 1 020-039 52 52 52 92 92 92 F F F
64002024-04-17T17:14:49.615Z 2 040-059 59 59 59 106 106 106 F F F
64012024-04-17T17:14:49.615Z 3 060-079 55 55 55 98 98 98 F F F
64022024-04-17T17:14:49.615Z 4 080-099 59 59 59 106 106 106 F F F
64032024-04-17T17:14:49.615Z 5 100-119 58 58 58 105 105 105 F F F
64042024-04-17T17:14:49.615Z 6 120-139 59 59 59 106 106 106 F F F
64052024-04-17T17:14:49.615Z 7 140-159 59 59 59 106 106 106 F F F
64062024-04-17T17:14:49.615Z 8 160-179 59 59 59 106 106 106 F F F
64072024-04-17T17:14:49.615Z 9 180-199 56 56 56 101 101 101 F F F
64082024-04-17T17:14:49.615Z 10 200-219 54 54 54 96 96 96 F F F
64092024-04-17T17:14:49.615Z 11 220-239 51 51 51 90 90 90 F F F
64102024-04-17T17:14:49.615Z 12 240-259 55 55 55 98 98 98 F F F
64112024-04-17T17:14:49.615Z 13 260-279 56 56 56 99 99 99 F F F
64122024-04-17T17:14:49.619Z 14 280-299 57 57 57 103 103 103 F F F
64132024-04-17T17:14:49.619Z 15 300-319 59 59 59 106 106 106 F F F
64142024-04-17T17:14:49.619Z 16 320-339 59 59 59 106 106 106 F F F
64152024-04-17T17:14:49.619Z 17 340-359 52 52 52 92 92 92 F F F
64162024-04-17T17:14:49.619Z 18 360-379 47 47 47 85 85 85 F F F
64172024-04-17T17:14:49.619Z 19 380-399 59 59 59 106 106 106 F F F
64182024-04-17T17:14:49.619Z 20 400-419 57 57 57 103 103 103 F F F
64192024-04-17T17:14:49.620Z 21 420-439 57 57 57 103 103 103 F F F
64202024-04-17T17:14:49.620Z 22 440-459 52 52 52 94 94 94 F F F
64212024-04-17T17:14:49.620Z 23 460-479 57 57 57 103 103 103 F F F
64222024-04-17T17:14:49.620Z 24 480-499 59 59 59 106 106 106 F F F
64232024-04-17T17:14:49.620Z 25 500-519 58 58 58 104 104 104 F F F
64242024-04-17T17:14:49.620Z 26 520-539 58 58 58 104 104 104 F F F
64252024-04-17T17:14:49.620Z 27 540-559 59 59 59 106 106 106 F F F
64262024-04-17T17:14:49.620Z 28 560-579 59 59 59 106 106 106 F F F
64272024-04-17T17:14:49.620Z 29 580-599 55 55 55 98 98 98 F F F
64282024-04-17T17:14:49.620ZMax gen: 59, Max flush: 106
64292024-04-17T17:14:49.620Z
64302024-04-17T17:14:49.620ZBegin loop 30
64312024-04-17T17:14:49.620ZDownstairs to restart: 1
64322024-04-17T17:14:49.639Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.24711955Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408}
64332024-04-17T17:14:49.647Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.247435459Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408}
64342024-04-17T17:14:49.647Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.247503504Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408}
64352024-04-17T17:14:49.647Z{"msg":"Crucible 6bcecd30-e900-4782-bb7d-d4345068ad42 has session id: 146dec8c-ef4e-42cd-89a7-32359e74c51d","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.247527615Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64362024-04-17T17:14:49.647Z{"msg":"Upstairs opts: Upstairs UUID: 6bcecd30-e900-4782-bb7d-d4345068ad42, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.247564907Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64372024-04-17T17:14:49.647Z{"msg":"Crucible stats registered with UUID: 6bcecd30-e900-4782-bb7d-d4345068ad42","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.247620371Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64382024-04-17T17:14:49.647ZCrucible runtime is spawned
64392024-04-17T17:14:49.647Z{"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.247847074Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408}
64402024-04-17T17:14:49.647Z{"msg":"6bcecd30-e900-4782-bb7d-d4345068ad42 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.247910218Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64412024-04-17T17:14:49.648Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.24794096Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"client":"0","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64422024-04-17T17:14:49.648Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.247981782Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"client":"0","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64432024-04-17T17:14:49.648Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.248013074Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"client":"1","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64442024-04-17T17:14:49.648Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.248062297Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"client":"1","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64452024-04-17T17:14:49.648Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.248093689Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"client":"2","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64462024-04-17T17:14:49.648Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.248125451Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"client":"2","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64472024-04-17T17:14:49.648Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.248157603Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64482024-04-17T17:14:49.648Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.248204406Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64492024-04-17T17:14:49.648Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.248224737Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64502024-04-17T17:14:49.648Z{"msg":"ds_connection connected from Ok(127.0.0.1:52866)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.248251489Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64512024-04-17T17:14:49.648Z{"msg":"ds_connection connected from Ok(127.0.0.1:64558)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.248295201Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64522024-04-17T17:14:49.648Z{"msg":"ds_connection connected from Ok(127.0.0.1:36715)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.248480673Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64532024-04-17T17:14:49.648Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.248997594Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"client":"0","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64542024-04-17T17:14:49.648Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.249043677Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"client":"0","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64552024-04-17T17:14:49.648Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.249066678Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"client":"2","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64562024-04-17T17:14:49.648Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.249106721Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"client":"2","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64572024-04-17T17:14:49.648Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.249146763Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"client":"1","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64582024-04-17T17:14:49.648Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.249186715Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"client":"1","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64592024-04-17T17:14:49.648Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.249608071Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"client":"2","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64602024-04-17T17:14:49.648Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.249644963Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"client":"0","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64612024-04-17T17:14:49.648Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.249833105Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"client":"1","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64622024-04-17T17:14:49.648Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.250131713Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"client":"2","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64632024-04-17T17:14:49.648Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.250165315Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"client":"2","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64642024-04-17T17:14:49.649Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.250180956Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64652024-04-17T17:14:49.649Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.250203157Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"client":"0","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64662024-04-17T17:14:49.649Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.250224608Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"client":"0","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64672024-04-17T17:14:49.649Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.250264371Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64682024-04-17T17:14:49.649Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.250319104Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"client":"1","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64692024-04-17T17:14:49.649Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.250358897Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"client":"1","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64702024-04-17T17:14:49.649Z{"msg":"[0]R flush_numbers[0..12]: [90, 92, 106, 98, 106, 105, 106, 106, 106, 101, 96, 90]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.250383878Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"":"downstairs","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64712024-04-17T17:14:49.649Z{"msg":"[0]R generation[0..12]: [51, 52, 59, 55, 59, 58, 59, 59, 59, 56, 54, 51]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.25041877Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"":"downstairs","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64722024-04-17T17:14:49.649Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.250455362Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"":"downstairs","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64732024-04-17T17:14:49.649Z{"msg":"[1]R flush_numbers[0..12]: [90, 92, 106, 98, 106, 105, 106, 106, 106, 101, 96, 90]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.250490274Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"":"downstairs","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64742024-04-17T17:14:49.649Z{"msg":"[1]R generation[0..12]: [51, 52, 59, 55, 59, 58, 59, 59, 59, 56, 54, 51]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.250523367Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"":"downstairs","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64752024-04-17T17:14:49.649Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.250567479Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"":"downstairs","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64762024-04-17T17:14:49.649Z{"msg":"[2]R flush_numbers[0..12]: [90, 92, 106, 98, 106, 105, 106, 106, 106, 101, 96, 90]"Wait for a query_work_queue command to finish before sending IO
64772024-04-17T17:14:49.649Z,"v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.250597431Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"":"downstairs","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64782024-04-17T17:14:49.649Z{"msg":"[2]R generation[0..12]: [51, 52, 59, 55, 59, 58, 59, 59, 59, 56, 54, 51]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.250634823Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"":"downstairs","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64792024-04-17T17:14:49.649Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.250672716Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"":"downstairs","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64802024-04-17T17:14:49.649Z{"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.250708568Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"":"downstairs","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64812024-04-17T17:14:49.649Z{"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.250735649Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"":"downstairs","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64822024-04-17T17:14:49.649Z{"msg":"Next flush: 107","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.250770131Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"":"downstairs","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64832024-04-17T17:14:49.649Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.250795413Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"":"downstairs","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64842024-04-17T17:14:49.649Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.250809204Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64852024-04-17T17:14:49.649Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.250828165Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"client":"0","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64862024-04-17T17:14:49.649Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.250841726Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"client":"1","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64872024-04-17T17:14:49.650Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.250861727Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"client":"2","":"downstairs","session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64882024-04-17T17:14:49.650Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.250874978Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64892024-04-17T17:14:49.650Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.250893799Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64902024-04-17T17:14:49.650Z{"msg":"6bcecd30-e900-4782-bb7d-d4345068ad42 is now active with session: 146dec8c-ef4e-42cd-89a7-32359e74c51d","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.25090688Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64912024-04-17T17:14:49.650Z{"msg":"Set Active after no reconciliation","v":0,"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
64922024-04-17T17:14:49.650Zname":"crucible","level":30,"time":"2024-04-17T17:14:48.250922551Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408,"session_id":"146dec8c-ef4e-42cd-89a7-32359e74c51d"}
64932024-04-17T17:14:49.650Z{"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:48.250941732Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408}
64942024-04-17T17:14:49.650ZLoading write count information from file "/tmp/test_repair_verify.data"
64952024-04-17T17:14:49.650ZRun Repair workload
64962024-04-17T17:14:49.654Z01/30 Write block 206 len 28672 data: 6 6 2 2 2 2 3
64972024-04-17T17:14:49.654Z02/30 Read block 282 len 36864
64982024-04-17T17:14:49.654Z03/30 Read block 199 len 40960
64992024-04-17T17:14:49.654Z04/30 Read block 283 len 28672
65002024-04-17T17:14:49.655Z05/30 Flush
65012024-04-17T17:14:49.655Z06/30 Write block 281 len 16384 data: 3 3 3 4
65022024-04-17T17:14:49.655Z07/30 Write block 241 len 24576 data: 8 6 7 7 7 7
65032024-04-17T17:14:49.655Z08/30 Write block 201 len 28672 data: 7 7 8 8 8 7 7
65042024-04-17T17:14:49.655Z09/30 Read block 306 len 32768
65052024-04-17T17:14:49.658Z10/30 Write block 13 len 40960 data: 6 7 5 3 5 4 3 4 6 7
65062024-04-17T17:14:49.658Z11/30 Write block 336 len 32768 data: 6 8 8 8 7 7 9 9
65072024-04-17T17:14:49.658Z12/30 Read block 26 len 24576
65082024-04-17T17:14:49.661Z13/30 Write block 414 len 24576 data: 10 9 8 7 6 6
65092024-04-17T17:14:49.661Z14/30 Flush
65102024-04-17T17:14:49.666Z15/30 Write block 14 len 40960 data: 8 6 4 6 5 4 5 7 8 7
65112024-04-17T17:14:49.667Z16/30 Read block 201 len 8192
65122024-04-17T17:14:49.667Z17/30 Flush
65132024-04-17T17:14:49.681Z18/30 Write block 197 len 28672 data: 3 5 5 6 8 8 9
65142024-04-17T17:14:49.681Z19/30 Write block 506 len 28672 data: 4 6 6 5 4 4 5
65152024-04-17T17:14:49.681Z20/30 Read block 476 len 32768
65162024-04-17T17:14:49.681Z21/30 Read block 438 len 20480
65172024-04-17T17:14:49.681Z22/30 Read block 53 len 40960
65182024-04-17T17:14:49.681Z23/30 Write block 59 len 16384 data: 5 6 6 6
65192024-04-17T17:14:49.681Z24/30 Write block 574 len 12288 data: 7 7 7
65202024-04-17T17:14:49.685Z25/30 Write block 525 len 28672 data: 3 4 5 6 7 7 7
65212024-04-17T17:14:49.685Z26/30 Read block 474 len 8192
65222024-04-17T17:14:49.685Z27/30 Write block 202 len 36864 data: 9 10 9 9 8 8 3 3 3
65232024-04-17T17:14:49.686Z28/30 Write block 98 len 16384 data: 5 7 7 7
65242024-04-17T17:14:49.686Z29/30 Write block 411 len 4096 data: 7
65252024-04-17T17:14:49.686Z30/30 Write block 55 len 20480 data: 5 5 7 6 6
65262024-04-17T17:14:49.686Z----------------------------------------------------------------
65272024-04-17T17:14:49.686Z Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:30
65282024-04-17T17:14:49.686ZStates: Active Active Active
65292024-04-17T17:14:49.686ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
65302024-04-17T17:14:49.686Z 1 Acked 1000 Write 7 Done Sent Done false
65312024-04-17T17:14:49.686Z 2 Acked 1001 Read 9 Done Sent Done false
65322024-04-17T17:14:49.686Z 3 Acked 1002 Read 10 Done Sent Done false
65332024-04-17T17:14:49.686Z 4 Acked 1003 Read 7 Done Sent Done false
65342024-04-17T17:14:49.686Z 5 Acked 1004 Flush 0 Done Sent Done false
65352024-04-17T17:14:49.686Z 6 Acked 1005 Write 4 Done Sent Done false
65362024-04-17T17:14:49.686Z 7 Acked 1006 Write 6 Done Sent Done false
65372024-04-17T17:14:49.686Z 8 Acked 1007 Write 7 Done Sent Done false
65382024-04-17T17:14:49.686Z 9 Acked 1008 Read 8 Done Sent Done false
65392024-04-17T17:14:49.686Z 10 Acked 1009 Write 10 Done Sent Done false
65402024-04-17T17:14:49.686Z 11 Acked 1010 Write 8 Done Sent Done false
65412024-04-17T17:14:49.686Z 12 Acked 1011 Read 6 Done Sent Done false
65422024-04-17T17:14:49.686Z 13 Acked 1012 Write 6 Done Sent Done false
65432024-04-17T17:14:49.686Z 14 Acked 1013 Flush 0 Done Sent Done false
65442024-04-17T17:14:49.686Z 15 Acked 1014 Write 10 Done Sent Done false
65452024-04-17T17:14:49.687Z 16 Acked 1015 Read 2 Done Sent Done false
65462024-04-17T17:14:49.687Z 17 Acked 1016 Flush 0 Done Sent Done false
65472024-04-17T17:14:49.687Z 18 Acked 1017 Write 7 Done Sent Done false
65482024-04-17T17:14:49.687Z 19 Acked 1018 Write 7 Done Sent Done false
65492024-04-17T17:14:49.687Z 20 Acked 1019 Read 8 Done Sent Done false
65502024-04-17T17:14:49.687Z 21 Acked 1020 Read 5 Done Sent Done false
65512024-04-17T17:14:49.687Z 22 Acked 1021 Read 10 Done Sent Done false
65522024-04-17T17:14:49.687Z 23 Acked 1022 Write 4 Done Sent Done false
65532024-04-17T17:14:49.687Z 24 Acked 1023 Write 3 Done Sent Done false
65542024-04-17T17:14:49.687Z 25 Acked 1024 Write 7 Done Sent Done false
65552024-04-17T17:14:49.687Z 26 Acked 1025 Read 2 Done Sent Done false
65562024-04-17T17:14:49.687Z 27 Acked 1026 Write 9 Done Sent Done false
65572024-04-17T17:14:49.687Z 28 Acked 1027 Write 4 Done Sent Done false
65582024-04-17T17:14:49.687Z 29 Acked 1028 Write 1 Done Sent Done false
65592024-04-17T17:14:49.687Z 30 Acked 1029 Write 5 Sent Sent Sent false
65602024-04-17T17:14:49.687Z STATES DS:0 DS:1 DS:2 TOTAL
65612024-04-17T17:14:49.687Z New 0 0 0 0
65622024-04-17T17:14:49.687Z Sent 1 30 1 32
65632024-04-17T17:14:49.687Z Done 29 0 29 58
65642024-04-17T17:14:49.687Z Skipped 0 0 0 0
65652024-04-17T17:14:49.687Z Error 0 0 0 0
65662024-04-17T17:14:49.687ZLast Flush: 1016 0 1016
65672024-04-17T17:14:49.687ZDownstairs last five completed:
65682024-04-17T17:14:49.687ZUpstairs last five completed: 30 29 28 27 26
65692024-04-17T17:14:49.688Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:14:48.287529109Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1408}
65702024-04-17T17:14:49.688ZWrote out file "/tmp/test_repair_verify.data"
65712024-04-17T17:14:49.688Z
65722024-04-17T17:14:49.691ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
65732024-04-17T17:14:49.705ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
65742024-04-17T17:14:49.705Z 0 000-019 60 51 60 109 90 109 F F F <---
65752024-04-17T17:14:49.705Z 1 020-039 60 52 60 109 92 109 F F F <---
65762024-04-17T17:14:49.705Z 2 040-059 59 59 59 106 106 106 T F T <---
65772024-04-17T17:14:49.705Z 3 060-079 55 55 55 98 98 98 T F T <---
65782024-04-17T17:14:49.706Z 4 080-099 59 59 59 106 106 106 T F T <---
65792024-04-17T17:14:49.706Z 5 100-119 58 58 58 105 105 105 T F T <---
65802024-04-17T17:14:49.706Z 6 120-139 59 59 59 106 106 106 F F F
65812024-04-17T17:14:49.706Z 7 140-159 59 59 59 106 106 106 F F F
65822024-04-17T17:14:49.709Z 8 160-179 59 59 59 106 106 106 F F F
65832024-04-17T17:14:49.710Z 9 180-199 56 56 56 101 101 101 T F T <---
65842024-04-17T17:14:49.710Z 10 200-219 60 54 60 108 96 108 T F T <---
65852024-04-17T17:14:49.710Z 11 220-239 51 51 51 90 90 90 F F F
65862024-04-17T17:14:49.710Z 12 240-259 60 55 60 108 98 108 F F F <---
65872024-04-17T17:14:49.710Z 13 260-279 56 56 56 99 99 99 F F F
65882024-04-17T17:14:49.710Z 14 280-299 60 57 60 108 103 108 F F F <---
65892024-04-17T17:14:49.710Z 15 300-319 59 59 59 106 106 106 F F F
65902024-04-17T17:14:49.710Z 16 320-339 60 59 60 108 106 108 F F F <---
65912024-04-17T17:14:49.710Z 17 340-359 60 52 60 108 92 108 F F F <---
65922024-04-17T17:14:49.710Z 18 360-379 47 47 47 85 85 85 F F F
65932024-04-17T17:14:49.710Z 19 380-399 59 59 59 106 106 106 F F F
65942024-04-17T17:14:49.710Z 20 400-419 60 57 60 108 103 108 T F T <---
65952024-04-17T17:14:49.710Z 21 420-439 57 57 57 103 103 103 F F F
65962024-04-17T17:14:49.710Z 22 440-459 52 52 52 94 94 94 F F F
65972024-04-17T17:14:49.710Z 23 460-479 57 57 57 103 103 103 F F F
65982024-04-17T17:14:49.710Z 24 480-499 59 59 59 106 106 106 F F F
65992024-04-17T17:14:49.710Z 25 500-519 58 58 58 104 104 104 T F T <---
66002024-04-17T17:14:49.710Z 26 520-539 58 58 58 104 104 104 T F T <---
66012024-04-17T17:14:49.710Z 27 540-559 59 59 59 106 106 106 F F F
66022024-04-17T17:14:49.710Z 28 560-579 59 59 59 106 106 106 T F T <---
66032024-04-17T17:14:49.710Z 29 580-599 55 55 55 98 98 98 F F F
66042024-04-17T17:14:49.710ZMax gen: 60, Max flush: 109
66052024-04-17T17:14:49.711ZError: Difference in extent metadata found!
66062024-04-17T17:14:49.711ZA Difference in extent metadata is expected here
66072024-04-17T17:14:49.711ZOn loop 30
66082024-04-17T17:14:49.711Z
66092024-04-17T17:14:49.714ZVerifying data now
66102024-04-17T17:14:54.449ZNo change after verify
66112024-04-17T17:14:54.449ZLoop: 30 Downstairs dump after verify (and repair):
66122024-04-17T17:14:54.464ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
66132024-04-17T17:14:54.464Z 0 000-019 60 60 60 109 109 109 F F F
66142024-04-17T17:14:54.464Z 1 020-039 60 60 60 109 109 109 F F F
66152024-04-17T17:14:54.464Z 2 040-059 61 61 61 110 110 110 F F F
66162024-04-17T17:14:54.464Z 3 060-079 61 61 61 110 110 110 F F F
66172024-04-17T17:14:54.464Z 4 080-099 61 61 61 110 110 110 F F F
66182024-04-17T17:14:54.464Z 5 100-119 61 61 61 110 110 110 F F F
66192024-04-17T17:14:54.464Z 6 120-139 59 59 59 106 106 106 F F F
66202024-04-17T17:14:54.464Z 7 140-159 59 59 59 106 106 106 F F F
66212024-04-17T17:14:54.464Z 8 160-179 59 59 59 106 106 106 F F F
66222024-04-17T17:14:54.468Z 9 180-199 61 61 61 110 110 110 F F F
66232024-04-17T17:14:54.468Z 10 200-219 61 61 61 110 110 110 F F F
66242024-04-17T17:14:54.468Z 11 220-239 51 51 51 90 90 90 F F F
66252024-04-17T17:14:54.468Z 12 240-259 60 60 60 108 108 108 F F F
66262024-04-17T17:14:54.468Z 13 260-279 56 56 56 99 99 99 F F F
66272024-04-17T17:14:54.468Z 14 280-299 60 60 60 108 108 108 F F F
66282024-04-17T17:14:54.468Z 15 300-319 59 59 59 106 106 106 F F F
66292024-04-17T17:14:54.468Z 16 320-339 60 60 60 108 108 108 F F F
66302024-04-17T17:14:54.468Z 17 340-359 60 60 60 108 108 108 F F F
66312024-04-17T17:14:54.468Z 18 360-379 47 47 47 85 85 85 F F F
66322024-04-17T17:14:54.468Z 19 380-399 59 59 59 106 106 106 F F F
66332024-04-17T17:14:54.468Z 20 400-419 61 61 61 110 110 110 F F F
66342024-04-17T17:14:54.468Z 21 420-439 57 57 57 103 103 103 F F F
66352024-04-17T17:14:54.468Z 22 440-459 52 52 52 94 94 94 F F F
66362024-04-17T17:14:54.468Z 23 460-479 57 57 57 103 103 103 F F F
66372024-04-17T17:14:54.469Z 24 480-499 59 59 59 106 106 106 F F F
66382024-04-17T17:14:54.469Z 25 500-519 61 61 61 110 110 110 F F F
66392024-04-17T17:14:54.469Z 26 520-539 61 61 61 110 110 110 F F F
66402024-04-17T17:14:54.469Z 27 540-559 59 59 59 106 106 106 F F F
66412024-04-17T17:14:54.469Z 28 560-579 61 61 61 110 110 110 F F F
66422024-04-17T17:14:54.469Z 29 580-599 55 55 55 98 98 98 F F F
66432024-04-17T17:14:54.469ZMax gen: 61, Max flush: 110
66442024-04-17T17:14:54.469Z
66452024-04-17T17:14:54.469ZBegin loop 31
66462024-04-17T17:14:54.469ZDownstairs to restart: 1
66472024-04-17T17:14:54.486Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.093202266Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416}
66482024-04-17T17:14:54.493Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.093492644Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416}
66492024-04-17T17:14:54.493Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.093559918Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416}
66502024-04-17T17:14:54.493Z{"msg":"Crucible 30e621e0-a364-497b-a8e7-92541c9c4502 has session id: d5a66736-6573-43cd-b0b7-a7864021f602","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.09358685Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66512024-04-17T17:14:54.493Z{Crucible runtime is spawned
66522024-04-17T17:14:54.493Z"msg":"Upstairs opts: Upstairs UUID: 30e621e0-a364-497b-a8e7-92541c9c4502, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.093649854Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66532024-04-17T17:14:54.493Z{"msg":"Crucible stats registered with UUID: 30e621e0-a364-497b-a8e7-92541c9c4502","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.093703807Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66542024-04-17T17:14:54.493Z{"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.093730109Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416}
66552024-04-17T17:14:54.493Z{"msg":"30e621e0-a364-497b-a8e7-92541c9c4502 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.093862986Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66562024-04-17T17:14:54.494Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.093893188Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"client":"0","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66572024-04-17T17:14:54.494Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.093909089Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"client":"0","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66582024-04-17T17:14:54.494Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.09392719Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"client":"1","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66592024-04-17T17:14:54.494Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.093962663Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"client":"1","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66602024-04-17T17:14:54.494Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.093978984Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"client":"2","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66612024-04-17T17:14:54.494Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.094010386Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"client":"2","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66622024-04-17T17:14:54.494Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.094035367Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66632024-04-17T17:14:54.494Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.094067979Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66642024-04-17T17:14:54.494Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.094104721Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66652024-04-17T17:14:54.494Z{"msg":"ds_connection connected from Ok(127.0.0.1:36663)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.094213678Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66662024-04-17T17:14:54.494Z{"msg":"ds_connection connected from Ok(127.0.0.1:36400)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.09424959Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66672024-04-17T17:14:54.494Z{"msg":"ds_connection connected from Ok(127.0.0.1:64258)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.094276211Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66682024-04-17T17:14:54.494Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.095040428Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"client":"2","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66692024-04-17T17:14:54.494Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.09507825Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"client":"2","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66702024-04-17T17:14:54.494Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.095094751Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"client":"0","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66712024-04-17T17:14:54.494Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.095108532Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"client":"0","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66722024-04-17T17:14:54.494Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.095133003Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"client":"1","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66732024-04-17T17:14:54.494Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.095151865Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"client":"1","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66742024-04-17T17:14:54.494Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.095535808Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"client":"0","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66752024-04-17T17:14:54.494Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.095672156Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"client":"2","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66762024-04-17T17:14:54.494Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.095954043Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"client":"1","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66772024-04-17T17:14:54.494Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.096207629Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"client":"0","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66782024-04-17T17:14:54.494Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.096252111Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"client":"0","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66792024-04-17T17:14:54.495Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.096274093Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66802024-04-17T17:14:54.495Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.096346047Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"client":"2","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66812024-04-17T17:14:54.495Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.096382029Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"client":"2","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66822024-04-17T17:14:54.495Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.0964007Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66832024-04-17T17:14:54.495Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.096540749Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"client":"1","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66842024-04-17T17:14:54.495Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.096578551Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"client":"1","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66852024-04-17T17:14:54.495Z{"msg":"[0]R flush_numbers[0..12]: [109, 109, 110, 110, 110, 110, 106, 106, 106, 110, 110, 90]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.096595722Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"":"downstairs","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66862024-04-17T17:14:54.495Z{"msg":"[0]R generation[0..12]: [60, 60, 61, 61, 61, 61, 59, 59, 59, 61, 61, 51]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.096617284Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"":"downstairs","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66872024-04-17T17:14:54.495Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.096632784Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"":"downstairs","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66882024-04-17T17:14:54.495Z{"msg":"[1]R flush_numbers[0..12]: [109, 109, 110, 110, 110, 110, 106, 106, 106, 110, 110, 90]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.096653846Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"":"downstairs","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66892024-04-17T17:14:54.495Z{"msg":"[1]R generation[0..12]: [60, 60, 61, 61, 61, 61, 59, 59, 59, 61, 61, 51]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.096667836Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"":"downstairs","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66902024-04-17T17:14:54.495Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.096688608Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"":"downstairs","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66912024-04-17T17:14:54.495Z{"msg":"[2]R flush_numbers[0..12]: [109, 109, 110, 110, 110, 110, 106, 106, 106, 110, 110, 90]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.096702549Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"":"downstairs","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66922024-04-17T17:14:54.495Z{"msg":"[2]R generation[0..12]: [60, 60, 61, 61, 61, 61, 59, 59, 59, 61, 61, 51]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.09672287Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
66932024-04-17T17:14:54.495Zd5a66736-6573-43cd-b0b7-a7864021f602"}
66942024-04-17T17:14:54.495Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.096740421Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"":"downstairs","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66952024-04-17T17:14:54.495Z{"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.096753922Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"":"downstairs","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66962024-04-17T17:14:54.495Z{"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.096766773Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"":"downstairs","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66972024-04-17T17:14:54.495Z{"msg":"Next flush: 111","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.096786394Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"":"downstairs","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66982024-04-17T17:14:54.495Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.096799624Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"":"downstairs","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
66992024-04-17T17:14:54.495Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.096812485Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
67002024-04-17T17:14:54.495Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.096832037Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"client":"0","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
67012024-04-17T17:14:54.495Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.096845717Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"client":"1","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
67022024-04-17T17:14:54.495Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.096870599Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"client":"2","":"downstairs","session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
67032024-04-17T17:14:54.496Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.09688399Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
67042024-04-17T17:14:54.496Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.09689588Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
67052024-04-17T17:14:54.496Z{"msg":"30e621e0-a364-497b-a8e7-92541c9c4502 is now active with session: d5a66736-6573-43cd-b0b7-a7864021f602","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.096915251Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
67062024-04-17T17:14:54.496Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.096927472Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416,"session_id":"d5a66736-6573-43cd-b0b7-a7864021f602"}
67072024-04-17T17:14:54.496Z{"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:53.096940093Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416}
67082024-04-17T17:14:54.496ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
67092024-04-17T17:14:54.499ZLoading write count information from file "/tmp/test_repair_verify.data"
67102024-04-17T17:14:54.499ZRun Repair workload
67112024-04-17T17:14:54.499Z01/30 Write block 498 len 12288 data: 6 7 8
67122024-04-17T17:14:54.499Z02/30 Read block 363 len 8192
67132024-04-17T17:14:54.500Z03/30 Flush
67142024-04-17T17:14:54.500Z04/30 Flush
67152024-04-17T17:14:54.500Z05/30 Write block 449 len 28672 data: 6 6 4 5 5 4 5
67162024-04-17T17:14:54.500Z06/30 Flush
67172024-04-17T17:14:54.500Z07/30 Write block 571 len 36864 data: 5 6 7 8 8 8 6 6 8
67182024-04-17T17:14:54.503Z08/30 Read block 533 len 28672
67192024-04-17T17:14:54.503Z09/30 Read block 30 len 20480
67202024-04-17T17:14:54.503Z10/30 Write block 90 len 16384 data: 8 9 9 9
67212024-04-17T17:14:54.507Z11/30 Write block 168 len 32768 data: 4 4 3 4 4 5 3 6
67222024-04-17T17:14:54.507Z12/30 Read block 431 len 24576
67232024-04-17T17:14:54.507Z13/30 Read block 285 len 40960
67242024-04-17T17:14:54.507Z14/30 Read block 560 len 36864
67252024-04-17T17:14:54.507Z15/30 Read block 447 len 28672
67262024-04-17T17:14:54.510Z16/30 Read block 171 len 32768
67272024-04-17T17:14:54.510Z17/30 Write block 120 len 8192 data: 6 5
67282024-04-17T17:14:54.510Z18/30 Read block 38 len 28672
67292024-04-17T17:14:54.510Z19/30 Write block 575 len 32768 data: 9 9 7 7 9 7 7 6
67302024-04-17T17:14:54.513Z20/30 Write block 53 len 16384 data: 4 6 6 6
67312024-04-17T17:14:54.513Z21/30 Read block 475 len 16384
67322024-04-17T17:14:54.514Z22/30 Read block 363 len 4096
67332024-04-17T17:14:54.514Z23/30 Write block 495 len 4096 data: 5
67342024-04-17T17:14:54.514Z24/30 Read block 269 len 20480
67352024-04-17T17:14:54.514Z25/30 Read block 434 len 32768
67362024-04-17T17:14:54.517Z26/30 Read block 352 len 40960
67372024-04-17T17:14:54.517Z27/30 Read block 49 len 24576
67382024-04-17T17:14:54.517Z28/30 Write block 583 len 20480 data: 6 6 7 7 5
67392024-04-17T17:14:54.517Z29/30 Read block 258 len 28672
67402024-04-17T17:14:54.521Z30/30 Write block 336 len 40960 data: 7 9 9 9 8 8 10 10 9 10
67412024-04-17T17:14:54.521Z----------------------------------------------------------------
67422024-04-17T17:14:54.521Z Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:26
67432024-04-17T17:14:54.521ZStates: Active Active Active
67442024-04-17T17:14:54.521ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
67452024-04-17T17:14:54.521Z 5 Acked 1004 Write 7 Done Done Done false
67462024-04-17T17:14:54.521Z 6 Acked 1005 Flush 0 Done Sent Done false
67472024-04-17T17:14:54.522Z 7 Acked 1006 Write 9 Done Sent Done false
67482024-04-17T17:14:54.522Z 8 Acked 1007 Read 7 Done Sent Done false
67492024-04-17T17:14:54.522Z 9 Acked 1008 Read 5 Done Sent Done false
67502024-04-17T17:14:54.522Z 10 Acked 1009 Write 4 Done Sent Done false
67512024-04-17T17:14:54.522Z 11 Acked 1010 Write 8 Done Sent Done false
67522024-04-17T17:14:54.522Z 12 Acked 1011 Read 6 Done Sent Done false
67532024-04-17T17:14:54.522Z 13 Acked 1012 Read 10 Done Sent Done false
67542024-04-17T17:14:54.522Z 14 Acked 1013 Read 9 Done Sent Done false
67552024-04-17T17:14:54.522Z 15 Acked 1014 Read 7 Done Sent Done false
67562024-04-17T17:14:54.522Z 16 Acked 1015 Read 8 Done Sent Done false
67572024-04-17T17:14:54.522Z 17 Acked 1016 Write 2 Done Sent Done false
67582024-04-17T17:14:54.522Z 18 Acked 1017 Read 7 Done Sent Done false
67592024-04-17T17:14:54.522Z 19 Acked 1018 Write 8 Done Sent Done false
67602024-04-17T17:14:54.522Z 20 Acked 1019 Write 4 Done Sent Done false
67612024-04-17T17:14:54.522Z 21 Acked 1020 Read 4 Done Sent Done false
67622024-04-17T17:14:54.522Z 22 Acked 1021 Read 1 Done Sent Done false
67632024-04-17T17:14:54.522Z 23 Acked 1022 Write 1 Done Sent Done false
67642024-04-17T17:14:54.522Z 24 Acked 1023 Read 5 Done Sent Done false
67652024-04-17T17:14:54.522Z 25 Acked 1024 Read 8 Done Sent Done false
67662024-04-17T17:14:54.522Z 26 Acked 1025 Read 10 Done Sent Done false
67672024-04-17T17:14:54.522Z 27 Acked 1026 Read 6 Done Sent Done false
67682024-04-17T17:14:54.522Z 28 Acked 1027 Write 5 Done Sent Done false
67692024-04-17T17:14:54.523Z 29 Acked 1028 Read 7 Done Sent Done false
67702024-04-17T17:14:54.523Z 30 Acked 1029 Write 10 Sent Sent Sent false
67712024-04-17T17:14:54.523Z STATES DS:0 DS:1 DS:2 TOTAL
67722024-04-17T17:14:54.523Z New 0 0 0 0
67732024-04-17T17:14:54.523Z Sent 1 25 1 27
67742024-04-17T17:14:54.523Z Done 25 1 25 51
67752024-04-17T17:14:54.523Z Skipped 0 0 0 0
67762024-04-17T17:14:54.523Z Error 0 0 0 0
67772024-04-17T17:14:54.523ZLast Flush: 1005 1003 1005
67782024-04-17T17:14:54.523ZDownstairs last five completed: 1003 1002 1001 1000
67792024-04-17T17:14:54.523ZUpstairs last five completed: 30 29 28 27 26
67802024-04-17T17:14:54.523Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:14:53.127523577Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1416}
67812024-04-17T17:14:54.523ZWrote out file "/tmp/test_repair_verify.data"
67822024-04-17T17:14:54.527Z
67832024-04-17T17:14:54.530ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
67842024-04-17T17:14:54.554ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
67852024-04-17T17:14:54.554Z 0 000-019 60 60 60 109 109 109 F F F
67862024-04-17T17:14:54.561Z 1 020-039 60 60 60 109 109 109 F F F
67872024-04-17T17:14:54.561Z 2 040-059 61 61 61 110 110 110 T F T <---
67882024-04-17T17:14:54.561Z 3 060-079 61 61 61 110 110 110 F F F
67892024-04-17T17:14:54.561Z 4 080-099 61 61 61 110 110 110 T F T <---
67902024-04-17T17:14:54.561Z 5 100-119 61 61 61 110 110 110 F F F
67912024-04-17T17:14:54.561Z 6 120-139 59 59 59 106 106 106 T F T <---
67922024-04-17T17:14:54.561Z 7 140-159 59 59 59 106 106 106 F F F
67932024-04-17T17:14:54.561Z 8 160-179 59 59 59 106 106 106 T F T <---
67942024-04-17T17:14:54.561Z 9 180-199 61 61 61 110 110 110 F F F
67952024-04-17T17:14:54.561Z 10 200-219 61 61 61 110 110 110 F F F
67962024-04-17T17:14:54.561Z 11 220-239 51 51 51 90 90 90 F F F
67972024-04-17T17:14:54.561Z 12 240-259 60 60 60 108 108 108 F F F
67982024-04-17T17:14:54.561Z 13 260-279 56 56 56 99 99 99 F F F
67992024-04-17T17:14:54.561Z 14 280-299 60 60 60 108 108 108 F F F
68002024-04-17T17:14:54.561Z 15 300-319 59 59 59 106 106 106 F F F
68012024-04-17T17:14:54.561Z 16 320-339 60 60 60 108 108 108 T F T <---
68022024-04-17T17:14:54.561Z 17 340-359 60 60 60 108 108 108 T F T <---
68032024-04-17T17:14:54.561Z 18 360-379 47 47 47 85 85 85 F F F
68042024-04-17T17:14:54.561Z 19 380-399 59 59 59 106 106 106 F F F
68052024-04-17T17:14:54.561Z 20 400-419 61 61 61 110 110 110 F F F
68062024-04-17T17:14:54.562Z 21 420-439 57 57 57 103 103 103 F F F
68072024-04-17T17:14:54.562Z 22 440-459 62 52 62 113 94 113 F T F <---
68082024-04-17T17:14:54.562Z 23 460-479 57 57 57 103 103 103 F F F
68092024-04-17T17:14:54.562Z 24 480-499 62 62 62 111 111 111 T F T <---
68102024-04-17T17:14:54.562Z 25 500-519 62 62 62 111 111 111 F F F
68112024-04-17T17:14:54.562Z 26 520-539 61 61 61 110 110 110 F F F
68122024-04-17T17:14:54.562Z 27 540-559 59 59 59 106 106 106 F F F
68132024-04-17T17:14:54.562Z 28 560-579 61 61 61 110 110 110 T F T <---
68142024-04-17T17:14:54.562Z 29 580-599 55 55 55 98 98 98 T F T <---
68152024-04-17T17:14:54.562ZMax gen: 62, Max flush: 113
68162024-04-17T17:14:54.562ZError: Difference in extent metadata found!
68172024-04-17T17:14:54.562ZA Difference in extent metadata is expected here
68182024-04-17T17:14:54.562ZOn loop 31
68192024-04-17T17:14:54.562Z
68202024-04-17T17:14:54.562ZVerifying data now
68212024-04-17T17:14:59.040ZNo change after verify
68222024-04-17T17:14:59.040ZLoop: 31 Downstairs dump after verify (and repair):
68232024-04-17T17:14:59.051ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
68242024-04-17T17:14:59.051Z 0 000-019 60 60 60 109 109 109 F F F
68252024-04-17T17:14:59.055Z 1 020-039 60 60 60 109 109 109 F F F
68262024-04-17T17:14:59.055Z 2 040-059 63 63 63 114 114 114 F F F
68272024-04-17T17:14:59.055Z 3 060-079 61 61 61 110 110 110 F F F
68282024-04-17T17:14:59.055Z 4 080-099 63 63 63 114 114 114 F F F
68292024-04-17T17:14:59.055Z 5 100-119 61 61 61 110 110 110 F F F
68302024-04-17T17:14:59.055Z 6 120-139 63 63 63 114 114 114 F F F
68312024-04-17T17:14:59.055Z 7 140-159 59 59 59 106 106 106 F F F
68322024-04-17T17:14:59.055Z 8 160-179 63 63 63 114 114 114 F F F
68332024-04-17T17:14:59.055Z 9 180-199 61 61 61 110 110 110 F F F
68342024-04-17T17:14:59.055Z 10 200-219 61 61 61 110 110 110 F F F
68352024-04-17T17:14:59.056Z 11 220-239 51 51 51 90 90 90 F F F
68362024-04-17T17:14:59.056Z 12 240-259 60 60 60 108 108 108 F F F
68372024-04-17T17:14:59.056Z 13 260-279 56 56 56 99 99 99 F F F
68382024-04-17T17:14:59.056Z 14 280-299 60 60 60 108 108 108 F F F
68392024-04-17T17:14:59.056Z 15 300-319 59 59 59 106 106 106 F F F
68402024-04-17T17:14:59.056Z 16 320-339 63 63 63 114 114 114 F F F
68412024-04-17T17:14:59.056Z 17 340-359 63 63 63 114 114 114 F F F
68422024-04-17T17:14:59.056Z 18 360-379 47 47 47 85 85 85 F F F
68432024-04-17T17:14:59.056Z 19 380-399 59 59 59 106 106 106 F F F
68442024-04-17T17:14:59.056Z 20 400-419 61 61 61 110 110 110 F F F
68452024-04-17T17:14:59.056Z 21 420-439 57 57 57 103 103 103 F F F
68462024-04-17T17:14:59.056Z 22 440-459 62 62 62 113 113 113 F F F
68472024-04-17T17:14:59.056Z 23 460-479 57 57 57 103 103 103 F F F
68482024-04-17T17:14:59.056Z 24 480-499 63 63 63 114 114 114 F F F
68492024-04-17T17:14:59.056Z 25 500-519 62 62 62 111 111 111 F F F
68502024-04-17T17:14:59.056Z 26 520-539 61 61 61 110 110 110 F F F
68512024-04-17T17:14:59.056Z 27 540-559 59 59 59 106 106 106 F F F
68522024-04-17T17:14:59.056Z 28 560-579 63 63 63 114 114 114 F F F
68532024-04-17T17:14:59.056Z 29 580-599 63 63 63 114 114 114 F F F
68542024-04-17T17:14:59.056ZMax gen: 63, Max flush: 114
68552024-04-17T17:14:59.056Z
68562024-04-17T17:14:59.057ZBegin loop 32
68572024-04-17T17:14:59.057ZDownstairs to restart: 1
68582024-04-17T17:14:59.074Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.681341943Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424}
68592024-04-17T17:14:59.074Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.681595008Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424}
68602024-04-17T17:14:59.081Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
68612024-04-17T17:14:59.081Z2024-04-17T17:14:57.681638831Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424}
68622024-04-17T17:14:59.081Z{"msg":"Crucible 90e76e91-aae4-4f7c-b06a-1799b30dd8e3 has session id: 29480cab-997d-4445-b2d6-7e34c1918936","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.681689874Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68632024-04-17T17:14:59.081Z{"msg":"Upstairs opts: Upstairs UUID: 90e76e91-aae4-4f7c-b06a-1799b30dd8e3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.681744957Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68642024-04-17T17:14:59.081Z{"msg":"Crucible stats registered with UUID: 90e76e91-aae4-4f7c-b06a-1799b30dd8e3","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.681769978Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68652024-04-17T17:14:59.081Z{"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.681806251Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424}
68662024-04-17T17:14:59.081Z{"msg":"90e76e91-aae4-4f7c-b06a-1799b30dd8e3 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.681822562Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68672024-04-17T17:14:59.081Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.681845203Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"client":"0","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68682024-04-17T17:14:59.081Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.681872335Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"client":"0","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68692024-04-17T17:14:59.082Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.681917267Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"client":"1","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68702024-04-17T17:14:59.082Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.681944759Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"client":"1","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68712024-04-17T17:14:59.082Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.681972771Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"client":"2","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68722024-04-17T17:14:59.082Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.682003133Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"client":"2","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68732024-04-17T17:14:59.082Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.682031265Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68742024-04-17T17:14:59.082Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.682069317Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68752024-04-17T17:14:59.082Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.682148232Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68762024-04-17T17:14:59.082Z{"msg":"ds_connection connected from Ok(127.0.0.1:59366)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.682200985Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68772024-04-17T17:14:59.082Z{"msg":"ds_connection connected from Ok(127.0.0.1:35442)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.682233047Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68782024-04-17T17:14:59.082Z{"msg":"ds_connection connected from Ok(127.0.0.1:36247)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.682275619Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68792024-04-17T17:14:59.082Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.682873905Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"client":"2","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68802024-04-17T17:14:59.082Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.682909758Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"client":"2","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68812024-04-17T17:14:59.082Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.682925048Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"client":"0","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68822024-04-17T17:14:59.082Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.68295402Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"client":"0","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68832024-04-17T17:14:59.082Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.683355455Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"client":"1","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68842024-04-17T17:14:59.082Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.683401968Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"client":"1","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68852024-04-17T17:14:59.082Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.683426309Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"client":"2","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68862024-04-17T17:14:59.082Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.683575998Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"client":"0","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68872024-04-17T17:14:59.082Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.684074068Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"client":"0","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68882024-04-17T17:14:59.082Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.684115711Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"client":"0","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68892024-04-17T17:14:59.082Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.684131582Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68902024-04-17T17:14:59.082Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.684145242Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"client":"1","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68912024-04-17T17:14:59.082Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.684180525Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"client":"2","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68922024-04-17T17:14:59.083Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.684202716Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"client":"2","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68932024-04-17T17:14:59.083Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.684217067Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68942024-04-17T17:14:59.083Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.684643803Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"client":"1","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68952024-04-17T17:14:59.083Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.684680895Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"client":"1","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68962024-04-17T17:14:59.083Z{"msg":"[0]R flush_numbers[0..12]: [109, 109, 114, 110, 114, 110, 114, 106, 114, 110, 110, 90]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.684696466Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"":"downstairs","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68972024-04-17T17:14:59.083Z{"msg":"[0]R generation[0..12]: [60, 60, 63, 61, 63, 61, 63, 59, 63, 61, 61, 51]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.684717517Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"":"downstairs","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68982024-04-17T17:14:59.083Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.684732308Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"":"downstairs","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
68992024-04-17T17:14:59.083Z{"msg":"[1]R flush_numbers[0..12]: [109, 109, 114, 110, 114, 110, 114, 106, 114, 110, 110, 90]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.684752679Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"":"downstairs","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
69002024-04-17T17:14:59.083Z{"msg":"[1]R generation[0..12]: [60, 60, 63, 61, 63, 61, 63, 59, 63, 61, 61, 51]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.68476684Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"":"downstairs","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
69012024-04-17T17:14:59.083Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.684793532Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"":"downstairs","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
69022024-04-17T17:14:59.083ZWait for a query_work_queue command to finish before sending IO
69032024-04-17T17:14:59.083Z{"msg":"[2]R flush_numbers[0..12]: [109, 109, 114, 110, 114, 110, 114, 106, 114, 110, 110, 90]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.684811983Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"":"downstairs","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
69042024-04-17T17:14:59.083Z{"msg":"[2]R generation[0..12]: [60, 60, 63, 61, 63, 61, 63, 59, 63, 61, 61, 51]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.684837455Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"":"downstairs","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
69052024-04-17T17:14:59.083Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.684855025Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"":"downstairs","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
69062024-04-17T17:14:59.083Z{"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.684876237Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"":"downstairs","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
69072024-04-17T17:14:59.083Z{"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.684889568Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"":"downstairs","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
69082024-04-17T17:14:59.083Z{"msg":"Next flush: 115","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.684909259Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"":"downstairs","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
69092024-04-17T17:14:59.083Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.68492211Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"":"downstairs","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
69102024-04-17T17:14:59.083Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.684942501Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
69112024-04-17T17:14:59.083Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.684954902Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"client":"0","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
69122024-04-17T17:14:59.083Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.684974843Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"client":"1","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
69132024-04-17T17:14:59.083Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.684989574Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"client":"2","":"downstairs","session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
69142024-04-17T17:14:59.083Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.685030236Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
69152024-04-17T17:14:59.084Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.685043377Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
69162024-04-17T17:14:59.084Z{"msg":"90e76e91-aae4-4f7c-b06a-1799b30dd8e3 is now active with session: 29480cab-997d-4445-b2d6-7e34c1918936","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.685056238Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
69172024-04-17T17:14:59.084Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.685068729Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424,"session_id":"29480cab-997d-4445-b2d6-7e34c1918936"}
69182024-04-17T17:14:59.084Z{"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:14:57.685081569Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424}
69192024-04-17T17:14:59.088ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
69202024-04-17T17:14:59.088ZLoading write count information from file "/tmp/test_repair_verify.data"
69212024-04-17T17:14:59.088ZRun Repair workload
69222024-04-17T17:14:59.088Z01/30 Write block 331 len 36864 data: 3 3 4 6 7 8 10 10 10
69232024-04-17T17:14:59.088Z02/30 Read block 19 len 40960
69242024-04-17T17:14:59.088Z03/30 Read block 400 len 32768
69252024-04-17T17:14:59.088Z04/30 Write block 543 len 40960 data: 6 5 3 3 4 5 5 5 6 7
69262024-04-17T17:14:59.088Z05/30 Write block 211 len 40960 data: 3 4 3 4 5 4 3 4 5 5
69272024-04-17T17:14:59.088Z06/30 Flush
69282024-04-17T17:14:59.092Z07/30 Write block 506 len 28672 data: 5 7 7 6 5 5 6
69292024-04-17T17:14:59.092Z08/30 Read block 436 len 4096
69302024-04-17T17:14:59.092Z09/30 Flush
69312024-04-17T17:14:59.096Z10/30 Write block 121 len 8192 data: 6 4
69322024-04-17T17:14:59.096Z11/30 Flush
69332024-04-17T17:14:59.099Z12/30 Write block 447 len 20480 data: 4 5 7 7 5
69342024-04-17T17:14:59.100Z13/30 Read block 343 len 40960
69352024-04-17T17:14:59.100Z14/30 Write block 586 len 20480 data: 8 6 4 4 5
69362024-04-17T17:14:59.100Z15/30 Read block 516 len 4096
69372024-04-17T17:14:59.100Z16/30 Read block 289 len 4096
69382024-04-17T17:14:59.100Z17/30 Read block 479 len 24576
69392024-04-17T17:14:59.103Z18/30 Read block 15 len 20480
69402024-04-17T17:14:59.103Z19/30 Read block 297 len 16384
69412024-04-17T17:14:59.103Z20/30 Write block 366 len 16384 data: 3 3 5 5
69422024-04-17T17:14:59.104Z21/30 Write block 67 len 8192 data: 6 5
69432024-04-17T17:14:59.104Z22/30 Read block 327 len 4096
69442024-04-17T17:14:59.107Z23/30 Read block 470 len 16384
69452024-04-17T17:14:59.107Z24/30 Read block 566 len 40960
69462024-04-17T17:14:59.107Z25/30 Write block 80 len 8192 data: 3 3
69472024-04-17T17:14:59.107Z26/30 Write block 223 len 40960 data: 6 7 7 8 8 6 5 3 3 4
69482024-04-17T17:14:59.107Z27/30 Read block 9 len 36864
69492024-04-17T17:14:59.111Z28/30 Read block 566 len 8192
69502024-04-17T17:14:59.112Z29/30 Read block 589 len 40960
69512024-04-17T17:14:59.112Z30/30 Write block 35 len 4096 data: 8
69522024-04-17T17:14:59.112Z----------------------------------------------------------------
69532024-04-17T17:14:59.112Z Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:24
69542024-04-17T17:14:59.112ZStates: Active Active Active
69552024-04-17T17:14:59.112ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
69562024-04-17T17:14:59.112Z 7 Acked 1006 Write 7 Done Done Done false
69572024-04-17T17:14:59.112Z 8 Acked 1007 Read 1 Done Done Done false
69582024-04-17T17:14:59.112Z 9 Acked 1008 Flush 0 Done Sent Done false
69592024-04-17T17:14:59.112Z 10 Acked 1009 Write 2 Done Sent Done false
69602024-04-17T17:14:59.112Z 11 Acked 1010 Flush 0 Done Sent Done false
69612024-04-17T17:14:59.112Z 12 Acked 1011 Write 5 Done Sent Done false
69622024-04-17T17:14:59.112Z 13 Acked 1012 Read 10 Done Sent Done false
69632024-04-17T17:14:59.112Z 14 Acked 1013 Write 5 Done Sent Done false
69642024-04-17T17:14:59.112Z 15 Acked 1014 Read 1 Done Sent Done false
69652024-04-17T17:14:59.112Z 16 Acked 1015 Read 1 Done Sent Done false
69662024-04-17T17:14:59.112Z 17 Acked 1016 Read 6 Done Sent Done false
69672024-04-17T17:14:59.112Z 18 Acked 1017 Read 5 Done Sent Done false
69682024-04-17T17:14:59.112Z 19 Acked 1018 Read 4 Done Sent Done false
69692024-04-17T17:14:59.112Z 20 Acked 1019 Write 4 Done Sent Done false
69702024-04-17T17:14:59.112Z 21 Acked 1020 Write 2 Done Sent Done false
69712024-04-17T17:14:59.112Z 22 Acked 1021 Read 1 Done Sent Done false
69722024-04-17T17:14:59.112Z 23 Acked 1022 Read 4 Done Sent Done false
69732024-04-17T17:14:59.112Z 24 Acked 1023 Read 10 Done Sent Done false
69742024-04-17T17:14:59.113Z 25 Acked 1024 Write 2 Done Sent Done false
69752024-04-17T17:14:59.113Z 26 Acked 1025 Write 10 Done Sent Done false
69762024-04-17T17:14:59.113Z 27 Acked 1026 Read 9 Done Sent Done false
69772024-04-17T17:14:59.113Z 28 Acked 1027 Read 2 Done Sent Done false
69782024-04-17T17:14:59.113Z 29 Acked 1028 Read 10 Done Sent Done false
69792024-04-17T17:14:59.113Z 30 Acked 1029 Write 1 Sent Sent Sent false
69802024-04-17T17:14:59.113Z STATES DS:0 DS:1 DS:2 TOTAL
69812024-04-17T17:14:59.113Z New 0 0 0 0
69822024-04-17T17:14:59.113Z Sent 1 22 1 24
69832024-04-17T17:14:59.113Z Done 23 2 23 48
69842024-04-17T17:14:59.113Z Skipped 0 0 0 0
69852024-04-17T17:14:59.113Z Error 0 0 0 0
69862024-04-17T17:14:59.113ZLast Flush: 1010 1005 1010
69872024-04-17T17:14:59.113ZDownstairs last five completed: 1005 1004 1003 1002 1001
69882024-04-17T17:14:59.113ZUpstairs last five completed: 30 29 28 27 26
69892024-04-17T17:14:59.113Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:14:57.718039977Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1424}
69902024-04-17T17:14:59.113ZWrote out file "/tmp/test_repair_verify.data"
69912024-04-17T17:14:59.117Z
69922024-04-17T17:14:59.120ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
69932024-04-17T17:14:59.135ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
69942024-04-17T17:14:59.135Z 0 000-019 60 60 60 109 109 109 F F F
69952024-04-17T17:14:59.139Z 1 020-039 60 60 60 109 109 109 T F T <---
69962024-04-17T17:14:59.139Z 2 040-059 63 63 63 114 114 114 F F F
69972024-04-17T17:14:59.139Z 3 060-079 61 61 61 110 110 110 T F T <---
69982024-04-17T17:14:59.139Z 4 080-099 63 63 63 114 114 114 T F T <---
69992024-04-17T17:14:59.139Z 5 100-119 61 61 61 110 110 110 F F F
70002024-04-17T17:14:59.139Z 6 120-139 64 63 64 117 114 117 F F F <---
70012024-04-17T17:14:59.139Z 7 140-159 59 59 59 106 106 106 F F F
70022024-04-17T17:14:59.139Z 8 160-179 63 63 63 114 114 114 F F F
70032024-04-17T17:14:59.139Z 9 180-199 61 61 61 110 110 110 F F F
70042024-04-17T17:14:59.139Z 10 200-219 64 64 64 115 115 115 F F F
70052024-04-17T17:14:59.140Z 11 220-239 64 64 64 115 115 115 T F T <---
70062024-04-17T17:14:59.140Z 12 240-259 60 60 60 108 108 108 F F F
70072024-04-17T17:14:59.140Z 13 260-279 56 56 56 99 99 99 F F F
70082024-04-17T17:14:59.140Z 14 280-299 60 60 60 108 108 108 F F F
70092024-04-17T17:14:59.140Z 15 300-319 59 59 59 106 106 106 F F F
70102024-04-17T17:14:59.140Z 16 320-339 64 64 64 115 115 115 F F F
70112024-04-17T17:14:59.140Z 17 340-359 63 63 63 114 114 114 F F F
70122024-04-17T17:14:59.140Z 18 360-379 47 47 47 85 85 85 T F T <---
70132024-04-17T17:14:59.140Z 19 380-399 59 59 59 106 106 106 F F F
70142024-04-17T17:14:59.140Z 20 400-419 61 61 61 110 110 110 F F F
70152024-04-17T17:14:59.140Z 21 420-439 57 57 57 103 103 103 F F F
70162024-04-17T17:14:59.140Z 22 440-459 62 62 62 113 113 113 T F T <---
70172024-04-17T17:14:59.140Z 23 460-479 57 57 57 103 103 103 F F F
70182024-04-17T17:14:59.140Z 24 480-499 63 63 63 114 114 114 F F F
70192024-04-17T17:14:59.140Z 25 500-519 64 62 64 116 111 116 F T F <---
70202024-04-17T17:14:59.140Z 26 520-539 61 61 61 110 110 110 F F F
70212024-04-17T17:14:59.140Z 27 540-559 64 64 64 115 115 115 F F F
70222024-04-17T17:14:59.140Z 28 560-579 63 63 63 114 114 114 F F F
70232024-04-17T17:14:59.140Z 29 580-599 63 63 63 114 114 114 T F T <---
70242024-04-17T17:14:59.140ZMax gen: 64, Max flush: 117
70252024-04-17T17:14:59.140ZError: Difference in extent metadata found!
70262024-04-17T17:14:59.140ZA Difference in extent metadata is expected here
70272024-04-17T17:14:59.140ZOn loop 32
70282024-04-17T17:14:59.140Z
70292024-04-17T17:14:59.145ZVerifying data now
70302024-04-17T17:15:03.586ZNo change after verify
70312024-04-17T17:15:03.586ZLoop: 32 Downstairs dump after verify (and repair):
70322024-04-17T17:15:03.601ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70332024-04-17T17:15:03.601Z 0 000-019 60 60 60 109 109 109 F F F
70342024-04-17T17:15:03.601Z 1 020-039 65 65 65 118 118 118 F F F
70352024-04-17T17:15:03.601Z 2 040-059 63 63 63 114 114 114 F F F
70362024-04-17T17:15:03.601Z 3 060-079 65 65 65 118 118 118 F F F
70372024-04-17T17:15:03.601Z 4 080-099 65 65 65 118 118 118 F F F
70382024-04-17T17:15:03.601Z 5 100-119 61 61 61 110 110 110 F F F
70392024-04-17T17:15:03.601Z 6 120-139 64 64 64 117 117 117 F F F
70402024-04-17T17:15:03.601Z 7 140-159 59 59 59 106 106 106 F F F
70412024-04-17T17:15:03.601Z 8 160-179 63 63 63 114 114 114 F F F
70422024-04-17T17:15:03.601Z 9 180-199 61 61 61 110 110 110 F F F
70432024-04-17T17:15:03.601Z 10 200-219 64 64 64 115 115 115 F F F
70442024-04-17T17:15:03.601Z 11 220-239 65 65 65 118 118 118 F F F
70452024-04-17T17:15:03.601Z 12 240-259 60 60 60 108 108 108 F F F
70462024-04-17T17:15:03.601Z 13 260-279 56 56 56 99 99 99 F F F
70472024-04-17T17:15:03.601Z 14 280-299 60 60 60 108 108 108 F F F
70482024-04-17T17:15:03.605Z 15 300-319 59 59 59 106 106 106 F F F
70492024-04-17T17:15:03.605Z 16 320-339 64 64 64 115 115 115 F F F
70502024-04-17T17:15:03.605Z 17 340-359 63 63 63 114 114 114 F F F
70512024-04-17T17:15:03.605Z 18 360-379 65 65 65 118 118 118 F F F
70522024-04-17T17:15:03.605Z 19 380-399 59 59 59 106 106 106 F F F
70532024-04-17T17:15:03.605Z 20 400-419 61 61 61 110 110 110 F F F
70542024-04-17T17:15:03.605Z 21 420-439 57 57 57 103 103 103 F F F
70552024-04-17T17:15:03.606Z 22 440-459 65 65 65 118 118 118 F F F
70562024-04-17T17:15:03.606Z 23 460-479 57 57 57 103 103 103 F F F
70572024-04-17T17:15:03.606Z 24 480-499 63 63 63 114 114 114 F F F
70582024-04-17T17:15:03.606Z 25 500-519 64 64 64 116 116 116 F F F
70592024-04-17T17:15:03.606Z 26 520-539 61 61 61 110 110 110 F F F
70602024-04-17T17:15:03.606Z 27 540-559 64 64 64 115 115 115 F F F
70612024-04-17T17:15:03.606Z 28 560-579 63 63 63 114 114 114 F F F
70622024-04-17T17:15:03.606Z 29 580-599 65 65 65 118 118 118 F F F
70632024-04-17T17:15:03.606ZMax gen: 65, Max flush: 118
70642024-04-17T17:15:03.606Z
70652024-04-17T17:15:03.606ZBegin loop 33
70662024-04-17T17:15:03.606ZDownstairs to restart: 2
70672024-04-17T17:15:03.627Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.23354175Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432}
70682024-04-17T17:15:03.627Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.23388812Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432}
70692024-04-17T17:15:03.627Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible"Crucible runtime is spawned
70702024-04-17T17:15:03.627Z,"level":30,"time":"2024-04-17T17:15:02.233928633Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432}
70712024-04-17T17:15:03.635Z{"msg":"Crucible 3f4385b9-1967-408e-9261-2e4d7412e741 has session id: 40df60f3-2ab2-4d4e-ad93-7892b021c85e","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.233952884Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
70722024-04-17T17:15:03.635Z{"msg":"Upstairs opts: Upstairs UUID: 3f4385b9-1967-408e-9261-2e4d7412e741, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.234029669Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
70732024-04-17T17:15:03.635Z{"msg":"Crucible stats registered with UUID: 3f4385b9-1967-408e-9261-2e4d7412e741","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.234068062Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
70742024-04-17T17:15:03.635Z{"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.234089983Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432}
70752024-04-17T17:15:03.635Z{"msg":"3f4385b9-1967-408e-9261-2e4d7412e741 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.234108684Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
70762024-04-17T17:15:03.635Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.234137956Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"client":"0","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
70772024-04-17T17:15:03.635Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.234162327Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"client":"0","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
70782024-04-17T17:15:03.635Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.234185018Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"client":"1","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
70792024-04-17T17:15:03.636Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.234218231Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"client":"1","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
70802024-04-17T17:15:03.636Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.234241532Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"client":"2","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
70812024-04-17T17:15:03.636Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.234273094Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"client":"2","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
70822024-04-17T17:15:03.636Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.234296635Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
70832024-04-17T17:15:03.636Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.234331227Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
70842024-04-17T17:15:03.636Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.234354109Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
70852024-04-17T17:15:03.636Z{"msg":"ds_connection connected from Ok(127.0.0.1:57222)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.234455155Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
70862024-04-17T17:15:03.636Z{"msg":"ds_connection connected from Ok(127.0.0.1:40858)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.234512218Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
70872024-04-17T17:15:03.636Z{"msg":"ds_connection connected from Ok(127.0.0.1:54353)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.234676078Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
70882024-04-17T17:15:03.636Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.235177829Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"client":"1","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
70892024-04-17T17:15:03.636Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.235213971Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"client":"1","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
70902024-04-17T17:15:03.636Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.235231472Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"client":"0","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
70912024-04-17T17:15:03.636Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.235323307Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"client":"0","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
70922024-04-17T17:15:03.636Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.235735343Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"client":"0","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
70932024-04-17T17:15:03.636Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.235868471Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"client":"1","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
70942024-04-17T17:15:03.636Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.236254964Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"client":"0","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
70952024-04-17T17:15:03.636Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.236316998Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"client":"0","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
70962024-04-17T17:15:03.636Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.236368511Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
70972024-04-17T17:15:03.636Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.236411924Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"client":"1","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
70982024-04-17T17:15:03.636Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.236457316Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"client":"1","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
70992024-04-17T17:15:03.636Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.236497639Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
71002024-04-17T17:15:03.636Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.2365225Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"client":"2","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
71012024-04-17T17:15:03.636Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.236537111Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"client":"2","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
71022024-04-17T17:15:03.636Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.236931705Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"client":"2","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
71032024-04-17T17:15:03.637Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.237364041Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"client":"2","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
71042024-04-17T17:15:03.637Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.237407944Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"client":"2","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
71052024-04-17T17:15:03.637Z{"msg":"[0]R flush_numbers[0..12]: [109, 118, 114, 118, 118, 110, 117, 106, 114, 110, 115, 118]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.237445356Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"":"downstairs","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
71062024-04-17T17:15:03.637Z{"msg":"[0]R generation[0..12]: [60, 65, 63, 65, 65, 61, 64, 59, 63, 61, 64, 65]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.237480448Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"":"downstairs","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
71072024-04-17T17:15:03.637Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.23751624Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"":"downstairs","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
71082024-04-17T17:15:03.637Z{"msg":"[1]R flush_numbers[0..12]: [109, 118, 114, 118, 118, 110, 117, 106, 114, 110, 115, 118]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.237541912Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"":"downstairs","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
71092024-04-17T17:15:03.637Z{"msg":"[1]R generation[0..12]: [60, 65, 63, 65, 65, 61, 64, 59, 63, 61, 64, 65]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.237566063Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,Wait for a query_work_queue command to finish before sending IO
71102024-04-17T17:15:03.637Z"":"downstairs","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
71112024-04-17T17:15:03.637Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.237655389Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"":"downstairs","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
71122024-04-17T17:15:03.637Z{"msg":"[2]R flush_numbers[0..12]: [109, 118, 114, 118, 118, 110, 117, 106, 114, 110, 115, 118]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.237699271Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"":"downstairs","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
71132024-04-17T17:15:03.637Z{"msg":"[2]R generation[0..12]: [60, 65, 63, 65, 65, 61, 64, 59, 63, 61, 64, 65]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.237737444Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"":"downstairs","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
71142024-04-17T17:15:03.637Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.237776226Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"":"downstairs","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
71152024-04-17T17:15:03.637Z{"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.237812909Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"":"downstairs","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
71162024-04-17T17:15:03.637Z{"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.237849321Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"":"downstairs","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
71172024-04-17T17:15:03.637Z{"msg":"Next flush: 119","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.237886073Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"":"downstairs","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
71182024-04-17T17:15:03.637Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.237921625Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"":"downstairs","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
71192024-04-17T17:15:03.637Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.237956817Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
71202024-04-17T17:15:03.637Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.237980689Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"client":"0","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
71212024-04-17T17:15:03.637Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.23801059Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"client":"1","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
71222024-04-17T17:15:03.637Z{"msg":"Transition from WaitQuorum to Active","v":0,"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
71232024-04-17T17:15:03.637Zname":"crucible","level":30,"time":"2024-04-17T17:15:02.238029502Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"client":"2","":"downstairs","session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
71242024-04-17T17:15:03.637Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.238042962Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
71252024-04-17T17:15:03.637Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.238062313Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
71262024-04-17T17:15:03.638Z{"msg":"3f4385b9-1967-408e-9261-2e4d7412e741 is now active with session: 40df60f3-2ab2-4d4e-ad93-7892b021c85e","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.238075454Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
71272024-04-17T17:15:03.638Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.238087295Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432,"session_id":"40df60f3-2ab2-4d4e-ad93-7892b021c85e"}
71282024-04-17T17:15:03.638Z{"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:02.238106426Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432}
71292024-04-17T17:15:03.638ZLoading write count information from file "/tmp/test_repair_verify.data"
71302024-04-17T17:15:03.638ZRun Repair workload
71312024-04-17T17:15:03.645Z01/30 Write block 310 len 40960 data: 5 5 8 8 9 8 7 8 8 7
71322024-04-17T17:15:03.645Z02/30 Read block 172 len 12288
71332024-04-17T17:15:03.645Z03/30 Read block 571 len 40960
71342024-04-17T17:15:03.645Z04/30 Write block 202 len 12288 data: 10 11 10
71352024-04-17T17:15:03.645Z05/30 Read block 460 len 40960
71362024-04-17T17:15:03.645Z06/30 Flush
71372024-04-17T17:15:03.645Z07/30 Flush
71382024-04-17T17:15:03.645Z08/30 Flush
71392024-04-17T17:15:03.645Z09/30 Write block 256 len 24576 data: 7 7 6 6 4 5
71402024-04-17T17:15:03.645Z10/30 Read block 271 len 8192
71412024-04-17T17:15:03.649Z11/30 Write block 408 len 40960 data: 4 5 6 8 9 10 11 10 9 8
71422024-04-17T17:15:03.649Z12/30 Read block 337 len 8192
71432024-04-17T17:15:03.649Z13/30 Read block 413 len 12288
71442024-04-17T17:15:03.649Z14/30 Read block 583 len 24576
71452024-04-17T17:15:03.649Z15/30 Write block 452 len 12288 data: 6 6 5
71462024-04-17T17:15:03.649Z16/30 Read block 218 len 24576
71472024-04-17T17:15:03.650Z17/30 Read block 229 len 40960
71482024-04-17T17:15:03.650Z18/30 Write block 467 len 20480 data: 5 4 5 5 6
71492024-04-17T17:15:03.650Z19/30 Write block 430 len 20480 data: 5 4 3 6 6
71502024-04-17T17:15:03.654Z20/30 Read block 246 len 8192
71512024-04-17T17:15:03.654Z21/30 Write block 226 len 20480 data: 9 9 7 6 4
71522024-04-17T17:15:03.654Z22/30 Write block 98 len 24576 data: 6 8 8 8 6 8
71532024-04-17T17:15:03.654Z23/30 Write block 418 len 12288 data: 7 7 6
71542024-04-17T17:15:03.654Z24/30 Write block 21 len 24576 data: 8 9 8 8 8 8
71552024-04-17T17:15:03.658Z25/30 Read block 185 len 24576
71562024-04-17T17:15:03.658Z26/30 Read block 488 len 16384
71572024-04-17T17:15:03.658Z27/30 Write block 58 len 36864 data: 7 7 7 7 7 7 8 6 6
71582024-04-17T17:15:03.658Z28/30 Write block 70 len 16384 data: 6 6 5 4
71592024-04-17T17:15:03.662Z29/30 Read block 96 len 12288
71602024-04-17T17:15:03.662Z30/30 Write block 368 len 40960 data: 6 6 5 7 4 5 4 4 2 4
71612024-04-17T17:15:03.662Z----------------------------------------------------------------
71622024-04-17T17:15:03.662Z Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:30
71632024-04-17T17:15:03.662ZStates: Active Active Active
71642024-04-17T17:15:03.662ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
71652024-04-17T17:15:03.662Z 1 Acked 1000 Write 10 Done Done Done false
71662024-04-17T17:15:03.662Z 2 Acked 1001 Read 3 Done Done Done false
71672024-04-17T17:15:03.662Z 3 Acked 1002 Read 10 Done Done Done false
71682024-04-17T17:15:03.662Z 4 Acked 1003 Write 3 Done Done Done false
71692024-04-17T17:15:03.662Z 5 Acked 1004 Read 10 Done Done Sent false
71702024-04-17T17:15:03.662Z 6 Acked 1005 Flush 0 Done Done Sent false
71712024-04-17T17:15:03.663Z 7 Acked 1006 Flush 0 Done Done Sent false
71722024-04-17T17:15:03.663Z 8 Acked 1007 Flush 0 Done Done Sent false
71732024-04-17T17:15:03.663Z 9 Acked 1008 Write 6 Done Done Sent false
71742024-04-17T17:15:03.663Z 10 Acked 1009 Read 2 Done Done Sent false
71752024-04-17T17:15:03.663Z 11 Acked 1010 Write 10 Done Done Sent false
71762024-04-17T17:15:03.663Z 12 Acked 1011 Read 2 Done Done Sent false
71772024-04-17T17:15:03.663Z 13 Acked 1012 Read 3 Done Done Sent false
71782024-04-17T17:15:03.663Z 14 Acked 1013 Read 6 Done Done Sent false
71792024-04-17T17:15:03.663Z 15 Acked 1014 Write 3 Done Done Sent false
71802024-04-17T17:15:03.663Z 16 Acked 1015 Read 6 Done Done Sent false
71812024-04-17T17:15:03.663Z 17 Acked 1016 Read 10 Done Done Sent false
71822024-04-17T17:15:03.663Z 18 Acked 1017 Write 5 Done Done Sent false
71832024-04-17T17:15:03.663Z 19 Acked 1018 Write 5 Done Done Sent false
71842024-04-17T17:15:03.663Z 20 Acked 1019 Read 2 Done Done Sent false
71852024-04-17T17:15:03.663Z 21 Acked 1020 Write 5 Done Done Sent false
71862024-04-17T17:15:03.663Z 22 Acked 1021 Write 6 Done Done Sent false
71872024-04-17T17:15:03.663Z 23 Acked 1022 Write 3 Done Done Sent false
71882024-04-17T17:15:03.663Z 24 Acked 1023 Write 6 Done Done Sent false
71892024-04-17T17:15:03.663Z 25 Acked 1024 Read 6 Done Done Sent false
71902024-04-17T17:15:03.663Z 26 Acked 1025 Read 4 Done Done Sent false
71912024-04-17T17:15:03.663Z 27 Acked 1026 Write 9 Done Done Sent false
71922024-04-17T17:15:03.663Z 28 Acked 1027 Write 4 Done Done Sent false
71932024-04-17T17:15:03.663Z 29 Acked 1028 Read 3 Done Done Sent false
71942024-04-17T17:15:03.663Z 30 Acked 1029 Write 10 Sent Sent Sent false
71952024-04-17T17:15:03.664Z STATES DS:0 DS:1 DS:2 TOTAL
71962024-04-17T17:15:03.664Z New 0 0 0 0
71972024-04-17T17:15:03.664Z Sent 1 1 26 28
71982024-04-17T17:15:03.664Z Done 29 29 4 62
71992024-04-17T17:15:03.664Z Skipped 0 0 0 0
72002024-04-17T17:15:03.664Z Error 0 0 0 0
72012024-04-17T17:15:03.664ZLast Flush: 1007 1007 0
72022024-04-17T17:15:03.664ZDownstairs last five completed:
72032024-04-17T17:15:03.664ZUpstairs last five completed: 30 29 28 27 26
72042024-04-17T17:15:03.664Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:15:02.268326098Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1432}
72052024-04-17T17:15:03.664ZWrote out file "/tmp/test_repair_verify.data"
72062024-04-17T17:15:03.668Z
72072024-04-17T17:15:03.672ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
72082024-04-17T17:15:03.690ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
72092024-04-17T17:15:03.690Z 0 000-019 60 60 60 109 109 109 F F F
72102024-04-17T17:15:03.690Z 1 020-039 65 65 65 118 118 118 T T F <---
72112024-04-17T17:15:03.690Z 2 040-059 63 63 63 114 114 114 T T F <---
72122024-04-17T17:15:03.690Z 3 060-079 65 65 65 118 118 118 T T F <---
72132024-04-17T17:15:03.690Z 4 080-099 65 65 65 118 118 118 T T F <---
72142024-04-17T17:15:03.694Z 5 100-119 61 61 61 110 110 110 T T F <---
72152024-04-17T17:15:03.694Z 6 120-139 64 64 64 117 117 117 F F F
72162024-04-17T17:15:03.694Z 7 140-159 59 59 59 106 106 106 F F F
72172024-04-17T17:15:03.694Z 8 160-179 63 63 63 114 114 114 F F F
72182024-04-17T17:15:03.694Z 9 180-199 61 61 61 110 110 110 F F F
72192024-04-17T17:15:03.694Z 10 200-219 66 66 64 119 119 115 F F T <---
72202024-04-17T17:15:03.694Z 11 220-239 65 65 65 118 118 118 T T F <---
72212024-04-17T17:15:03.694Z 12 240-259 60 60 60 108 108 108 T T F <---
72222024-04-17T17:15:03.694Z 13 260-279 56 56 56 99 99 99 T T F <---
72232024-04-17T17:15:03.694Z 14 280-299 60 60 60 108 108 108 F F F
72242024-04-17T17:15:03.695Z 15 300-319 66 66 59 119 119 106 F F T <---
72252024-04-17T17:15:03.695Z 16 320-339 64 64 64 115 115 115 F F F
72262024-04-17T17:15:03.695Z 17 340-359 63 63 63 114 114 114 F F F
72272024-04-17T17:15:03.695Z 18 360-379 65 65 65 118 118 118 T T F <---
72282024-04-17T17:15:03.695Z 19 380-399 59 59 59 106 106 106 F F F
72292024-04-17T17:15:03.695Z 20 400-419 61 61 61 110 110 110 T T F <---
72302024-04-17T17:15:03.695Z 21 420-439 57 57 57 103 103 103 T T F <---
72312024-04-17T17:15:03.695Z 22 440-459 65 65 65 118 118 118 T T F <---
72322024-04-17T17:15:03.695Z 23 460-479 57 57 57 103 103 103 T T F <---
72332024-04-17T17:15:03.695Z 24 480-499 63 63 63 114 114 114 F F F
72342024-04-17T17:15:03.695Z 25 500-519 64 64 64 116 116 116 F F F
72352024-04-17T17:15:03.695Z 26 520-539 61 61 61 110 110 110 F F F
72362024-04-17T17:15:03.695Z 27 540-559 64 64 64 115 115 115 F F F
72372024-04-17T17:15:03.695Z 28 560-579 63 63 63 114 114 114 F F F
72382024-04-17T17:15:03.695Z 29 580-599 65 65 65 118 118 118 F F F
72392024-04-17T17:15:03.695ZMax gen: 66, Max flush: 119
72402024-04-17T17:15:03.695ZError: Difference in extent metadata found!
72412024-04-17T17:15:03.695ZA Difference in extent metadata is expected here
72422024-04-17T17:15:03.695ZOn loop 33
72432024-04-17T17:15:03.695Z
72442024-04-17T17:15:03.700ZVerifying data now
72452024-04-17T17:15:18.387ZNo change after verify
72462024-04-17T17:15:18.387ZLoop: 33 Downstairs dump after verify (and repair):
72472024-04-17T17:15:18.402ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
72482024-04-17T17:15:18.402Z 0 000-019 60 60 60 109 109 109 F F F
72492024-04-17T17:15:18.402Z 1 020-039 67 67 67 120 120 120 F F F
72502024-04-17T17:15:18.403Z 2 040-059 67 67 67 120 120 120 F F F
72512024-04-17T17:15:18.403Z 3 060-079 67 67 67 120 120 120 F F F
72522024-04-17T17:15:18.403Z 4 080-099 67 67 67 120 120 120 F F F
72532024-04-17T17:15:18.403Z 5 100-119 67 67 67 120 120 120 F F F
72542024-04-17T17:15:18.403Z 6 120-139 64 64 64 117 117 117 F F F
72552024-04-17T17:15:18.406Z 7 140-159 59 59 59 106 106 106 F F F
72562024-04-17T17:15:18.406Z 8 160-179 63 63 63 114 114 114 F F F
72572024-04-17T17:15:18.407Z 9 180-199 61 61 61 110 110 110 F F F
72582024-04-17T17:15:18.407Z 10 200-219 66 66 66 119 119 119 F F F
72592024-04-17T17:15:18.407Z 11 220-239 67 67 67 120 120 120 F F F
72602024-04-17T17:15:18.407Z 12 240-259 67 67 67 120 120 120 F F F
72612024-04-17T17:15:18.407Z 13 260-279 67 67 67 120 120 120 F F F
72622024-04-17T17:15:18.407Z 14 280-299 60 60 60 108 108 108 F F F
72632024-04-17T17:15:18.407Z 15 300-319 66 66 66 119 119 119 F F F
72642024-04-17T17:15:18.407Z 16 320-339 64 64 64 115 115 115 F F F
72652024-04-17T17:15:18.407Z 17 340-359 63 63 63 114 114 114 F F F
72662024-04-17T17:15:18.407Z 18 360-379 67 67 67 120 120 120 F F F
72672024-04-17T17:15:18.407Z 19 380-399 59 59 59 106 106 106 F F F
72682024-04-17T17:15:18.407Z 20 400-419 67 67 67 120 120 120 F F F
72692024-04-17T17:15:18.407Z 21 420-439 67 67 67 120 120 120 F F F
72702024-04-17T17:15:18.407Z 22 440-459 67 67 67 120 120 120 F F F
72712024-04-17T17:15:18.407Z 23 460-479 67 67 67 120 120 120 F F F
72722024-04-17T17:15:18.407Z 24 480-499 63 63 63 114 114 114 F F F
72732024-04-17T17:15:18.407Z 25 500-519 64 64 64 116 116 116 F F F
72742024-04-17T17:15:18.407Z 26 520-539 61 61 61 110 110 110 F F F
72752024-04-17T17:15:18.407Z 27 540-559 64 64 64 115 115 115 F F F
72762024-04-17T17:15:18.407Z 28 560-579 63 63 63 114 114 114 F F F
72772024-04-17T17:15:18.407Z 29 580-599 65 65 65 118 118 118 F F F
72782024-04-17T17:15:18.407ZMax gen: 67, Max flush: 120
72792024-04-17T17:15:18.407Z
72802024-04-17T17:15:18.408ZBegin loop 34
72812024-04-17T17:15:18.408ZDownstairs to restart: 2
72822024-04-17T17:15:18.426Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.03363603Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440}
72832024-04-17T17:15:18.426Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.033936698Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440}
72842024-04-17T17:15:18.434Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30Crucible runtime is spawned
72852024-04-17T17:15:18.434Z,"time":"2024-04-17T17:15:17.033986251Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440}
72862024-04-17T17:15:18.434Z{"msg":"Crucible d1bc5462-4f74-4a95-ad4c-fbb933a0ec43 has session id: 15c21756-da32-4e76-9d38-3cc41e6824d0","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.034018583Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
72872024-04-17T17:15:18.434Z{"msg":"Upstairs opts: Upstairs UUID: d1bc5462-4f74-4a95-ad4c-fbb933a0ec43, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.034088627Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
72882024-04-17T17:15:18.434Z{"msg":"Crucible stats registered with UUID: d1bc5462-4f74-4a95-ad4c-fbb933a0ec43","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.034128589Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
72892024-04-17T17:15:18.434Z{"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.034145931Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440}
72902024-04-17T17:15:18.434Z{"msg":"d1bc5462-4f74-4a95-ad4c-fbb933a0ec43 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.034231436Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
72912024-04-17T17:15:18.435Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.034261478Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"client":"0","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
72922024-04-17T17:15:18.435Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.034279469Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"client":"0","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
72932024-04-17T17:15:18.435Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.034311471Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"client":"1","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
72942024-04-17T17:15:18.435Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.034328692Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"client":"1","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
72952024-04-17T17:15:18.435Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.034345253Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"client":"2","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
72962024-04-17T17:15:18.435Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.034382495Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"client":"2","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
72972024-04-17T17:15:18.435Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.034419427Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
72982024-04-17T17:15:18.435Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.034445359Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
72992024-04-17T17:15:18.435Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.03446286Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73002024-04-17T17:15:18.435Z{"msg":"ds_connection connected from Ok(127.0.0.1:36400)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.034754598Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73012024-04-17T17:15:18.435Z{"msg":"ds_connection connected from Ok(127.0.0.1:40639)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.034815161Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73022024-04-17T17:15:18.435Z{"msg":"ds_connection connected from Ok(127.0.0.1:45840)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.034851033Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73032024-04-17T17:15:18.435Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.035491202Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"client":"0","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73042024-04-17T17:15:18.435Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.035534985Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"client":"0","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73052024-04-17T17:15:18.435Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.035665573Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"client":"1","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73062024-04-17T17:15:18.435Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.035727126Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"client":"1","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73072024-04-17T17:15:18.435Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.035775499Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"client":"2","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73082024-04-17T17:15:18.435Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.035817212Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"client":"2","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73092024-04-17T17:15:18.435Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.036063167Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"client":"0","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73102024-04-17T17:15:18.435Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.036210026Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"client":"1","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73112024-04-17T17:15:18.435Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.036456401Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"client":"2","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73122024-04-17T17:15:18.435Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.036591979Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"client":"0","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73132024-04-17T17:15:18.436Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.036629591Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"client":"0","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73142024-04-17T17:15:18.436Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.036647502Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73152024-04-17T17:15:18.436Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.036673184Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"client":"1","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73162024-04-17T17:15:18.436Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.036699165Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"client":"1","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73172024-04-17T17:15:18.436Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.036724337Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73182024-04-17T17:15:18.436Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.03694136Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"client":"2","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73192024-04-17T17:15:18.436Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.037010434Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"client":"2","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73202024-04-17T17:15:18.436Z{"msg":"[0]R flush_numbers[0..12]: [109, 120, 120, 120, 120, 120, 117, 106, 114, 110, 119, 120]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.037040606Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"":"downstairs","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73212024-04-17T17:15:18.436Z{"msg":"[0]R generation[0..12]: [60, 67, 67, 67, 67, 67, 64, 59, 63, 61, 66, 67]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.037056077Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"":"downstairs","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73222024-04-17T17:15:18.436Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.037077348Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"":"downstairs","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73232024-04-17T17:15:18.436Z{"msg":"[1]R flush_numbers[0..12]: [109, 120, 120, 120, 120, 120, 117, 106, 114, 110, 119, 120]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.037092109Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"":"downstairs","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73242024-04-17T17:15:18.436Z{"msg":"Wait for a query_work_queue command to finish before sending IO
73252024-04-17T17:15:18.436Z[1]R generation[0..12]: [60, 67, 67, 67, 67, 67, 64, 59, 63, 61, 66, 67]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.03710909Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"":"downstairs","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73262024-04-17T17:15:18.436Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.037124471Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"":"downstairs","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73272024-04-17T17:15:18.436Z{"msg":"[2]R flush_numbers[0..12]: [109, 120, 120, 120, 120, 120, 117, 106, 114, 110, 119, 120]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.037138432Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"":"downstairs","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73282024-04-17T17:15:18.436Z{"msg":"[2]R generation[0..12]: [60, 67, 67, 67, 67, 67, 64, 59, 63, 61, 66, 67]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.037159263Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"":"downstairs","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73292024-04-17T17:15:18.436Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.037173724Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"":"downstairs","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73302024-04-17T17:15:18.436Z{"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.037193735Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"":"downstairs","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73312024-04-17T17:15:18.436Z{"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.037207196Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"":"downstairs","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73322024-04-17T17:15:18.436Z{"msg":"Next flush: 121","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.037220497Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"":"downstairs","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73332024-04-17T17:15:18.436Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.037240348Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"":"downstairs","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73342024-04-17T17:15:18.436Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.037253569Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73352024-04-17T17:15:18.436Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.03726592Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"client":"0","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73362024-04-17T17:15:18.437Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.037285491Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"client":"1","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73372024-04-17T17:15:18.437Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.037298632Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"client":"2","":"downstairs","session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73382024-04-17T17:15:18.437Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.037311462Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73392024-04-17T17:15:18.437Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.037330274Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73402024-04-17T17:15:18.437Z{"msg":"d1bc5462-4f74-4a95-ad4c-fbb933a0ec43 is now active with session: 15c21756-da32-4e76-9d38-3cc41e6824d0","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.037342924Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73412024-04-17T17:15:18.437Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.037355795Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440,"session_id":"15c21756-da32-4e76-9d38-3cc41e6824d0"}
73422024-04-17T17:15:18.437Z{"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:17.037374906Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440}
73432024-04-17T17:15:18.437ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
73442024-04-17T17:15:18.441ZLoading write count information from file "/tmp/test_repair_verify.data"
73452024-04-17T17:15:18.441ZRun Repair workload
73462024-04-17T17:15:18.441Z01/30 Flush
73472024-04-17T17:15:18.441Z02/30 Write block 98 len 12288 data: 7 9 9
73482024-04-17T17:15:18.441Z03/30 Write block 581 len 28672 data: 8 7 7 7 8 9 7
73492024-04-17T17:15:18.441Z04/30 Read block 114 len 8192
73502024-04-17T17:15:18.441Z05/30 Write block 546 len 28672 data: 4 5 6 6 6 7 8
73512024-04-17T17:15:18.441Z06/30 Read block 322 len 40960
73522024-04-17T17:15:18.441Z07/30 Flush
73532024-04-17T17:15:18.441Z08/30 Write block 370 len 28672 data: 6 8 5 6 5 5 3
73542024-04-17T17:15:18.448Z09/30 Write block 435 len 28672 data: 7 7 7 8 9 8 8
73552024-04-17T17:15:18.448Z10/30 Read block 512 len 4096
73562024-04-17T17:15:18.448Z11/30 Flush
73572024-04-17T17:15:18.448Z12/30 Write block 561 len 28672 data: 7 6 5 6 4 5 5
73582024-04-17T17:15:18.448Z13/30 Write block 220 len 8192 data: 6 5
73592024-04-17T17:15:18.448Z14/30 Read block 427 len 24576
73602024-04-17T17:15:18.448Z15/30 Read block 301 len 4096
73612024-04-17T17:15:18.452Z16/30 Read block 52 len 8192
73622024-04-17T17:15:18.452Z17/30 Write block 548 len 36864 data: 7 7 7 8 9 7 7 7 6
73632024-04-17T17:15:18.452Z18/30 Write block 144 len 28672 data: 7 6 5 3 3 4 4
73642024-04-17T17:15:18.452Z19/30 Read block 293 len 28672
73652024-04-17T17:15:18.456Z20/30 Read block 108 len 40960
73662024-04-17T17:15:18.456Z21/30 Write block 503 len 8192 data: 7 8
73672024-04-17T17:15:18.456Z22/30 Flush
73682024-04-17T17:15:18.461Z23/30 Write block 326 len 8192 data: 3 3
73692024-04-17T17:15:18.461Z24/30 Read block 282 len 20480
73702024-04-17T17:15:18.461Z25/30 Read block 37 len 4096
73712024-04-17T17:15:18.461Z26/30 Write block 32 len 16384 data: 12 13 12 9
73722024-04-17T17:15:18.461Z27/30 Write block 275 len 24576 data: 7 7 3 3 4 4
73732024-04-17T17:15:18.461Z28/30 Read block 268 len 20480
73742024-04-17T17:15:18.466Z29/30 Write block 57 len 16384 data: 8 8 8 8
73752024-04-17T17:15:18.466Z30/30 Write block 556 len 4096 data: 7
73762024-04-17T17:15:18.466Z----------------------------------------------------------------
73772024-04-17T17:15:18.466Z Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:29
73782024-04-17T17:15:18.466ZStates: Active Active Active
73792024-04-17T17:15:18.466ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
73802024-04-17T17:15:18.466Z 2 Acked 1001 Write 3 Done Done Done false
73812024-04-17T17:15:18.466Z 3 Acked 1002 Write 7 Done Done Sent false
73822024-04-17T17:15:18.466Z 4 Acked 1003 Read 2 Done Done Sent false
73832024-04-17T17:15:18.466Z 5 Acked 1004 Write 7 Done Done Sent false
73842024-04-17T17:15:18.466Z 6 Acked 1005 Read 10 Done Done Sent false
73852024-04-17T17:15:18.466Z 7 Acked 1006 Flush 0 Done Done Sent false
73862024-04-17T17:15:18.466Z 8 Acked 1007 Write 7 Done Done Sent false
73872024-04-17T17:15:18.466Z 9 Acked 1008 Write 7 Done Done Sent false
73882024-04-17T17:15:18.466Z 10 Acked 1009 Read 1 Done Done Sent false
73892024-04-17T17:15:18.466Z 11 Acked 1010 Flush 0 Done Done Sent false
73902024-04-17T17:15:18.466Z 12 Acked 1011 Write 7 Done Done Sent false
73912024-04-17T17:15:18.466Z 13 Acked 1012 Write 2 Done Done Sent false
73922024-04-17T17:15:18.466Z 14 Acked 1013 Read 6 Done Done Sent false
73932024-04-17T17:15:18.466Z 15 Acked 1014 Read 1 Done Done Sent false
73942024-04-17T17:15:18.466Z 16 Acked 1015 Read 2 Done Done Sent false
73952024-04-17T17:15:18.467Z 17 Acked 1016 Write 9 Done Done Sent false
73962024-04-17T17:15:18.467Z 18 Acked 1017 Write 7 Done Done Sent false
73972024-04-17T17:15:18.467Z 19 Acked 1018 Read 7 Done Done Sent false
73982024-04-17T17:15:18.467Z 20 Acked 1019 Read 10 Done Done Sent false
73992024-04-17T17:15:18.467Z 21 Acked 1020 Write 2 Done Done Sent false
74002024-04-17T17:15:18.467Z 22 Acked 1021 Flush 0 Done Done Sent false
74012024-04-17T17:15:18.467Z 23 Acked 1022 Write 2 Done Done Sent false
74022024-04-17T17:15:18.467Z 24 Acked 1023 Read 5 Done Done Sent false
74032024-04-17T17:15:18.467Z 25 Acked 1024 Read 1 Done Done Sent false
74042024-04-17T17:15:18.467Z 26 Acked 1025 Write 4 Done Done Sent false
74052024-04-17T17:15:18.467Z 27 Acked 1026 Write 6 Done Done Sent false
74062024-04-17T17:15:18.467Z 28 Acked 1027 Read 5 Done Done Sent false
74072024-04-17T17:15:18.467Z 29 Acked 1028 Write 4 Sent Done Sent false
74082024-04-17T17:15:18.467Z 30 Acked 1029 Write 1 Sent Sent Sent false
74092024-04-17T17:15:18.467Z STATES DS:0 DS:1 DS:2 TOTAL
74102024-04-17T17:15:18.467Z New 0 0 0 0
74112024-04-17T17:15:18.467Z Sent 2 1 28 31
74122024-04-17T17:15:18.467Z Done 27 28 1 56
74132024-04-17T17:15:18.467Z Skipped 0 0 0 0
74142024-04-17T17:15:18.467Z Error 0 0 0 0
74152024-04-17T17:15:18.467ZLast Flush: 1021 1021 1000
74162024-04-17T17:15:18.467ZDownstairs last five completed: 1000
74172024-04-17T17:15:18.468ZUpstairs last five completed: 30 29 28 27 26
74182024-04-17T17:15:18.468Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:15:17.069857105Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1440}
74192024-04-17T17:15:18.468ZWrote out file "/tmp/test_repair_verify.data"
74202024-04-17T17:15:18.471Z
74212024-04-17T17:15:18.471ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
74222024-04-17T17:15:18.487ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
74232024-04-17T17:15:18.487Z 0 000-019 60 60 60 109 109 109 F F F
74242024-04-17T17:15:18.492Z 1 020-039 67 67 67 120 120 120 T T F <---
74252024-04-17T17:15:18.492Z 2 040-059 67 67 67 120 120 120 T T F <---
74262024-04-17T17:15:18.493Z 3 060-079 67 67 67 120 120 120 T T F <---
74272024-04-17T17:15:18.493Z 4 080-099 68 68 67 122 122 120 F F T <---
74282024-04-17T17:15:18.493Z 5 100-119 68 68 67 122 122 120 F F T <---
74292024-04-17T17:15:18.493Z 6 120-139 64 64 64 117 117 117 F F F
74302024-04-17T17:15:18.493Z 7 140-159 68 68 59 124 124 106 F F F <---
74312024-04-17T17:15:18.493Z 8 160-179 63 63 63 114 114 114 F F F
74322024-04-17T17:15:18.493Z 9 180-199 61 61 61 110 110 110 F F F
74332024-04-17T17:15:18.493Z 10 200-219 66 66 66 119 119 119 F F F
74342024-04-17T17:15:18.493Z 11 220-239 68 68 67 124 124 120 F F F <---
74352024-04-17T17:15:18.493Z 12 240-259 67 67 67 120 120 120 F F F
74362024-04-17T17:15:18.493Z 13 260-279 67 67 67 120 120 120 T T F <---
74372024-04-17T17:15:18.493Z 14 280-299 60 60 60 108 108 108 T T F <---
74382024-04-17T17:15:18.493Z 15 300-319 66 66 66 119 119 119 F F F
74392024-04-17T17:15:18.493Z 16 320-339 64 64 64 115 115 115 T T F <---
74402024-04-17T17:15:18.493Z 17 340-359 63 63 63 114 114 114 F F F
74412024-04-17T17:15:18.493Z 18 360-379 68 68 67 123 123 120 F F F <---
74422024-04-17T17:15:18.493Z 19 380-399 59 59 59 106 106 106 F F F
74432024-04-17T17:15:18.493Z 20 400-419 67 67 67 120 120 120 F F F
74442024-04-17T17:15:18.493Z 21 420-439 68 68 67 123 123 120 F F F <---
74452024-04-17T17:15:18.493Z 22 440-459 68 68 67 123 123 120 F F F <---
74462024-04-17T17:15:18.493Z 23 460-479 67 67 67 120 120 120 F F F
74472024-04-17T17:15:18.493Z 24 480-499 63 63 63 114 114 114 F F F
74482024-04-17T17:15:18.493Z 25 500-519 68 68 64 124 124 116 F F F <---
74492024-04-17T17:15:18.494Z 26 520-539 61 61 61 110 110 110 F F F
74502024-04-17T17:15:18.494Z 27 540-559 68 68 64 124 124 115 T T F <---
74512024-04-17T17:15:18.494Z 28 560-579 68 68 63 124 124 114 F F F <---
74522024-04-17T17:15:18.494Z 29 580-599 68 68 65 122 122 118 F F F <---
74532024-04-17T17:15:18.494ZMax gen: 68, Max flush: 124
74542024-04-17T17:15:18.494ZError: Difference in extent metadata found!
74552024-04-17T17:15:18.494ZA Difference in extent metadata is expected here
74562024-04-17T17:15:18.494ZOn loop 34
74572024-04-17T17:15:18.494Z
74582024-04-17T17:15:18.497ZVerifying data now
74592024-04-17T17:15:33.269ZNo change after verify
74602024-04-17T17:15:33.269ZLoop: 34 Downstairs dump after verify (and repair):
74612024-04-17T17:15:33.283ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
74622024-04-17T17:15:33.283Z 0 000-019 60 60 60 109 109 109 F F F
74632024-04-17T17:15:33.283Z 1 020-039 69 69 69 125 125 125 F F F
74642024-04-17T17:15:33.284Z 2 040-059 69 69 69 125 125 125 F F F
74652024-04-17T17:15:33.284Z 3 060-079 69 69 69 125 125 125 F F F
74662024-04-17T17:15:33.284Z 4 080-099 68 68 68 122 122 122 F F F
74672024-04-17T17:15:33.284Z 5 100-119 68 68 68 122 122 122 F F F
74682024-04-17T17:15:33.284Z 6 120-139 64 64 64 117 117 117 F F F
74692024-04-17T17:15:33.284Z 7 140-159 68 68 68 124 124 124 F F F
74702024-04-17T17:15:33.284Z 8 160-179 63 63 63 114 114 114 F F F
74712024-04-17T17:15:33.288Z 9 180-199 61 61 61 110 110 110 F F F
74722024-04-17T17:15:33.288Z 10 200-219 66 66 66 119 119 119 F F F
74732024-04-17T17:15:33.288Z 11 220-239 68 68 68 124 124 124 F F F
74742024-04-17T17:15:33.288Z 12 240-259 67 67 67 120 120 120 F F F
74752024-04-17T17:15:33.288Z 13 260-279 69 69 69 125 125 125 F F F
74762024-04-17T17:15:33.288Z 14 280-299 69 69 69 125 125 125 F F F
74772024-04-17T17:15:33.288Z 15 300-319 66 66 66 119 119 119 F F F
74782024-04-17T17:15:33.288Z 16 320-339 69 69 69 125 125 125 F F F
74792024-04-17T17:15:33.288Z 17 340-359 63 63 63 114 114 114 F F F
74802024-04-17T17:15:33.288Z 18 360-379 68 68 68 123 123 123 F F F
74812024-04-17T17:15:33.288Z 19 380-399 59 59 59 106 106 106 F F F
74822024-04-17T17:15:33.288Z 20 400-419 67 67 67 120 120 120 F F F
74832024-04-17T17:15:33.288Z 21 420-439 68 68 68 123 123 123 F F F
74842024-04-17T17:15:33.288Z 22 440-459 68 68 68 123 123 123 F F F
74852024-04-17T17:15:33.288Z 23 460-479 67 67 67 120 120 120 F F F
74862024-04-17T17:15:33.288Z 24 480-499 63 63 63 114 114 114 F F F
74872024-04-17T17:15:33.288Z 25 500-519 68 68 68 124 124 124 F F F
74882024-04-17T17:15:33.288Z 26 520-539 61 61 61 110 110 110 F F F
74892024-04-17T17:15:33.288Z 27 540-559 69 69 69 125 125 125 F F F
74902024-04-17T17:15:33.288Z 28 560-579 68 68 68 124 124 124 F F F
74912024-04-17T17:15:33.289Z 29 580-599 68 68 68 122 122 122 F F F
74922024-04-17T17:15:33.289ZMax gen: 69, Max flush: 125
74932024-04-17T17:15:33.289Z
74942024-04-17T17:15:33.289ZBegin loop 35
74952024-04-17T17:15:33.289ZDownstairs to restart: 0
74962024-04-17T17:15:33.307Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.914818226Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448}
74972024-04-17T17:15:33.307Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level"Crucible runtime is spawned
74982024-04-17T17:15:33.315Z:30,"time":"2024-04-17T17:15:31.915163297Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448}
74992024-04-17T17:15:33.315Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.915223061Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448}
75002024-04-17T17:15:33.315Z{"msg":"Crucible 4baa8873-3197-4901-b0c7-20ece62e79b4 has session id: f5cd4d51-957d-484d-a00c-baf06c9719d1","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.915255102Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75012024-04-17T17:15:33.315Z{"msg":"Upstairs opts: Upstairs UUID: 4baa8873-3197-4901-b0c7-20ece62e79b4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.915318196Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75022024-04-17T17:15:33.315Z{"msg":"Crucible stats registered with UUID: 4baa8873-3197-4901-b0c7-20ece62e79b4","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.915363919Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75032024-04-17T17:15:33.315Z{"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.915402722Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448}
75042024-04-17T17:15:33.315Z{"msg":"4baa8873-3197-4901-b0c7-20ece62e79b4 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.915427933Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75052024-04-17T17:15:33.315Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.915466355Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"client":"0","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75062024-04-17T17:15:33.315Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.915501448Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"client":"0","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75072024-04-17T17:15:33.315Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.91554907Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"client":"1","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75082024-04-17T17:15:33.315Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.915583353Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"client":"1","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75092024-04-17T17:15:33.315Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.915620375Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"client":"2","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75102024-04-17T17:15:33.315Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.915660537Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"client":"2","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75112024-04-17T17:15:33.316Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.915686539Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75122024-04-17T17:15:33.316Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.915777894Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75132024-04-17T17:15:33.316Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.915826307Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75142024-04-17T17:15:33.316Z{"msg":"ds_connection connected from Ok(127.0.0.1:40956)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.91586889Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75152024-04-17T17:15:33.316Z{"msg":"ds_connection connected from Ok(127.0.0.1:65243)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.915911002Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75162024-04-17T17:15:33.316Z{"msg":"ds_connection connected from Ok(127.0.0.1:55098)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.915941034Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75172024-04-17T17:15:33.316Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.916456635Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"client":"1","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75182024-04-17T17:15:33.316Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.916507589Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"client":"1","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75192024-04-17T17:15:33.316Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.91653389Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"client":"2","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75202024-04-17T17:15:33.316Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.916551321Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"client":"2","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75212024-04-17T17:15:33.316Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.916976747Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"client":"2","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75222024-04-17T17:15:33.316Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.917148627Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"client":"1","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75232024-04-17T17:15:33.316Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.917258954Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"client":"0","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75242024-04-17T17:15:33.316Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.917307277Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"client":"0","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75252024-04-17T17:15:33.316Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.917500619Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"client":"2","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75262024-04-17T17:15:33.316Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.917551872Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"client":"2","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75272024-04-17T17:15:33.316Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.917574903Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75282024-04-17T17:15:33.316Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.917742963Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"client":"1","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75292024-04-17T17:15:33.316Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.917778286Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"client":"1","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75302024-04-17T17:15:33.316Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.917793787Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75312024-04-17T17:15:33.316Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.917818098Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"client":"0","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75322024-04-17T17:15:33.316Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.918238533Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"client":"0","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75332024-04-17T17:15:33.316Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.918280536Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"client":"0","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75342024-04-17T17:15:33.317Z{"msg":"[0]R flush_numbers[0..12]: [109, 125, 125, 125, 122, 122, 117, 124, 114, 110, 119, 124]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.918297547Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"":"downstairs","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75352024-04-17T17:15:33.317Z{"msg":"[0]R generation[0..12]: [60, 69, 69, 69, 68, 68, 64, 68, 63, 61, 66, 68]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.918318928Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"":"downstairs","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75362024-04-17T17:15:33.317Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.918334779Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"":"downstairs","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75372024-04-17T17:15:33.317Z{"msg":"[1]R flush_numbers[0..12]: [109, 125, 125, 125, 122, 122, 117, 124, 114, 110, 119, 124]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.91835597Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"":"downstairs","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75382024-04-17T17:15:33.317Z{"msg":"[1]R generation[0..12]: [60, 69, 69, 69, 68, 68, 64, 68, 63, 61, 66, 68]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.918370641Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"":"downstairs","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75392024-04-17T17:15:33.317Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.918392493Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"":"downstairs","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75402024-04-17T17:15:33.317Z{"msg":"[2]R flush_numbers[0..12]: [109, 125, 125, 125, 122, 122, 117, 124, 114, 110, 119, 124]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.918406464Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO
75412024-04-17T17:15:33.317Zsession_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75422024-04-17T17:15:33.317Z{"msg":"[2]R generation[0..12]: [60, 69, 69, 69, 68, 68, 64, 68, 63, 61, 66, 68]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.918430955Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"":"downstairs","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75432024-04-17T17:15:33.317Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.918444816Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"":"downstairs","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75442024-04-17T17:15:33.317Z{"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.918464237Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"":"downstairs","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75452024-04-17T17:15:33.317Z{"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.918477848Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"":"downstairs","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75462024-04-17T17:15:33.317Z{"msg":"Next flush: 126","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.918496959Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"":"downstairs","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75472024-04-17T17:15:33.317Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.91850981Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"":"downstairs","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75482024-04-17T17:15:33.317Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.918530471Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75492024-04-17T17:15:33.317Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.918543022Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"client":"0","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75502024-04-17T17:15:33.317Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.918556703Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"client":"1","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75512024-04-17T17:15:33.317Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.918569583Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"client":"2","":"downstairs","session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75522024-04-17T17:15:33.317Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.918589455Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75532024-04-17T17:15:33.317Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.918602925Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75542024-04-17T17:15:33.317Z{"msg":"4baa8873-3197-4901-b0c7-20ece62e79b4 is now active with session: f5cd4d51-957d-484d-a00c-baf06c9719d1","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.918622217Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75552024-04-17T17:15:33.317Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.918634277Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448,"session_id":"f5cd4d51-957d-484d-a00c-baf06c9719d1"}
75562024-04-17T17:15:33.317Z{"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:31.918652708Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448}
75572024-04-17T17:15:33.317ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
75582024-04-17T17:15:33.322ZLoading write count information from file "/tmp/test_repair_verify.data"
75592024-04-17T17:15:33.322ZRun Repair workload
75602024-04-17T17:15:33.322Z01/30 Write block 234 len 8192 data: 5 7
75612024-04-17T17:15:33.322Z02/30 Flush
75622024-04-17T17:15:33.322Z03/30 Write block 332 len 8192 data: 4 5
75632024-04-17T17:15:33.322Z04/30 Read block 346 len 32768
75642024-04-17T17:15:33.322Z05/30 Read block 547 len 32768
75652024-04-17T17:15:33.322Z06/30 Write block 514 len 12288 data: 8 8 6
75662024-04-17T17:15:33.322Z07/30 Read block 21 len 24576
75672024-04-17T17:15:33.322Z08/30 Read block 144 len 32768
75682024-04-17T17:15:33.322Z09/30 Read block 556 len 12288
75692024-04-17T17:15:33.322Z10/30 Read block 576 len 4096
75702024-04-17T17:15:33.322Z11/30 Read block 424 len 4096
75712024-04-17T17:15:33.322Z12/30 Write block 477 len 8192 data: 7 10
75722024-04-17T17:15:33.322Z13/30 Read block 2 len 20480
75732024-04-17T17:15:33.326Z14/30 Read block 411 len 36864
75742024-04-17T17:15:33.326Z15/30 Read block 579 len 28672
75752024-04-17T17:15:33.326Z16/30 Write block 489 len 8192 data: 7 7
75762024-04-17T17:15:33.326Z17/30 Read block 184 len 28672
75772024-04-17T17:15:33.326Z18/30 Write block 549 len 8192 data: 8 8
75782024-04-17T17:15:33.329Z19/30 Write block 494 len 20480 data: 4 6 5 5 7
75792024-04-17T17:15:33.329Z20/30 Flush
75802024-04-17T17:15:33.329Z21/30 Write block 325 len 8192 data: 3 4
75812024-04-17T17:15:33.333Z22/30 Read block 67 len 20480
75822024-04-17T17:15:33.333Z23/30 Flush
75832024-04-17T17:15:33.333Z24/30 Flush
75842024-04-17T17:15:33.336Z25/30 Write block 574 len 8192 data: 9 10
75852024-04-17T17:15:33.336Z26/30 Write block 228 len 28672 data: 8 7 5 4 5 5 6
75862024-04-17T17:15:33.336Z27/30 Read block 491 len 16384
75872024-04-17T17:15:33.336Z28/30 Write block 243 len 20480 data: 8 8 8 8 7
75882024-04-17T17:15:33.336Z29/30 Read block 118 len 8192
75892024-04-17T17:15:33.341Z30/30 Write block 323 len 40960 data: 2 3 4 5 4 4 4 3 4 5
75902024-04-17T17:15:33.341Z----------------------------------------------------------------
75912024-04-17T17:15:33.341Z Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:30
75922024-04-17T17:15:33.341ZStates: Active Active Active
75932024-04-17T17:15:33.341ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
75942024-04-17T17:15:33.341Z 1 Acked 1000 Write 2 Sent Done Done false
75952024-04-17T17:15:33.341Z 2 Acked 1001 Flush 0 Sent Done Done false
75962024-04-17T17:15:33.341Z 3 Acked 1002 Write 2 Sent Done Done false
75972024-04-17T17:15:33.341Z 4 Acked 1003 Read 8 Sent Done Done false
75982024-04-17T17:15:33.341Z 5 Acked 1004 Read 8 Sent Done Done false
75992024-04-17T17:15:33.341Z 6 Acked 1005 Write 3 Sent Done Done false
76002024-04-17T17:15:33.341Z 7 Acked 1006 Read 6 Sent Done Done false
76012024-04-17T17:15:33.341Z 8 Acked 1007 Read 8 Sent Done Done false
76022024-04-17T17:15:33.341Z 9 Acked 1008 Read 3 Sent Done Done false
76032024-04-17T17:15:33.341Z 10 Acked 1009 Read 1 Sent Done Done false
76042024-04-17T17:15:33.341Z 11 Acked 1010 Read 1 Sent Done Done false
76052024-04-17T17:15:33.341Z 12 Acked 1011 Write 2 Sent Done Done false
76062024-04-17T17:15:33.341Z 13 Acked 1012 Read 5 Sent Done Done false
76072024-04-17T17:15:33.341Z 14 Acked 1013 Read 9 Sent Done Done false
76082024-04-17T17:15:33.341Z 15 Acked 1014 Read 7 Sent Done Done false
76092024-04-17T17:15:33.341Z 16 Acked 1015 Write 2 Sent Done Done false
76102024-04-17T17:15:33.342Z 17 Acked 1016 Read 7 Sent Done Done false
76112024-04-17T17:15:33.342Z 18 Acked 1017 Write 2 Sent Done Done false
76122024-04-17T17:15:33.342Z 19 Acked 1018 Write 5 Sent Done Done false
76132024-04-17T17:15:33.342Z 20 Acked 1019 Flush 0 Sent Done Done false
76142024-04-17T17:15:33.342Z 21 Acked 1020 Write 2 Sent Done Done false
76152024-04-17T17:15:33.342Z 22 Acked 1021 Read 5 Sent Done Done false
76162024-04-17T17:15:33.342Z 23 Acked 1022 Flush 0 Sent Done Done false
76172024-04-17T17:15:33.342Z 24 Acked 1023 Flush 0 Sent Done Done false
76182024-04-17T17:15:33.342Z 25 Acked 1024 Write 2 Sent Done Done false
76192024-04-17T17:15:33.342Z 26 Acked 1025 Write 7 Sent Done Done false
76202024-04-17T17:15:33.342Z 27 Acked 1026 Read 4 Sent Done Done false
76212024-04-17T17:15:33.342Z 28 Acked 1027 Write 5 Sent Done Done false
76222024-04-17T17:15:33.342Z 29 Acked 1028 Read 2 Sent Done Done false
76232024-04-17T17:15:33.342Z 30 Acked 1029 Write 10 Sent Sent Sent false
76242024-04-17T17:15:33.342Z STATES DS:0 DS:1 DS:2 TOTAL
76252024-04-17T17:15:33.342Z New 0 0 0 0
76262024-04-17T17:15:33.342Z Sent 30 1 1 32
76272024-04-17T17:15:33.342Z Done 0 29 29 58
76282024-04-17T17:15:33.342Z Skipped 0 0 0 0
76292024-04-17T17:15:33.342Z Error 0 0 0 0
76302024-04-17T17:15:33.342ZLast Flush: 0 1023 1023
76312024-04-17T17:15:33.342ZDownstairs last five completed:
76322024-04-17T17:15:33.342ZUpstairs last five completed: 30 29 28 27 26
76332024-04-17T17:15:33.343Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:15:31.946699808Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1448}
76342024-04-17T17:15:33.343ZWrote out file "/tmp/test_repair_verify.data"
76352024-04-17T17:15:33.346Z
76362024-04-17T17:15:33.347ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
76372024-04-17T17:15:33.362ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
76382024-04-17T17:15:33.362Z 0 000-019 60 60 60 109 109 109 F F F
76392024-04-17T17:15:33.362Z 1 020-039 69 69 69 125 125 125 F F F
76402024-04-17T17:15:33.362Z 2 040-059 69 69 69 125 125 125 F F F
76412024-04-17T17:15:33.362Z 3 060-079 69 69 69 125 125 125 F F F
76422024-04-17T17:15:33.362Z 4 080-099 68 68 68 122 122 122 F F F
76432024-04-17T17:15:33.362Z 5 100-119 68 68 68 122 122 122 F F F
76442024-04-17T17:15:33.366Z 6 120-139 64 64 64 117 117 117 F F F
76452024-04-17T17:15:33.366Z 7 140-159 68 68 68 124 124 124 F F F
76462024-04-17T17:15:33.366Z 8 160-179 63 63 63 114 114 114 F F F
76472024-04-17T17:15:33.366Z 9 180-199 61 61 61 110 110 110 F F F
76482024-04-17T17:15:33.366Z 10 200-219 66 66 66 119 119 119 F F F
76492024-04-17T17:15:33.366Z 11 220-239 68 70 70 124 126 126 F T T <---
76502024-04-17T17:15:33.366Z 12 240-259 67 67 67 120 120 120 F T T <---
76512024-04-17T17:15:33.366Z 13 260-279 69 69 69 125 125 125 F F F
76522024-04-17T17:15:33.366Z 14 280-299 69 69 69 125 125 125 F F F
76532024-04-17T17:15:33.366Z 15 300-319 66 66 66 119 119 119 F F F
76542024-04-17T17:15:33.366Z 16 320-339 69 70 70 125 128 128 F T T <---
76552024-04-17T17:15:33.366Z 17 340-359 63 63 63 114 114 114 F F F
76562024-04-17T17:15:33.366Z 18 360-379 68 68 68 123 123 123 F F F
76572024-04-17T17:15:33.366Z 19 380-399 59 59 59 106 106 106 F F F
76582024-04-17T17:15:33.366Z 20 400-419 67 67 67 120 120 120 F F F
76592024-04-17T17:15:33.366Z 21 420-439 68 68 68 123 123 123 F F F
76602024-04-17T17:15:33.367Z 22 440-459 68 68 68 123 123 123 F F F
76612024-04-17T17:15:33.367Z 23 460-479 67 70 70 120 127 127 F F F <---
76622024-04-17T17:15:33.367Z 24 480-499 63 70 70 114 127 127 F F F <---
76632024-04-17T17:15:33.367Z 25 500-519 68 70 70 124 127 127 F F F <---
76642024-04-17T17:15:33.367Z 26 520-539 61 61 61 110 110 110 F F F
76652024-04-17T17:15:33.367Z 27 540-559 69 70 70 125 127 127 F F F <---
76662024-04-17T17:15:33.367Z 28 560-579 68 68 68 124 124 124 F T T <---
76672024-04-17T17:15:33.367Z 29 580-599 68 68 68 122 122 122 F F F
76682024-04-17T17:15:33.367ZMax gen: 70, Max flush: 128
76692024-04-17T17:15:33.367ZError: Difference in extent metadata found!
76702024-04-17T17:15:33.367ZA Difference in extent metadata is expected here
76712024-04-17T17:15:33.367ZOn loop 35
76722024-04-17T17:15:33.367Z
76732024-04-17T17:15:33.371ZVerifying data now
76742024-04-17T17:15:39.263ZNo change after verify
76752024-04-17T17:15:39.263ZLoop: 35 Downstairs dump after verify (and repair):
76762024-04-17T17:15:39.263ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
76772024-04-17T17:15:39.263Z 0 000-019 60 60 60 109 109 109 F F F
76782024-04-17T17:15:39.263Z 1 020-039 69 69 69 125 125 125 F F F
76792024-04-17T17:15:39.263Z 2 040-059 69 69 69 125 125 125 F F F
76802024-04-17T17:15:39.263Z 3 060-079 69 69 69 125 125 125 F F F
76812024-04-17T17:15:39.263Z 4 080-099 68 68 68 122 122 122 F F F
76822024-04-17T17:15:39.263Z 5 100-119 68 68 68 122 122 122 F F F
76832024-04-17T17:15:39.264Z 6 120-139 64 64 64 117 117 117 F F F
76842024-04-17T17:15:39.264Z 7 140-159 68 68 68 124 124 124 F F F
76852024-04-17T17:15:39.264Z 8 160-179 63 63 63 114 114 114 F F F
76862024-04-17T17:15:39.264Z 9 180-199 61 61 61 110 110 110 F F F
76872024-04-17T17:15:39.264Z 10 200-219 66 66 66 119 119 119 F F F
76882024-04-17T17:15:39.264Z 11 220-239 71 71 71 129 129 129 F F F
76892024-04-17T17:15:39.264Z 12 240-259 71 71 71 129 129 129 F F F
76902024-04-17T17:15:39.264Z 13 260-279 69 69 69 125 125 125 F F F
76912024-04-17T17:15:39.264Z 14 280-299 69 69 69 125 125 125 F F F
76922024-04-17T17:15:39.264Z 15 300-319 66 66 66 119 119 119 F F F
76932024-04-17T17:15:39.264Z 16 320-339 71 71 71 129 129 129 F F F
76942024-04-17T17:15:39.264Z 17 340-359 63 63 63 114 114 114 F F F
76952024-04-17T17:15:39.264Z 18 360-379 68 68 68 123 123 123 F F F
76962024-04-17T17:15:39.264Z 19 380-399 59 59 59 106 106 106 F F F
76972024-04-17T17:15:39.264Z 20 400-419 67 67 67 120 120 120 F F F
76982024-04-17T17:15:39.264Z 21 420-439 68 68 68 123 123 123 F F F
76992024-04-17T17:15:39.264Z 22 440-459 68 68 68 123 123 123 F F F
77002024-04-17T17:15:39.264Z 23 460-479 70 70 70 127 127 127 F F F
77012024-04-17T17:15:39.265Z 24 480-499 70 70 70 127 127 127 F F F
77022024-04-17T17:15:39.265Z 25 500-519 70 70 70 127 127 127 F F F
77032024-04-17T17:15:39.265Z 26 520-539 61 61 61 110 110 110 F F F
77042024-04-17T17:15:39.265Z 27 540-559 70 70 70 127 127 127 F F F
77052024-04-17T17:15:39.265Z 28 560-579 71 71 71 129 129 129 F F F
77062024-04-17T17:15:39.265Z 29 580-599 68 68 68 122 122 122 F F F
77072024-04-17T17:15:39.265ZMax gen: 71, Max flush: 129
77082024-04-17T17:15:39.265Z
77092024-04-17T17:15:39.265ZBegin loop 36
77102024-04-17T17:15:39.265ZDownstairs to restart: 0
77112024-04-17T17:15:39.265Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.411497199Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456}
77122024-04-17T17:15:39.265Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.411759934Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456}
77132024-04-17T17:15:39.265Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.411799987Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456}
77142024-04-17T17:15:39.265ZCrucible runtime is spawned
77152024-04-17T17:15:39.265Z{"msg":"Crucible dfe6226e-610d-474a-b6f2-a085afddb19d has session id: 590e70b8-06bd-4567-be76-0c66e69a0788","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.41184972Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77162024-04-17T17:15:39.265Z{"msg":"Upstairs opts: Upstairs UUID: dfe6226e-610d-474a-b6f2-a085afddb19d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.411907143Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77172024-04-17T17:15:39.265Z{"msg":"Crucible stats registered with UUID: dfe6226e-610d-474a-b6f2-a085afddb19d","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.411934615Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77182024-04-17T17:15:39.265Z{"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.411948416Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456}
77192024-04-17T17:15:39.265Z{"msg":"dfe6226e-610d-474a-b6f2-a085afddb19d active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.411966717Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77202024-04-17T17:15:39.265Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.411979808Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"client":"0","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77212024-04-17T17:15:39.265Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.412027091Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"client":"0","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77222024-04-17T17:15:39.265Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.412063473Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"client":"1","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77232024-04-17T17:15:39.265Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.412099505Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"client":"1","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77242024-04-17T17:15:39.266Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.412133417Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"client":"2","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77252024-04-17T17:15:39.266Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.412189771Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"client":"2","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77262024-04-17T17:15:39.266Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.412221422Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77272024-04-17T17:15:39.266Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.412254795Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77282024-04-17T17:15:39.266Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.412280726Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77292024-04-17T17:15:39.266Z{"msg":"ds_connection connected from Ok(127.0.0.1:42785)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.412314478Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77302024-04-17T17:15:39.266Z{"msg":"ds_connection connected from Ok(127.0.0.1:53983)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.41234287Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77312024-04-17T17:15:39.266Z{"msg":"ds_connection connected from Ok(127.0.0.1:47965)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.412379292Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77322024-04-17T17:15:39.266Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.4131659Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"client":"2","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77332024-04-17T17:15:39.266Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.413225093Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"client":"2","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77342024-04-17T17:15:39.266Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.413261265Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"client":"1","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77352024-04-17T17:15:39.266Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.413308698Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"client":"1","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77362024-04-17T17:15:39.266Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.41334315Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"client":"0","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77372024-04-17T17:15:39.266Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.413376823Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"client":"0","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77382024-04-17T17:15:39.266Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.413792618Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"client":"2","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77392024-04-17T17:15:39.266Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.413848001Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"client":"0","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77402024-04-17T17:15:39.266Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.413912705Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"client":"1","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77412024-04-17T17:15:39.266Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.414478649Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"client":"0","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77422024-04-17T17:15:39.266Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.414514932Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"client":"0","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77432024-04-17T17:15:39.266Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.414537543Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77442024-04-17T17:15:39.266Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.414557044Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"client":"1","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77452024-04-17T17:15:39.266Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.414577665Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"client":"1","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77462024-04-17T17:15:39.266Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.414597757Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77472024-04-17T17:15:39.267Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.414616588Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"client":"2","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77482024-04-17T17:15:39.267Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.414637079Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"client":"2","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77492024-04-17T17:15:39.267Z{"msg":"[0]R flush_numbers[0..12]: [109, 125, 125, 125, 122, 122, 117, 124, 114, 110, 119, 129]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
77502024-04-17T17:15:39.267Z,"time":"2024-04-17T17:15:36.41465766Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"":"downstairs","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77512024-04-17T17:15:39.267Z{"msg":"[0]R generation[0..12]: [60, 69, 69, 69, 68, 68, 64, 68, 63, 61, 66, 71]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.414678491Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"":"downstairs","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77522024-04-17T17:15:39.267Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.414699163Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"":"downstairs","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77532024-04-17T17:15:39.267Z{"msg":"[1]R flush_numbers[0..12]: [109, 125, 125, 125, 122, 122, 117, 124, 114, 110, 119, 129]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.414719854Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"":"downstairs","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77542024-04-17T17:15:39.267Z{"msg":"[1]R generation[0..12]: [60, 69, 69, 69, 68, 68, 64, 68, 63, 61, 66, 71]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.414733615Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"":"downstairs","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77552024-04-17T17:15:39.267Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.414754446Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"":"downstairs","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77562024-04-17T17:15:39.267Z{"msg":"[2]R flush_numbers[0..12]: [109, 125, 125, 125, 122, 122, 117, 124, 114, 110, 119, 129]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.414776057Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"":"downstairs","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77572024-04-17T17:15:39.267Z{"msg":"[2]R generation[0..12]: [60, 69, 69, 69, 68, 68, 64, 68, 63, 61, 66, 71]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.414796069Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"":"downstairs","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77582024-04-17T17:15:39.267Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.414811939Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"":"downstairs","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77592024-04-17T17:15:39.267Z{"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.414831821Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"":"downstairs","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77602024-04-17T17:15:39.267Z{"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.414851712Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"":"downstairs","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77612024-04-17T17:15:39.267Z{"msg":"Next flush: 130","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.414871093Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"":"downstairs","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77622024-04-17T17:15:39.267Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.414884114Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"":"downstairs","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77632024-04-17T17:15:39.267Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.414903635Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77642024-04-17T17:15:39.267Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.414922336Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"client":"0","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77652024-04-17T17:15:39.267Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.414941937Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"client":"1","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77662024-04-17T17:15:39.268Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
77672024-04-17T17:15:39.268ZTransition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.414964709Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"client":"2","":"downstairs","session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77682024-04-17T17:15:39.268Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.41498466Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77692024-04-17T17:15:39.268Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.415002891Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77702024-04-17T17:15:39.268Z{"msg":"dfe6226e-610d-474a-b6f2-a085afddb19d is now active with session: 590e70b8-06bd-4567-be76-0c66e69a0788","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.415016042Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77712024-04-17T17:15:39.268Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.415027792Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456,"session_id":"590e70b8-06bd-4567-be76-0c66e69a0788"}
77722024-04-17T17:15:39.268Z{"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:36.415046684Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456}
77732024-04-17T17:15:39.268ZLoading write count information from file "/tmp/test_repair_verify.data"
77742024-04-17T17:15:39.277ZRun Repair workload
77752024-04-17T17:15:39.277ZError: Difference in extent metadata found!
77762024-04-17T17:15:39.278Z01/30 Write block 326 len 24576 data: 6 5 5 5 4 5
77772024-04-17T17:15:39.278Z02/30 Read block 306 len 28672
77782024-04-17T17:15:39.278Z03/30 Write block 521 len 20480 data: 6 6 6 3 4
77792024-04-17T17:15:39.278Z04/30 Write block 397 len 40960 data: 4 5 5 5 5 5 7 7 6 6
77802024-04-17T17:15:39.278Z05/30 Write block 200 len 32768 data: 7 9 11 12 11 10 9 9
77812024-04-17T17:15:39.278Z06/30 Read block 506 len 36864
77822024-04-17T17:15:39.278Z07/30 Write block 597 len 12288 data: 5 4 4
77832024-04-17T17:15:39.279Z08/30 Read block 527 len 32768
77842024-04-17T17:15:39.279Z09/30 Write block 465 len 8192 data: 7 6
77852024-04-17T17:15:39.279Z10/30 Write block 578 len 40960 data: 8 10 8 9 8 8 8 9 10 8
77862024-04-17T17:15:39.279Z11/30 Flush
77872024-04-17T17:15:39.279Z12/30 Write block 29 len 32768 data: 8 7 9 13 14 13 10 8
77882024-04-17T17:15:39.279Z13/30 Read block 74 len 32768
77892024-04-17T17:15:39.279Z14/30 Write block 213 len 20480 data: 4 5 6 5 4
77902024-04-17T17:15:39.279Z15/30 Read block 157 len 8192
77912024-04-17T17:15:39.279Z16/30 Write block 582 len 16384 data: 9 9 9 10
77922024-04-17T17:15:39.279Z17/30 Write block 54 len 4096 data: 7
77932024-04-17T17:15:39.279Z18/30 Write block 28 len 32768 data: 5 9 8 10 14 15 14 11
77942024-04-17T17:15:39.279Z19/30 Write block 227 len 32768 data: 10 9 8 6 5 6 6 7
77952024-04-17T17:15:39.279Z20/30 Read block 294 len 32768
77962024-04-17T17:15:39.279Z21/30 Write block 543 len 28672 data: 7 6 4 5 6 8 9
77972024-04-17T17:15:39.279Z22/30 Write block 419 len 16384 data: 8 7 4 6
77982024-04-17T17:15:39.279Z23/30 Read block 466 len 32768
77992024-04-17T17:15:39.279Z24/30 Read block 316 len 8192
78002024-04-17T17:15:39.280Z25/30 Flush
78012024-04-17T17:15:39.280Z26/30 Write block 418 len 12288 data: 8 9 8
78022024-04-17T17:15:39.280Z27/30 Write block 269 len 32768 data: 4 4 5 5 6 6 8 8
78032024-04-17T17:15:39.280Z28/30 Read block 452 len 40960
78042024-04-17T17:15:39.280Z29/30 Write block 480 len 36864 data: 11 10 10 12 12 10 11 8 8
78052024-04-17T17:15:39.280Z30/30 Write block 163 len 4096 data: 7
78062024-04-17T17:15:39.280Z----------------------------------------------------------------
78072024-04-17T17:15:39.280Z Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30
78082024-04-17T17:15:39.280ZStates: Active Active Active
78092024-04-17T17:15:39.280ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
78102024-04-17T17:15:39.280Z 1 Acked 1000 Write 6 Sent Done Done false
78112024-04-17T17:15:39.280Z 2 Acked 1001 Read 7 Sent Done Done false
78122024-04-17T17:15:39.280Z 3 Acked 1002 Write 5 Sent Done Done false
78132024-04-17T17:15:39.280Z 4 Acked 1003 Write 10 Sent Done Done false
78142024-04-17T17:15:39.280Z 5 Acked 1004 Write 8 Sent Done Done false
78152024-04-17T17:15:39.280Z 6 Acked 1005 Read 9 Sent Done Done false
78162024-04-17T17:15:39.280Z 7 Acked 1006 Write 3 Sent Done Done false
78172024-04-17T17:15:39.280Z 8 Acked 1007 Read 8 Sent Done Done false
78182024-04-17T17:15:39.280Z 9 Acked 1008 Write 2 Sent Done Done false
78192024-04-17T17:15:39.280Z 10 Acked 1009 Write 10 Sent Done Done false
78202024-04-17T17:15:39.280Z 11 Acked 1010 Flush 0 Sent Done Done false
78212024-04-17T17:15:39.280Z 12 Acked 1011 Write 8 Sent Done Done false
78222024-04-17T17:15:39.280Z 13 Acked 1012 Read 8 Sent Done Done false
78232024-04-17T17:15:39.280Z 14 Acked 1013 Write 5 Sent Done Done false
78242024-04-17T17:15:39.281Z 15 Acked 1014 Read 2 Sent Done Done false
78252024-04-17T17:15:39.281Z 16 Acked 1015 Write 4 Sent Done Done false
78262024-04-17T17:15:39.281Z 17 Acked 1016 Write 1 Sent Done Done false
78272024-04-17T17:15:39.281Z 18 Acked 1017 Write 8 Sent Done Done false
78282024-04-17T17:15:39.281Z 19 Acked 1018 Write 8 Sent Done Done false
78292024-04-17T17:15:39.281Z 20 Acked 1019 Read 8 Sent Done Done false
78302024-04-17T17:15:39.281Z 21 Acked 1020 Write 7 Sent Done Done false
78312024-04-17T17:15:39.281Z 22 Acked 1021 Write 4 Sent Done Done false
78322024-04-17T17:15:39.281Z 23 Acked 1022 Read 8 Sent Done Done false
78332024-04-17T17:15:39.281Z 24 Acked 1023 Read 2 Sent Done Done false
78342024-04-17T17:15:39.281Z 25 Acked 1024 Flush 0 Sent Done Done false
78352024-04-17T17:15:39.281Z 26 Acked 1025 Write 3 Sent Done Done false
78362024-04-17T17:15:39.281Z 27 Acked 1026 Write 8 Sent Done Done false
78372024-04-17T17:15:39.281Z 28 Acked 1027 Read 10 Sent Done Done false
78382024-04-17T17:15:39.281Z 29 Acked 1028 Write 9 Sent Sent Sent false
78392024-04-17T17:15:39.281Z 30 Acked 1029 Write 1 Sent Sent Sent false
78402024-04-17T17:15:39.281Z STATES DS:0 DS:1 DS:2 TOTAL
78412024-04-17T17:15:39.281Z New 0 0 0 0
78422024-04-17T17:15:39.281Z Sent 30 2 2 34
78432024-04-17T17:15:39.281Z Done 0 28 28 56
78442024-04-17T17:15:39.281Z Skipped 0 0 0 0
78452024-04-17T17:15:39.281Z Error 0 0 0 0
78462024-04-17T17:15:39.281ZLast Flush: 0 1024 1024
78472024-04-17T17:15:39.282ZDownstairs last five completed:
78482024-04-17T17:15:39.282ZUpstairs last five completed: 30 29 28 27 26
78492024-04-17T17:15:39.282Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:15:36.450057816Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1456}
78502024-04-17T17:15:39.282ZWrote out file "/tmp/test_repair_verify.data"
78512024-04-17T17:15:39.282Z
78522024-04-17T17:15:39.282ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
78532024-04-17T17:15:39.282ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
78542024-04-17T17:15:39.282Z 0 000-019 60 60 60 109 109 109 F F F
78552024-04-17T17:15:39.282Z 1 020-039 69 72 72 125 131 131 F F F <---
78562024-04-17T17:15:39.282Z 2 040-059 69 72 72 125 131 131 F F F <---
78572024-04-17T17:15:39.282Z 3 060-079 69 69 69 125 125 125 F F F
78582024-04-17T17:15:39.282Z 4 080-099 68 68 68 122 122 122 F F F
78592024-04-17T17:15:39.282Z 5 100-119 68 68 68 122 122 122 F F F
78602024-04-17T17:15:39.282Z 6 120-139 64 64 64 117 117 117 F F F
78612024-04-17T17:15:39.282Z 7 140-159 68 68 68 124 124 124 F F F
78622024-04-17T17:15:39.282Z 8 160-179 63 63 63 114 114 114 F T T <---
78632024-04-17T17:15:39.282Z 9 180-199 61 61 61 110 110 110 F F F
78642024-04-17T17:15:39.282Z 10 200-219 66 72 72 119 131 131 F F F <---
78652024-04-17T17:15:39.283Z 11 220-239 71 72 72 129 131 131 F F F <---
78662024-04-17T17:15:39.283Z 12 240-259 71 71 71 129 129 129 F F F
78672024-04-17T17:15:39.283Z 13 260-279 69 69 69 125 125 125 F T T <---
78682024-04-17T17:15:39.283Z 14 280-299 69 69 69 125 125 125 F F F
78692024-04-17T17:15:39.283Z 15 300-319 66 66 66 119 119 119 F F F
78702024-04-17T17:15:39.283Z 16 320-339 71 72 72 129 130 130 F F F <---
78712024-04-17T17:15:39.287Z 17 340-359 63 63 63 114 114 114 F F F
78722024-04-17T17:15:39.287Z 18 360-379 68 68 68 123 123 123 F F F
78732024-04-17T17:15:39.287Z 19 380-399 59 72 72 106 130 130 F F F <---
78742024-04-17T17:15:39.294Z 20 400-419 67 72 72 120 131 131 F T T <---
78752024-04-17T17:15:39.294Z 21 420-439 68 72 72 123 131 131 F T T <---
78762024-04-17T17:15:39.294Z 22 440-459 68 68 68 123 123 123 F F F
78772024-04-17T17:15:39.295Z 23 460-479 70 72 72 127 130 130 F F F <---
78782024-04-17T17:15:39.295Z 24 480-499 70 70 70 127 127 127 F T T <---
78792024-04-17T17:15:39.295Z 25 500-519 70 70 70 127 127 127 F F F
78802024-04-17T17:15:39.295Z 26 520-539 61 72 72 110 130 130 F F F <---
78812024-04-17T17:15:39.295Z 27 540-559 70 72 72 127 131 131 F F F <---
78822024-04-17T17:15:39.295Z 28 560-579 71 72 72 129 130 130 F F F <---
78832024-04-17T17:15:39.295Z 29 580-599 68 72 72 122 131 131 F F F <---
78842024-04-17T17:15:39.295ZMax gen: 72, Max flush: 131
78852024-04-17T17:15:39.295ZA Difference in extent metadata is expected here
78862024-04-17T17:15:39.295ZOn loop 36
78872024-04-17T17:15:39.296Z
78882024-04-17T17:15:39.296ZVerifying data now
78892024-04-17T17:15:42.593ZNo change after verify
78902024-04-17T17:15:42.596ZLoop: 36 Downstairs dump after verify (and repair):
78912024-04-17T17:15:42.606ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
78922024-04-17T17:15:42.607Z 0 000-019 60 60 60 109 109 109 F F F
78932024-04-17T17:15:42.611Z 1 020-039 72 72 72 131 131 131 F F F
78942024-04-17T17:15:42.611Z 2 040-059 72 72 72 131 131 131 F F F
78952024-04-17T17:15:42.611Z 3 060-079 69 69 69 125 125 125 F F F
78962024-04-17T17:15:42.611Z 4 080-099 68 68 68 122 122 122 F F F
78972024-04-17T17:15:42.611Z 5 100-119 68 68 68 122 122 122 F F F
78982024-04-17T17:15:42.611Z 6 120-139 64 64 64 117 117 117 F F F
78992024-04-17T17:15:42.611Z 7 140-159 68 68 68 124 124 124 F F F
79002024-04-17T17:15:42.611Z 8 160-179 73 73 73 132 132 132 F F F
79012024-04-17T17:15:42.611Z 9 180-199 61 61 61 110 110 110 F F F
79022024-04-17T17:15:42.611Z 10 200-219 72 72 72 131 131 131 F F F
79032024-04-17T17:15:42.611Z 11 220-239 72 72 72 131 131 131 F F F
79042024-04-17T17:15:42.611Z 12 240-259 71 71 71 129 129 129 F F F
79052024-04-17T17:15:42.611Z 13 260-279 73 73 73 132 132 132 F F F
79062024-04-17T17:15:42.611Z 14 280-299 69 69 69 125 125 125 F F F
79072024-04-17T17:15:42.611Z 15 300-319 66 66 66 119 119 119 F F F
79082024-04-17T17:15:42.611Z 16 320-339 72 72 72 130 130 130 F F F
79092024-04-17T17:15:42.611Z 17 340-359 63 63 63 114 114 114 F F F
79102024-04-17T17:15:42.611Z 18 360-379 68 68 68 123 123 123 F F F
79112024-04-17T17:15:42.611Z 19 380-399 72 72 72 130 130 130 F F F
79122024-04-17T17:15:42.611Z 20 400-419 73 73 73 132 132 132 F F F
79132024-04-17T17:15:42.611Z 21 420-439 73 73 73 132 132 132 F F F
79142024-04-17T17:15:42.612Z 22 440-459 68 68 68 123 123 123 F F F
79152024-04-17T17:15:42.612Z 23 460-479 72 72 72 130 130 130 F F F
79162024-04-17T17:15:42.612Z 24 480-499 73 73 73 132 132 132 F F F
79172024-04-17T17:15:42.612Z 25 500-519 70 70 70 127 127 127 F F F
79182024-04-17T17:15:42.612Z 26 520-539 72 72 72 130 130 130 F F F
79192024-04-17T17:15:42.612Z 27 540-559 72 72 72 131 131 131 F F F
79202024-04-17T17:15:42.612Z 28 560-579 72 72 72 130 130 130 F F F
79212024-04-17T17:15:42.612Z 29 580-599 72 72 72 131 131 131 F F F
79222024-04-17T17:15:42.612ZMax gen: 73, Max flush: 132
79232024-04-17T17:15:42.612Z
79242024-04-17T17:15:42.612ZBegin loop 37
79252024-04-17T17:15:42.612ZDownstairs to restart: 0
79262024-04-17T17:15:42.629Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.23636595Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464}
79272024-04-17T17:15:42.629Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.236599344Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464}
79282024-04-17T17:15:42.629Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.236617296Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464}
79292024-04-17T17:15:42.629Z{"msg":"Crucible a5bb1dfc-f47b-4716-b93e-38f07cbcc528 has session id: 60c32fde-c0ba-47e4-9918-2d62947f50ef","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.236631397Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79302024-04-17T17:15:42.636Z{"msg":"Upstairs opts: Upstairs UUID: a5bb1dfc-f47b-4716-b93e-38f07cbcc528, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.236675349Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79312024-04-17T17:15:42.636Z{"msg":"Crucible stats registered with UUID: a5bb1dfc-f47b-4716-b93e-38f07cbcc528","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.236708651Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79322024-04-17T17:15:42.636ZCrucible runtime is spawned
79332024-04-17T17:15:42.636Z{"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.236890082Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464}
79342024-04-17T17:15:42.636Z{"msg":"a5bb1dfc-f47b-4716-b93e-38f07cbcc528 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.236970687Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79352024-04-17T17:15:42.636Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.237008699Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"client":"0","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79362024-04-17T17:15:42.636Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.23702603Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"client":"0","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79372024-04-17T17:15:42.636Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.237062413Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"client":"1","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79382024-04-17T17:15:42.636Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.237090674Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"client":"1","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79392024-04-17T17:15:42.636Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.237113126Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"client":"2","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79402024-04-17T17:15:42.636Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.237135687Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"client":"2","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79412024-04-17T17:15:42.636Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.237152758Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79422024-04-17T17:15:42.636Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.23718294Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79432024-04-17T17:15:42.636Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.237217312Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79442024-04-17T17:15:42.637Z{"msg":"ds_connection connected from Ok(127.0.0.1:55938)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.237394073Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79452024-04-17T17:15:42.637Z{"msg":"ds_connection connected from Ok(127.0.0.1:61912)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.237470087Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79462024-04-17T17:15:42.637Z{"msg":"ds_connection connected from Ok(127.0.0.1:64563)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.237498149Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79472024-04-17T17:15:42.637Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.238242534Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"client":"1","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79482024-04-17T17:15:42.637Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.238296818Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"client":"1","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79492024-04-17T17:15:42.637Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.238326609Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"client":"2","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79502024-04-17T17:15:42.637Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.238355021Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"client":"2","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79512024-04-17T17:15:42.637Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.238390473Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"client":"0","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79522024-04-17T17:15:42.637Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.238419505Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"client":"0","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79532024-04-17T17:15:42.637Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.23884025Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"client":"2","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79542024-04-17T17:15:42.637Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.238925145Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"client":"1","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79552024-04-17T17:15:42.637Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.238955837Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"client":"0","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79562024-04-17T17:15:42.637Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.23949465Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"client":"2","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79572024-04-17T17:15:42.637Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.239541533Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"client":"2","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79582024-04-17T17:15:42.637Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.239560224Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79592024-04-17T17:15:42.637Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.239580875Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"client":"1","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79602024-04-17T17:15:42.637Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.239594626Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"client":"1","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79612024-04-17T17:15:42.637Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.239614567Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79622024-04-17T17:15:42.637Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.239626658Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"client":"0","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79632024-04-17T17:15:42.637Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.239646759Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"client":"0","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79642024-04-17T17:15:42.637Z{"msg":"[0]R flush_numbers[0..12]: [109, 131, 131, 125, 122, 122, 117, 124, 132, 110, 131, 131]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.2396613Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"":"downstairs","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79652024-04-17T17:15:42.637Z{"msg":"[0]R generation[0..12]: [60, 72, 72, 69, 68, 68, 64, 68, 73, 61, 72, 72]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.239681841Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"":"downstairs","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79662024-04-17T17:15:42.637Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
79672024-04-17T17:15:42.638Z,"time":"2024-04-17T17:15:41.239696582Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"":"downstairs","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79682024-04-17T17:15:42.638Z{"msg":"[1]R flush_numbers[0..12]: [109, 131, 131, 125, 122, 122, 117, 124, 132, 110, 131, 131]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.239723414Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"":"downstairs","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79692024-04-17T17:15:42.638Z{"msg":"[1]R generation[0..12]: [60, 72, 72, 69, 68, 68, 64, 68, 73, 61, 72, 72]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.239755516Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"":"downstairs","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79702024-04-17T17:15:42.638Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.239788928Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"":"downstairs","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79712024-04-17T17:15:42.638Z{"msg":"[2]R flush_numbers[0..12]: [109, 131, 131, 125, 122, 122, 117, 124, 132, 110, 131, 131]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.23983014Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"":"downstairs","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79722024-04-17T17:15:42.638Z{"msg":"[2]R generation[0..12]: [60, 72, 72, 69, 68, 68, 64, 68, 73, 61, 72, 72]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.239847092Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"":"downstairs","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79732024-04-17T17:15:42.638Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.239869163Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"":"downstairs","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79742024-04-17T17:15:42.638Z{"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.239882414Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"":"downstairs","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79752024-04-17T17:15:42.638Z{"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.239898955Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"":"downstairs","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79762024-04-17T17:15:42.638Z{"msg":"Next flush: 133","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.239911575Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"":"downstairs","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79772024-04-17T17:15:42.638Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.239936647Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"":"downstairs","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79782024-04-17T17:15:42.638Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.239950278Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79792024-04-17T17:15:42.638Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.239962719Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"client":"0","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79802024-04-17T17:15:42.638Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.239975559Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"client":"1","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79812024-04-17T17:15:42.638Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.23998857Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"client":"2","":"downstairs","session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79822024-04-17T17:15:42.638Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.240019942Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79832024-04-17T17:15:42.638Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.240033223Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79842024-04-17T17:15:42.638Z{"msg":"a5bb1dfc-f47b-4716-b93e-38f07cbcc528 is now active with session: 60c32fde-c0ba-47e4-9918-2d62947f50ef","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.240045953Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79852024-04-17T17:15:42.638Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.240065295Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464,"session_id":"60c32fde-c0ba-47e4-9918-2d62947f50ef"}
79862024-04-17T17:15:42.638Z{"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:41.240077915Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464}
79872024-04-17T17:15:42.638ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
79882024-04-17T17:15:42.645ZLoading write count information from file "/tmp/test_repair_verify.data"
79892024-04-17T17:15:42.645ZRun Repair workload
79902024-04-17T17:15:42.645Z01/30 Write block 366 len 8192 data: 4 4
79912024-04-17T17:15:42.645Z02/30 Read block 43 len 24576
79922024-04-17T17:15:42.645Z03/30 Flush
79932024-04-17T17:15:42.645Z04/30 Write block 391 len 16384 data: 5 6 5 6
79942024-04-17T17:15:42.645Z05/30 Read block 371 len 28672
79952024-04-17T17:15:42.645Z06/30 Write block 122 len 4096 data: 5
79962024-04-17T17:15:42.645Z07/30 Write block 117 len 4096 data: 5
79972024-04-17T17:15:42.645Z08/30 Read block 207 len 12288
79982024-04-17T17:15:42.645Z09/30 Write block 222 len 36864 data: 5 7 8 8 10 11 10 9 7
79992024-04-17T17:15:42.645Z10/30 Read block 180 len 28672
80002024-04-17T17:15:42.645Z11/30 Read block 27 len 16384
80012024-04-17T17:15:42.645Z12/30 Write block 392 len 40960 data: 7 6 7 4 4 5 6 6 6 6
80022024-04-17T17:15:42.645Z13/30 Read block 514 len 28672
80032024-04-17T17:15:42.649Z14/30 Read block 347 len 16384
80042024-04-17T17:15:42.649Z15/30 Read block 360 len 4096
80052024-04-17T17:15:42.649Z16/30 Read block 38 len 8192
80062024-04-17T17:15:42.649Z17/30 Write block 465 len 40960 data: 8 7 6 5 6 6 7 6 5 5
80072024-04-17T17:15:42.649Z18/30 Read block 105 len 4096
80082024-04-17T17:15:42.652Z19/30 Write block 195 len 12288 data: 3 3 4
80092024-04-17T17:15:42.652Z20/30 Write block 178 len 8192 data: 6 6
80102024-04-17T17:15:42.652Z21/30 Read block 433 len 16384
80112024-04-17T17:15:42.652Z22/30 Read block 286 len 28672
80122024-04-17T17:15:42.656Z23/30 Write block 60 len 24576 data: 9 8 8 8 9 7
80132024-04-17T17:15:42.656Z24/30 Read block 515 len 24576
80142024-04-17T17:15:42.656Z25/30 Write block 99 len 24576 data: 10 10 9 7 9 9
80152024-04-17T17:15:42.656Z26/30 Flush
80162024-04-17T17:15:42.661Z27/30 Write block 226 len 40960 data: 11 12 11 10 8 6 7 7 8 8
80172024-04-17T17:15:42.664Z28/30 Write block 273 len 32768 data: 7 7 9 9 4 4 5 5
80182024-04-17T17:15:42.664Z29/30 Read block 533 len 8192
80192024-04-17T17:15:42.664Z30/30 Read block 306 len 16384
80202024-04-17T17:15:42.668Z----------------------------------------------------------------
80212024-04-17T17:15:42.668Z Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:27
80222024-04-17T17:15:42.668ZStates: Active Active Active
80232024-04-17T17:15:42.668ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
80242024-04-17T17:15:42.668Z 4 Acked 1003 Write 4 Done Done Done false
80252024-04-17T17:15:42.668Z 5 Acked 1004 Read 7 Done Done Done false
80262024-04-17T17:15:42.668Z 6 Acked 1005 Write 1 Sent Done Done false
80272024-04-17T17:15:42.668Z 7 Acked 1006 Write 1 Sent Done Done false
80282024-04-17T17:15:42.668Z 8 Acked 1007 Read 3 Sent Done Done false
80292024-04-17T17:15:42.669Z 9 Acked 1008 Write 9 Sent Done Done false
80302024-04-17T17:15:42.669Z 10 Acked 1009 Read 7 Sent Done Done false
80312024-04-17T17:15:42.669Z 11 Acked 1010 Read 4 Sent Done Done false
80322024-04-17T17:15:42.669Z 12 Acked 1011 Write 10 Sent Done Done false
80332024-04-17T17:15:42.669Z 13 Acked 1012 Read 7 Sent Done Done false
80342024-04-17T17:15:42.669Z 14 Acked 1013 Read 4 Sent Done Done false
80352024-04-17T17:15:42.669Z 15 Acked 1014 Read 1 Sent Done Done false
80362024-04-17T17:15:42.669Z 16 Acked 1015 Read 2 Sent Done Done false
80372024-04-17T17:15:42.669Z 17 Acked 1016 Write 10 Sent Done Done false
80382024-04-17T17:15:42.669Z 18 Acked 1017 Read 1 Sent Done Done false
80392024-04-17T17:15:42.669Z 19 Acked 1018 Write 3 Sent Done Done false
80402024-04-17T17:15:42.669Z 20 Acked 1019 Write 2 Sent Done Done false
80412024-04-17T17:15:42.669Z 21 Acked 1020 Read 4 Sent Done Done false
80422024-04-17T17:15:42.669Z 22 Acked 1021 Read 7 Sent Done Done false
80432024-04-17T17:15:42.669Z 23 Acked 1022 Write 6 Sent Done Done false
80442024-04-17T17:15:42.669Z 24 Acked 1023 Read 6 Sent Done Done false
80452024-04-17T17:15:42.669Z 25 Acked 1024 Write 6 Sent Done Done false
80462024-04-17T17:15:42.669Z 26 Acked 1025 Flush 0 Sent Done Done false
80472024-04-17T17:15:42.669Z 27 Acked 1026 Write 10 Sent Done Done false
80482024-04-17T17:15:42.669Z 28 Acked 1027 Write 8 Sent Done Done false
80492024-04-17T17:15:42.669Z 29 Acked 1028 Read 2 Sent Done Done false
80502024-04-17T17:15:42.669Z 30 Acked 1029 Read 4 Sent Sent Done false
80512024-04-17T17:15:42.669Z STATES DS:0 DS:1 DS:2 TOTAL
80522024-04-17T17:15:42.669Z New 0 0 0 0
80532024-04-17T17:15:42.670Z Sent 25 1 0 26
80542024-04-17T17:15:42.670Z Done 2 26 27 55
80552024-04-17T17:15:42.670Z Skipped 0 0 0 0
80562024-04-17T17:15:42.670Z Error 0 0 0 0
80572024-04-17T17:15:42.670ZLast Flush: 1002 1025 1025
80582024-04-17T17:15:42.670ZDownstairs last five completed: 1002 1001 1000
80592024-04-17T17:15:42.670ZUpstairs last five completed: 30 29 28 27 26
80602024-04-17T17:15:42.670Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:15:41.272945407Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1464}
80612024-04-17T17:15:42.670ZWrote out file "/tmp/test_repair_verify.data"
80622024-04-17T17:15:42.674Z
80632024-04-17T17:15:42.674ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
80642024-04-17T17:15:42.688ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
80652024-04-17T17:15:42.688Z 0 000-019 60 60 60 109 109 109 F F F
80662024-04-17T17:15:42.692Z 1 020-039 72 72 72 131 131 131 F F F
80672024-04-17T17:15:42.692Z 2 040-059 72 72 72 131 131 131 F F F
80682024-04-17T17:15:42.692Z 3 060-079 69 74 74 125 134 134 F F F <---
80692024-04-17T17:15:42.692Z 4 080-099 68 74 74 122 134 134 F F F <---
80702024-04-17T17:15:42.692Z 5 100-119 68 74 74 122 134 134 F F F <---
80712024-04-17T17:15:42.692Z 6 120-139 64 74 74 117 134 134 F F F <---
80722024-04-17T17:15:42.692Z 7 140-159 68 68 68 124 124 124 F F F
80732024-04-17T17:15:42.692Z 8 160-179 73 74 74 132 134 134 F F F <---
80742024-04-17T17:15:42.692Z 9 180-199 61 74 74 110 134 134 F F F <---
80752024-04-17T17:15:42.692Z 10 200-219 72 72 72 131 131 131 F F F
80762024-04-17T17:15:42.692Z 11 220-239 72 74 74 131 134 134 F T T <---
80772024-04-17T17:15:42.692Z 12 240-259 71 71 71 129 129 129 F F F
80782024-04-17T17:15:42.692Z 13 260-279 73 73 73 132 132 132 F T T <---
80792024-04-17T17:15:42.692Z 14 280-299 69 69 69 125 125 125 F T T <---
80802024-04-17T17:15:42.692Z 15 300-319 66 66 66 119 119 119 F F F
80812024-04-17T17:15:42.692Z 16 320-339 72 72 72 130 130 130 F F F
80822024-04-17T17:15:42.692Z 17 340-359 63 63 63 114 114 114 F F F
80832024-04-17T17:15:42.692Z 18 360-379 74 74 74 133 133 133 F F F
80842024-04-17T17:15:42.693Z 19 380-399 72 74 74 130 134 134 T F F <---
80852024-04-17T17:15:42.693Z 20 400-419 73 74 74 132 134 134 F F F <---
80862024-04-17T17:15:42.693Z 21 420-439 73 73 73 132 132 132 F F F
80872024-04-17T17:15:42.693Z 22 440-459 68 68 68 123 123 123 F F F
80882024-04-17T17:15:42.693Z 23 460-479 72 74 74 130 134 134 F F F <---
80892024-04-17T17:15:42.693Z 24 480-499 73 73 73 132 132 132 F F F
80902024-04-17T17:15:42.693Z 25 500-519 70 70 70 127 127 127 F F F
80912024-04-17T17:15:42.693Z 26 520-539 72 72 72 130 130 130 F F F
80922024-04-17T17:15:42.693Z 27 540-559 72 72 72 131 131 131 F F F
80932024-04-17T17:15:42.693Z 28 560-579 72 72 72 130 130 130 F F F
80942024-04-17T17:15:42.693Z 29 580-599 72 72 72 131 131 131 F F F
80952024-04-17T17:15:42.693ZMax gen: 74, Max flush: 134
80962024-04-17T17:15:42.693ZError: Difference in extent metadata found!
80972024-04-17T17:15:42.693ZA Difference in extent metadata is expected here
80982024-04-17T17:15:42.693ZOn loop 37
80992024-04-17T17:15:42.693Z
81002024-04-17T17:15:42.697ZVerifying data now
81012024-04-17T17:15:47.254ZNo change after verify
81022024-04-17T17:15:47.258ZLoop: 37 Downstairs dump after verify (and repair):
81032024-04-17T17:15:47.269ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
81042024-04-17T17:15:47.269Z 0 000-019 60 60 60 109 109 109 F F F
81052024-04-17T17:15:47.269Z 1 020-039 72 72 72 131 131 131 F F F
81062024-04-17T17:15:47.269Z 2 040-059 72 72 72 131 131 131 F F F
81072024-04-17T17:15:47.269Z 3 060-079 74 74 74 134 134 134 F F F
81082024-04-17T17:15:47.273Z 4 080-099 74 74 74 134 134 134 F F F
81092024-04-17T17:15:47.273Z 5 100-119 74 74 74 134 134 134 F F F
81102024-04-17T17:15:47.273Z 6 120-139 74 74 74 134 134 134 F F F
81112024-04-17T17:15:47.273Z 7 140-159 68 68 68 124 124 124 F F F
81122024-04-17T17:15:47.273Z 8 160-179 74 74 74 134 134 134 F F F
81132024-04-17T17:15:47.273Z 9 180-199 74 74 74 134 134 134 F F F
81142024-04-17T17:15:47.274Z 10 200-219 72 72 72 131 131 131 F F F
81152024-04-17T17:15:47.274Z 11 220-239 75 75 75 135 135 135 F F F
81162024-04-17T17:15:47.274Z 12 240-259 71 71 71 129 129 129 F F F
81172024-04-17T17:15:47.274Z 13 260-279 75 75 75 135 135 135 F F F
81182024-04-17T17:15:47.274Z 14 280-299 75 75 75 135 135 135 F F F
81192024-04-17T17:15:47.274Z 15 300-319 66 66 66 119 119 119 F F F
81202024-04-17T17:15:47.274Z 16 320-339 72 72 72 130 130 130 F F F
81212024-04-17T17:15:47.274Z 17 340-359 63 63 63 114 114 114 F F F
81222024-04-17T17:15:47.274Z 18 360-379 74 74 74 133 133 133 F F F
81232024-04-17T17:15:47.274Z 19 380-399 74 74 74 134 134 134 F F F
81242024-04-17T17:15:47.274Z 20 400-419 74 74 74 134 134 134 F F F
81252024-04-17T17:15:47.274Z 21 420-439 73 73 73 132 132 132 F F F
81262024-04-17T17:15:47.274Z 22 440-459 68 68 68 123 123 123 F F F
81272024-04-17T17:15:47.274Z 23 460-479 74 74 74 134 134 134 F F F
81282024-04-17T17:15:47.274Z 24 480-499 73 73 73 132 132 132 F F F
81292024-04-17T17:15:47.274Z 25 500-519 70 70 70 127 127 127 F F F
81302024-04-17T17:15:47.274Z 26 520-539 72 72 72 130 130 130 F F F
81312024-04-17T17:15:47.274Z 27 540-559 72 72 72 131 131 131 F F F
81322024-04-17T17:15:47.275Z 28 560-579 72 72 72 130 130 130 F F F
81332024-04-17T17:15:47.275Z 29 580-599 72 72 72 131 131 131 F F F
81342024-04-17T17:15:47.275ZMax gen: 75, Max flush: 135
81352024-04-17T17:15:47.275Z
81362024-04-17T17:15:47.275ZBegin loop 38
81372024-04-17T17:15:47.275ZDownstairs to restart: 1
81382024-04-17T17:15:47.292Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.899903584Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472}
81392024-04-17T17:15:47.300Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.900202843Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472}
81402024-04-17T17:15:47.300Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30Crucible runtime is spawned
81412024-04-17T17:15:47.300Z,"time":"2024-04-17T17:15:45.900247465Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472}
81422024-04-17T17:15:47.300Z{"msg":"Crucible d76cf8bc-f5a5-4ac8-8351-04760a897e75 has session id: 4458f6f8-dda5-482b-8b0b-f09b270f884a","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.900276917Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81432024-04-17T17:15:47.300Z{"msg":"Upstairs opts: Upstairs UUID: d76cf8bc-f5a5-4ac8-8351-04760a897e75, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.900348431Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81442024-04-17T17:15:47.300Z{"msg":"Crucible stats registered with UUID: d76cf8bc-f5a5-4ac8-8351-04760a897e75","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.900394364Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81452024-04-17T17:15:47.300Z{"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.900436537Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472}
81462024-04-17T17:15:47.300Z{"msg":"d76cf8bc-f5a5-4ac8-8351-04760a897e75 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.90049616Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81472024-04-17T17:15:47.300Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.900533323Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"client":"0","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81482024-04-17T17:15:47.300Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.900558904Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"client":"0","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81492024-04-17T17:15:47.300Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.900581466Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"client":"1","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81502024-04-17T17:15:47.300Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.900600687Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"client":"1","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81512024-04-17T17:15:47.300Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.900623028Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"client":"2","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81522024-04-17T17:15:47.300Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.90065475Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"client":"2","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81532024-04-17T17:15:47.300Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.900670221Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81542024-04-17T17:15:47.300Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.900708253Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81552024-04-17T17:15:47.300Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.900725724Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81562024-04-17T17:15:47.300Z{"msg":"ds_connection connected from Ok(127.0.0.1:36551)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.900888864Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81572024-04-17T17:15:47.300Z{"msg":"ds_connection connected from Ok(127.0.0.1:57529)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.900937497Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81582024-04-17T17:15:47.301Z{"msg":"ds_connection connected from Ok(127.0.0.1:64734)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.900963939Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81592024-04-17T17:15:47.301Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.901753537Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"client":"2","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81602024-04-17T17:15:47.301Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.901788319Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"client":"2","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81612024-04-17T17:15:47.301Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.90180384Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"client":"0","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81622024-04-17T17:15:47.301Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.901864923Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"client":"0","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81632024-04-17T17:15:47.301Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.901892065Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"client":"1","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81642024-04-17T17:15:47.301Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.901918617Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"client":"1","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81652024-04-17T17:15:47.301Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.902424097Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"client":"2","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81662024-04-17T17:15:47.301Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.90246063Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"client":"1","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81672024-04-17T17:15:47.301Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.902488161Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"client":"0","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81682024-04-17T17:15:47.301Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.90295352Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"client":"2","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81692024-04-17T17:15:47.301Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.902990682Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"client":"2","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81702024-04-17T17:15:47.301Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.903008453Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81712024-04-17T17:15:47.301Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.903032984Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"client":"1","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81722024-04-17T17:15:47.301Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.903048255Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"client":"1","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81732024-04-17T17:15:47.301Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.903068576Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81742024-04-17T17:15:47.301Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.903171403Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"client":"0","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81752024-04-17T17:15:47.301Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.903197574Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"client":"0","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81762024-04-17T17:15:47.301Z{"msg":"[0]R flush_numbers[0..12]: [109, 131, 131, 134, 134, 134, 134, 124, 134, 134, 131, 135]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.903212815Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"":"downstairs","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81772024-04-17T17:15:47.301Z{"msg":"[0]R generation[0..12]: [60, 72, 72, 74, 74, 74, 74, 68, 74, 74, 72, 75]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.903233656Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"":"downstairs","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81782024-04-17T17:15:47.301Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.903248467Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"":"downstairs","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81792024-04-17T17:15:47.301Z{"msg":"[1]R flush_numbers[0..12]: [109, 131, 131, 134, 134, 134, 134, 124, 134, 134, 131, 135]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.903262458Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"":"downstairs","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81802024-04-17T17:15:47.301Z{"msg":"[1]R generation[0..12]: [60, 72, 72, 74, 74, 74, 74, 68, 74, 74, 72, 75]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.903276499Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"":"downstairs","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81812024-04-17T17:15:47.301Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.90329048Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"":"downstairs","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81822024-04-17T17:15:47.302Z{"msg":"[2]R flush_numbers[0..12]: [109, 131, 131, 134, 134, 134, 134, 124, 134, 134, 131, 135]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.903318091Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"":"downstairs","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81832024-04-17T17:15:47.302Z{"msg":"[2]R generation[0..12]: [60, 72, 72, 74, 74, 74, 74, 68, 74, 74, 72, 75]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.903332232Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"":"downstairs","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81842024-04-17T17:15:47.302Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.903346393Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"":"downstairs","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81852024-04-17T17:15:47.302Z{"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.903359184Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"":"downstairs","":"downstairsWait for a query_work_queue command to finish before sending IO
81862024-04-17T17:15:47.302Z","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81872024-04-17T17:15:47.302Z{"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.903382895Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"":"downstairs","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81882024-04-17T17:15:47.302Z{"msg":"Next flush: 136","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.903396386Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"":"downstairs","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81892024-04-17T17:15:47.302Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.903416677Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"":"downstairs","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81902024-04-17T17:15:47.302Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.903430098Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81912024-04-17T17:15:47.302Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.90344881Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"client":"0","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81922024-04-17T17:15:47.302Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.90346224Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"client":"1","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81932024-04-17T17:15:47.302Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.903482101Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"client":"2","":"downstairs","session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81942024-04-17T17:15:47.302Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.903495432Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81952024-04-17T17:15:47.302Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.903513983Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81962024-04-17T17:15:47.302Z{"msg":"d76cf8bc-f5a5-4ac8-8351-04760a897e75 is now active with session: 4458f6f8-dda5-482b-8b0b-f09b270f884a","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.903526844Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81972024-04-17T17:15:47.302Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.903545615Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472,"session_id":"4458f6f8-dda5-482b-8b0b-f09b270f884a"}
81982024-04-17T17:15:47.303Z{"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:45.903557806Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472}
81992024-04-17T17:15:47.303ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
82002024-04-17T17:15:47.307ZLoading write count information from file "/tmp/test_repair_verify.data"
82012024-04-17T17:15:47.307ZRun Repair workload
82022024-04-17T17:15:47.307Z01/30 Write block 268 len 20480 data: 3 5 5 6 6
82032024-04-17T17:15:47.307Z02/30 Write block 90 len 16384 data: 9 10 10 10
82042024-04-17T17:15:47.307Z03/30 Write block 584 len 24576 data: 10 11 11 9 5 5
82052024-04-17T17:15:47.307Z04/30 Write block 163 len 36864 data: 8 5 4 5 6 5 5 4 5
82062024-04-17T17:15:47.307Z05/30 Flush
82072024-04-17T17:15:47.307Z06/30 Write block 374 len 40960 data: 6 6 4 5 5 6 4 3 3 4
82082024-04-17T17:15:47.307Z07/30 Write block 129 len 16384 data: 5 7 8 9
82092024-04-17T17:15:47.311Z08/30 Write block 541 len 36864 data: 5 6 8 7 5 6 7 9 10
82102024-04-17T17:15:47.311Z09/30 Read block 284 len 20480
82112024-04-17T17:15:47.311Z10/30 Write block 195 len 16384 data: 4 4 5 6
82122024-04-17T17:15:47.311Z11/30 Read block 113 len 20480
82132024-04-17T17:15:47.311Z12/30 Read block 121 len 36864
82142024-04-17T17:15:47.311Z13/30 Read block 462 len 36864
82152024-04-17T17:15:47.315Z14/30 Read block 24 len 20480
82162024-04-17T17:15:47.315Z15/30 Read block 360 len 36864
82172024-04-17T17:15:47.315Z16/30 Read block 467 len 16384
82182024-04-17T17:15:47.315Z17/30 Read block 566 len 4096
82192024-04-17T17:15:47.315Z18/30 Flush
82202024-04-17T17:15:47.318Z19/30 Write block 259 len 24576 data: 7 5 6 5 5 5
82212024-04-17T17:15:47.318Z20/30 Write block 480 len 24576 data: 12 11 11 13 13 11
82222024-04-17T17:15:47.322Z21/30 Read block 194 len 8192
82232024-04-17T17:15:47.322Z22/30 Read block 20 len 36864
82242024-04-17T17:15:47.322Z23/30 Write block 577 len 4096 data: 8
82252024-04-17T17:15:47.322Z24/30 Read block 537 len 16384
82262024-04-17T17:15:47.322Z25/30 Flush
82272024-04-17T17:15:47.363Z26/30 Write block 32 len 8192 data: 15 16
82282024-04-17T17:15:47.363Z27/30 Write block 306 len 12288 data: 6 4 4
82292024-04-17T17:15:47.363Z28/30 Read block 315 len 24576
82302024-04-17T17:15:47.363Z29/30 Read block 142 len 28672
82312024-04-17T17:15:47.363Z30/30 Read block 98 len 12288
82322024-04-17T17:15:47.363Z----------------------------------------------------------------
82332024-04-17T17:15:47.363Z Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:30
82342024-04-17T17:15:47.363ZStates: Active Active Active
82352024-04-17T17:15:47.363ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
82362024-04-17T17:15:47.364Z 1 Acked 1000 Write 5 Done Sent Done false
82372024-04-17T17:15:47.364Z 2 Acked 1001 Write 4 Done Sent Done false
82382024-04-17T17:15:47.364Z 3 Acked 1002 Write 6 Done Sent Done false
82392024-04-17T17:15:47.364Z 4 Acked 1003 Write 9 Done Sent Done false
82402024-04-17T17:15:47.364Z 5 Acked 1004 Flush 0 Done Sent Done false
82412024-04-17T17:15:47.364Z 6 Acked 1005 Write 10 Done Sent Done false
82422024-04-17T17:15:47.364Z 7 Acked 1006 Write 4 Done Sent Done false
82432024-04-17T17:15:47.364Z 8 Acked 1007 Write 9 Done Sent Done false
82442024-04-17T17:15:47.364Z 9 Acked 1008 Read 5 Done Sent Done false
82452024-04-17T17:15:47.364Z 10 Acked 1009 Write 4 Done Sent Done false
82462024-04-17T17:15:47.364Z 11 Acked 1010 Read 5 Done Sent Done false
82472024-04-17T17:15:47.364Z 12 Acked 1011 Read 9 Done Sent Done false
82482024-04-17T17:15:47.364Z 13 Acked 1012 Read 9 Done Sent Done false
82492024-04-17T17:15:47.364Z 14 Acked 1013 Read 5 Done Sent Done false
82502024-04-17T17:15:47.364Z 15 Acked 1014 Read 9 Done Sent Done false
82512024-04-17T17:15:47.364Z 16 Acked 1015 Read 4 Done Sent Done false
82522024-04-17T17:15:47.364Z 17 Acked 1016 Read 1 Done Sent Done false
82532024-04-17T17:15:47.364Z 18 Acked 1017 Flush 0 Done Sent Done false
82542024-04-17T17:15:47.364Z 19 Acked 1018 Write 6 Done Sent Done false
82552024-04-17T17:15:47.364Z 20 Acked 1019 Write 6 Done Sent Done false
82562024-04-17T17:15:47.364Z 21 Acked 1020 Read 2 Done Sent Done false
82572024-04-17T17:15:47.364Z 22 Acked 1021 Read 9 Done Sent Done false
82582024-04-17T17:15:47.364Z 23 Acked 1022 Write 1 Done Sent Done false
82592024-04-17T17:15:47.365Z 24 Acked 1023 Read 4 Done Sent Done false
82602024-04-17T17:15:47.365Z 25 Acked 1024 Flush 0 Done Sent Done false
82612024-04-17T17:15:47.365Z 26 Acked 1025 Write 2 Done Sent Done false
82622024-04-17T17:15:47.365Z 27 Acked 1026 Write 3 Done Sent Done false
82632024-04-17T17:15:47.365Z 28 Acked 1027 Read 6 Done Sent Done false
82642024-04-17T17:15:47.365Z 29 Acked 1028 Read 7 Done Sent Done false
82652024-04-17T17:15:47.365Z 30 Acked 1029 Read 3 Done Sent Done false
82662024-04-17T17:15:47.365Z STATES DS:0 DS:1 DS:2 TOTAL
82672024-04-17T17:15:47.365Z New 0 0 0 0
82682024-04-17T17:15:47.365Z Sent 0 30 0 30
82692024-04-17T17:15:47.365Z Done 30 0 30 60
82702024-04-17T17:15:47.365Z Skipped 0 0 0 0
82712024-04-17T17:15:47.365Z Error 0 0 0 0
82722024-04-17T17:15:47.365ZLast Flush: 1024 0 1024
82732024-04-17T17:15:47.365ZDownstairs last five completed:
82742024-04-17T17:15:47.365ZUpstairs last five completed: 30 29 28 27 26
82752024-04-17T17:15:47.365Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:15:45.935737966Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1472}
82762024-04-17T17:15:47.365ZWrote out file "/tmp/test_repair_verify.data"
82772024-04-17T17:15:47.365Z
82782024-04-17T17:15:47.365ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
82792024-04-17T17:15:47.370ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
82802024-04-17T17:15:47.370Z 0 000-019 60 60 60 109 109 109 F F F
82812024-04-17T17:15:47.370Z 1 020-039 72 72 72 131 131 131 T F T <---
82822024-04-17T17:15:47.370Z 2 040-059 72 72 72 131 131 131 F F F
82832024-04-17T17:15:47.370Z 3 060-079 74 74 74 134 134 134 F F F
82842024-04-17T17:15:47.370Z 4 080-099 76 74 76 136 134 136 F F F <---
82852024-04-17T17:15:47.370Z 5 100-119 74 74 74 134 134 134 F F F
82862024-04-17T17:15:47.370Z 6 120-139 76 74 76 137 134 137 F F F <---
82872024-04-17T17:15:47.370Z 7 140-159 68 68 68 124 124 124 F F F
82882024-04-17T17:15:47.370Z 8 160-179 76 74 76 136 134 136 F F F <---
82892024-04-17T17:15:47.370Z 9 180-199 76 74 76 137 134 137 F F F <---
82902024-04-17T17:15:47.370Z 10 200-219 72 72 72 131 131 131 F F F
82912024-04-17T17:15:47.370Z 11 220-239 75 75 75 135 135 135 F F F
82922024-04-17T17:15:47.370Z 12 240-259 76 71 76 138 129 138 F F F <---
82932024-04-17T17:15:47.370Z 13 260-279 76 75 76 138 135 138 F F F <---
82942024-04-17T17:15:47.370Z 14 280-299 75 75 75 135 135 135 F F F
82952024-04-17T17:15:47.371Z 15 300-319 66 66 66 119 119 119 T F T <---
82962024-04-17T17:15:47.371Z 16 320-339 72 72 72 130 130 130 F F F
82972024-04-17T17:15:47.371Z 17 340-359 63 63 63 114 114 114 F F F
82982024-04-17T17:15:47.371Z 18 360-379 76 74 76 137 133 137 F F F <---
82992024-04-17T17:15:47.371Z 19 380-399 76 74 76 137 134 137 F F F <---
83002024-04-17T17:15:47.371Z 20 400-419 74 74 74 134 134 134 F F F
83012024-04-17T17:15:47.371Z 21 420-439 73 73 73 132 132 132 F F F
83022024-04-17T17:15:47.371Z 22 440-459 68 68 68 123 123 123 F F F
83032024-04-17T17:15:47.371Z 23 460-479 74 74 74 134 134 134 F F F
83042024-04-17T17:15:47.371Z 24 480-499 76 73 76 138 132 138 F F F <---
83052024-04-17T17:15:47.371Z 25 500-519 70 70 70 127 127 127 F F F
83062024-04-17T17:15:47.371Z 26 520-539 72 72 72 130 130 130 F F F
83072024-04-17T17:15:47.371Z 27 540-559 76 72 76 137 131 137 F F F <---
83082024-04-17T17:15:47.371Z 28 560-579 76 72 76 138 130 138 F F F <---
83092024-04-17T17:15:47.371Z 29 580-599 76 72 76 136 131 136 F F F <---
83102024-04-17T17:15:47.371ZMax gen: 76, Max flush: 138
83112024-04-17T17:15:47.371ZError: Difference in extent metadata found!
83122024-04-17T17:15:47.371ZA Difference in extent metadata is expected here
83132024-04-17T17:15:47.371ZOn loop 38
83142024-04-17T17:15:47.371Z
83152024-04-17T17:15:47.371ZVerifying data now
83162024-04-17T17:15:51.995ZNo change after verify
83172024-04-17T17:15:51.995ZLoop: 38 Downstairs dump after verify (and repair):
83182024-04-17T17:15:52.010ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83192024-04-17T17:15:52.010Z 0 000-019 60 60 60 109 109 109 F F F
83202024-04-17T17:15:52.015Z 1 020-039 77 77 77 139 139 139 F F F
83212024-04-17T17:15:52.015Z 2 040-059 72 72 72 131 131 131 F F F
83222024-04-17T17:15:52.015Z 3 060-079 74 74 74 134 134 134 F F F
83232024-04-17T17:15:52.015Z 4 080-099 76 76 76 136 136 136 F F F
83242024-04-17T17:15:52.015Z 5 100-119 74 74 74 134 134 134 F F F
83252024-04-17T17:15:52.015Z 6 120-139 76 76 76 137 137 137 F F F
83262024-04-17T17:15:52.015Z 7 140-159 68 68 68 124 124 124 F F F
83272024-04-17T17:15:52.015Z 8 160-179 76 76 76 136 136 136 F F F
83282024-04-17T17:15:52.015Z 9 180-199 76 76 76 137 137 137 F F F
83292024-04-17T17:15:52.016Z 10 200-219 72 72 72 131 131 131 F F F
83302024-04-17T17:15:52.016Z 11 220-239 75 75 75 135 135 135 F F F
83312024-04-17T17:15:52.016Z 12 240-259 76 76 76 138 138 138 F F F
83322024-04-17T17:15:52.016Z 13 260-279 76 76 76 138 138 138 F F F
83332024-04-17T17:15:52.016Z 14 280-299 75 75 75 135 135 135 F F F
83342024-04-17T17:15:52.016Z 15 300-319 77 77 77 139 139 139 F F F
83352024-04-17T17:15:52.016Z 16 320-339 72 72 72 130 130 130 F F F
83362024-04-17T17:15:52.016Z 17 340-359 63 63 63 114 114 114 F F F
83372024-04-17T17:15:52.016Z 18 360-379 76 76 76 137 137 137 F F F
83382024-04-17T17:15:52.016Z 19 380-399 76 76 76 137 137 137 F F F
83392024-04-17T17:15:52.016Z 20 400-419 74 74 74 134 134 134 F F F
83402024-04-17T17:15:52.016Z 21 420-439 73 73 73 132 132 132 F F F
83412024-04-17T17:15:52.016Z 22 440-459 68 68 68 123 123 123 F F F
83422024-04-17T17:15:52.016Z 23 460-479 74 74 74 134 134 134 F F F
83432024-04-17T17:15:52.016Z 24 480-499 76 76 76 138 138 138 F F F
83442024-04-17T17:15:52.016Z 25 500-519 70 70 70 127 127 127 F F F
83452024-04-17T17:15:52.016Z 26 520-539 72 72 72 130 130 130 F F F
83462024-04-17T17:15:52.016Z 27 540-559 76 76 76 137 137 137 F F F
83472024-04-17T17:15:52.016Z 28 560-579 76 76 76 138 138 138 F F F
83482024-04-17T17:15:52.016Z 29 580-599 76 76 76 136 136 136 F F F
83492024-04-17T17:15:52.016ZMax gen: 77, Max flush: 139
83502024-04-17T17:15:52.016Z
83512024-04-17T17:15:52.016ZBegin loop 39
83522024-04-17T17:15:52.017ZDownstairs to restart: 1
83532024-04-17T17:15:52.032Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.639731462Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480}
83542024-04-17T17:15:52.032Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.639952265Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480}
83552024-04-17T17:15:52.032Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.639978317Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480}
83562024-04-17T17:15:52.039Z{"msg":"Crucible b05d23f6-941c-4451-ac24-e8d261f2ba29 has session id: dd834a9a-74eb-43f4-a8a1-8bf743bca64a","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.639992378Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83572024-04-17T17:15:52.039Z{"msg":"Upstairs opts: Upstairs UUID: b05d23f6-941c-4451-ac24-e8d261f2ba29, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.640045131Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83582024-04-17T17:15:52.039Z{Crucible runtime is spawned
83592024-04-17T17:15:52.039Z"msg":"Crucible stats registered with UUID: b05d23f6-941c-4451-ac24-e8d261f2ba29","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.640106565Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83602024-04-17T17:15:52.039Z{"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.640172429Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480}
83612024-04-17T17:15:52.039Z{"msg":"b05d23f6-941c-4451-ac24-e8d261f2ba29 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.640229142Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83622024-04-17T17:15:52.039Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.640281545Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"client":"0","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83632024-04-17T17:15:52.039Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.640313647Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"client":"0","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83642024-04-17T17:15:52.039Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.64036035Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"client":"1","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83652024-04-17T17:15:52.039Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.640396142Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"client":"1","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83662024-04-17T17:15:52.039Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.640436615Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"client":"2","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83672024-04-17T17:15:52.039Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.640478597Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"client":"2","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83682024-04-17T17:15:52.040Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.64052222Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83692024-04-17T17:15:52.040Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.640711632Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83702024-04-17T17:15:52.040Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.640742423Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83712024-04-17T17:15:52.040Z{"msg":"ds_connection connected from Ok(127.0.0.1:53826)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.640786376Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83722024-04-17T17:15:52.040Z{"msg":"ds_connection connected from Ok(127.0.0.1:46046)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.640816688Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83732024-04-17T17:15:52.040Z{"msg":"ds_connection connected from Ok(127.0.0.1:48717)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.640865411Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83742024-04-17T17:15:52.040Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.64118329Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"client":"0","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83752024-04-17T17:15:52.040Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.641235963Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"client":"0","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83762024-04-17T17:15:52.040Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.641305917Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"client":"2","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83772024-04-17T17:15:52.040Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.641362661Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"client":"2","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83782024-04-17T17:15:52.040Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.641709882Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"client":"2","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83792024-04-17T17:15:52.040Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.64184599Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"client":"0","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83802024-04-17T17:15:52.040Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.641888593Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"client":"1","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83812024-04-17T17:15:52.040Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.641905894Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"client":"1","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83822024-04-17T17:15:52.040Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.642228473Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"client":"2","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83832024-04-17T17:15:52.040Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.642317949Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"client":"2","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83842024-04-17T17:15:52.040Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.642357351Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83852024-04-17T17:15:52.040Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.642381393Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"client":"0","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83862024-04-17T17:15:52.040Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.642411035Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"client":"0","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83872024-04-17T17:15:52.040Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.642429005Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83882024-04-17T17:15:52.040Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.642590175Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"client":"1","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83892024-04-17T17:15:52.040Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.643006171Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"client":"1","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83902024-04-17T17:15:52.041Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.643043173Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"client":"1","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83912024-04-17T17:15:52.041Z{"msg":"[0]R flush_numbers[0..12]: [109, 139, 131, 134, 136, 134, 137, 124, 136, 137, 131, 135]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.643068254Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"":"downstairs","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83922024-04-17T17:15:52.041Z{"msg":"[0]R generation[0..12]: [60, 77, 72, 74, 76, 74, 76, 68, 76, 76, 72, 75]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.643102416Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"":"downstairs","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83932024-04-17T17:15:52.041Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.643138628Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"":"downstairs","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83942024-04-17T17:15:52.041Z{"msg":"[1]R flush_numbers[0..12]: [109, 139, 131, 134, 136, 134, 137, 124, 136, 137, 131, 135]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.64315736Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"":"downstairs","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83952024-04-17T17:15:52.041Z{"msg":"[1]R generation[0..12]: [60, 77, 72, 74, 76, 74, 76, 68, 76, 76, 72, 75]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.643178581Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"":"downstairs","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83962024-04-17T17:15:52.041ZWait for a query_work_queue command to finish before sending IO
83972024-04-17T17:15:52.041Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.643198352Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"":"downstairs","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83982024-04-17T17:15:52.041Z{"msg":"[2]R flush_numbers[0..12]: [109, 139, 131, 134, 136, 134, 137, 124, 136, 137, 131, 135]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.643219553Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"":"downstairs","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
83992024-04-17T17:15:52.041Z{"msg":"[2]R generation[0..12]: [60, 77, 72, 74, 76, 74, 76, 68, 76, 76, 72, 75]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.643234544Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"":"downstairs","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
84002024-04-17T17:15:52.041Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.643254946Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"":"downstairs","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
84012024-04-17T17:15:52.041Z{"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.643268306Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"":"downstairs","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
84022024-04-17T17:15:52.041Z{"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.643288057Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"":"downstairs","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
84032024-04-17T17:15:52.041Z{"msg":"Next flush: 140","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.643300968Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"":"downstairs","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
84042024-04-17T17:15:52.041Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.643320399Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"":"downstairs","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
84052024-04-17T17:15:52.041Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.6433339Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
84062024-04-17T17:15:52.041Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.643352592Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"client":"0","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
84072024-04-17T17:15:52.041Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.643365982Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"client":"1","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
84082024-04-17T17:15:52.041Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.643385263Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"client":"2","":"downstairs","session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
84092024-04-17T17:15:52.041Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.643398634Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
84102024-04-17T17:15:52.041Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.643411065Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
84112024-04-17T17:15:52.041Z{"msg":"b05d23f6-941c-4451-ac24-e8d261f2ba29 is now active with session: dd834a9a-74eb-43f4-a8a1-8bf743bca64a","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.643425526Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
84122024-04-17T17:15:52.041Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.643437847Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480,"session_id":"dd834a9a-74eb-43f4-a8a1-8bf743bca64a"}
84132024-04-17T17:15:52.042Z{"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:15:50.643450027Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480}
84142024-04-17T17:15:52.042ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
84152024-04-17T17:15:52.045ZLoading write count information from file "/tmp/test_repair_verify.data"
84162024-04-17T17:15:52.045ZRun Repair workload
84172024-04-17T17:15:52.046Z01/30 Write block 204 len 28672 data: 12 11 10 10 4 4 4
84182024-04-17T17:15:52.046Z02/30 Write block 128 len 40960 data: 5 6 8 9 10 7 9 7 8 7
84192024-04-17T17:15:52.046Z03/30 Write block 591 len 8192 data: 4 6
84202024-04-17T17:15:52.046Z04/30 Write block 96 len 20480 data: 6 6 8 11 11
84212024-04-17T17:15:52.046Z05/30 Read block 462 len 24576
84222024-04-17T17:15:52.046Z06/30 Write block 308 len 20480 data: 5 4 6 6 9
84232024-04-17T17:15:52.046Z07/30 Read block 565 len 12288
84242024-04-17T17:15:52.046Z08/30 Write block 560 len 36864 data: 9 8 7 6 7 5 6 6 7
84252024-04-17T17:15:52.049Z09/30 Write block 481 len 32768 data: 12 12 14 14 12 12 9 9
84262024-04-17T17:15:52.049Z10/30 Read block 17 len 12288
84272024-04-17T17:15:52.050Z11/30 Write block 211 len 40960 data: 4 5 5 6 7 6 5 5 6 7
84282024-04-17T17:15:52.050Z12/30 Write block 476 len 8192 data: 4 8
84292024-04-17T17:15:52.050Z13/30 Read block 392 len 36864
84302024-04-17T17:15:52.053Z14/30 Write block 416 len 8192 data: 10 9
84312024-04-17T17:15:52.053Z15/30 Write block 89 len 16384 data: 9 10 11 11
84322024-04-17T17:15:52.053Z16/30 Read block 164 len 28672
84332024-04-17T17:15:52.053Z17/30 Read block 170 len 32768
84342024-04-17T17:15:52.056Z18/30 Write block 229 len 20480 data: 11 9 7 8 8
84352024-04-17T17:15:52.057Z19/30 Write block 409 len 12288 data: 6 7 9
84362024-04-17T17:15:52.057Z20/30 Read block 95 len 12288
84372024-04-17T17:15:52.057Z21/30 Write block 166 len 28672 data: 6 7 6 6 5 6 5
84382024-04-17T17:15:52.060Z22/30 Read block 128 len 40960
84392024-04-17T17:15:52.060Z23/30 Read block 291 len 36864
84402024-04-17T17:15:52.060Z24/30 Read block 541 len 32768
84412024-04-17T17:15:52.060Z25/30 Write block 406 len 24576 data: 7 6 5 7 8 10
84422024-04-17T17:15:52.060Z26/30 Read block 578 len 28672
84432024-04-17T17:15:52.065Z27/30 Read block 538 len 16384
84442024-04-17T17:15:52.065Z28/30 Write block 405 len 12288 data: 7 8 7
84452024-04-17T17:15:52.065Z29/30 Read block 530 len 16384
84462024-04-17T17:15:52.065Z30/30 Read block 248 len 20480
84472024-04-17T17:15:52.065Z----------------------------------------------------------------
84482024-04-17T17:15:52.065Z Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30
84492024-04-17T17:15:52.065ZStates: Active Active Active
84502024-04-17T17:15:52.065ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
84512024-04-17T17:15:52.065Z 1 Acked 1000 Write 7 Done Done Done false
84522024-04-17T17:15:52.065Z 2 Acked 1001 Write 10 Done Done Done false
84532024-04-17T17:15:52.065Z 3 Acked 1002 Write 2 Done Done Done false
84542024-04-17T17:15:52.065Z 4 Acked 1003 Write 5 Done Done Done false
84552024-04-17T17:15:52.065Z 5 Acked 1004 Read 6 Done Done Done false
84562024-04-17T17:15:52.065Z 6 Acked 1005 Write 5 Done Sent Done false
84572024-04-17T17:15:52.065Z 7 Acked 1006 Read 3 Done Sent Done false
84582024-04-17T17:15:52.065Z 8 Acked 1007 Write 9 Done Sent Done false
84592024-04-17T17:15:52.065Z 9 Acked 1008 Write 8 Done Sent Done false
84602024-04-17T17:15:52.065Z 10 Acked 1009 Read 3 Done Sent Done false
84612024-04-17T17:15:52.065Z 11 Acked 1010 Write 10 Done Sent Done false
84622024-04-17T17:15:52.065Z 12 Acked 1011 Write 2 Done Sent Done false
84632024-04-17T17:15:52.065Z 13 Acked 1012 Read 9 Done Sent Done false
84642024-04-17T17:15:52.065Z 14 Acked 1013 Write 2 Done Sent Done false
84652024-04-17T17:15:52.066Z 15 Acked 1014 Write 4 Done Sent Done false
84662024-04-17T17:15:52.066Z 16 Acked 1015 Read 7 Done Sent Done false
84672024-04-17T17:15:52.066Z 17 Acked 1016 Read 8 Done Sent Done false
84682024-04-17T17:15:52.066Z 18 Acked 1017 Write 5 Done Sent Done false
84692024-04-17T17:15:52.066Z 19 Acked 1018 Write 3 Done Sent Done false
84702024-04-17T17:15:52.066Z 20 Acked 1019 Read 3 Done Sent Done false
84712024-04-17T17:15:52.066Z 21 Acked 1020 Write 7 Done Sent Done false
84722024-04-17T17:15:52.066Z 22 Acked 1021 Read 10 Done Sent Done false
84732024-04-17T17:15:52.066Z 23 Acked 1022 Read 9 Done Sent Done false
84742024-04-17T17:15:52.066Z 24 Acked 1023 Read 8 Done Sent Done false
84752024-04-17T17:15:52.066Z 25 Acked 1024 Write 6 Done Sent Done false
84762024-04-17T17:15:52.066Z 26 Acked 1025 Read 7 Done Sent Done false
84772024-04-17T17:15:52.066Z 27 Acked 1026 Read 4 Done Sent Done false
84782024-04-17T17:15:52.066Z 28 Acked 1027 Write 3 Done Sent Done false
84792024-04-17T17:15:52.066Z 29 Acked 1028 Read 4 Done Sent Done false
84802024-04-17T17:15:52.066Z 30 Acked 1029 Read 5 Done Sent Done false
84812024-04-17T17:15:52.066Z STATES DS:0 DS:1 DS:2 TOTAL
84822024-04-17T17:15:52.066Z New 0 0 0 0
84832024-04-17T17:15:52.066Z Sent 0 25 0 25
84842024-04-17T17:15:52.066Z Done 30 5 30 65
84852024-04-17T17:15:52.066Z Skipped 0 0 0 0
84862024-04-17T17:15:52.066Z Error 0 0 0 0
84872024-04-17T17:15:52.066ZLast Flush: 0 0 0
84882024-04-17T17:15:52.066ZDownstairs last five completed:
84892024-04-17T17:15:52.067ZUpstairs last five completed: 30 29 28 27 26
84902024-04-17T17:15:52.070Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:15:50.672273144Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1480}
84912024-04-17T17:15:52.070ZWrote out file "/tmp/test_repair_verify.data"
84922024-04-17T17:15:52.070Z
84932024-04-17T17:15:52.074ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
84942024-04-17T17:15:52.091ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
84952024-04-17T17:15:52.091Z 0 000-019 60 60 60 109 109 109 F F F
84962024-04-17T17:15:52.091Z 1 020-039 77 77 77 139 139 139 F F F
84972024-04-17T17:15:52.091Z 2 040-059 72 72 72 131 131 131 F F F
84982024-04-17T17:15:52.091Z 3 060-079 74 74 74 134 134 134 F F F
84992024-04-17T17:15:52.091Z 4 080-099 76 76 76 136 136 136 T T T
85002024-04-17T17:15:52.091Z 5 100-119 74 74 74 134 134 134 T T T
85012024-04-17T17:15:52.094Z 6 120-139 76 76 76 137 137 137 T T T
85022024-04-17T17:15:52.094Z 7 140-159 68 68 68 124 124 124 F F F
85032024-04-17T17:15:52.095Z 8 160-179 76 76 76 136 136 136 T F T <---
85042024-04-17T17:15:52.095Z 9 180-199 76 76 76 137 137 137 F F F
85052024-04-17T17:15:52.095Z 10 200-219 72 72 72 131 131 131 T T T
85062024-04-17T17:15:52.095Z 11 220-239 75 75 75 135 135 135 T F T <---
85072024-04-17T17:15:52.095Z 12 240-259 76 76 76 138 138 138 F F F
85082024-04-17T17:15:52.095Z 13 260-279 76 76 76 138 138 138 F F F
85092024-04-17T17:15:52.095Z 14 280-299 75 75 75 135 135 135 F F F
85102024-04-17T17:15:52.095Z 15 300-319 77 77 77 139 139 139 T F T <---
85112024-04-17T17:15:52.095Z 16 320-339 72 72 72 130 130 130 F F F
85122024-04-17T17:15:52.095Z 17 340-359 63 63 63 114 114 114 F F F
85132024-04-17T17:15:52.095Z 18 360-379 76 76 76 137 137 137 F F F
85142024-04-17T17:15:52.095Z 19 380-399 76 76 76 137 137 137 F F F
85152024-04-17T17:15:52.095Z 20 400-419 74 74 74 134 134 134 T F T <---
85162024-04-17T17:15:52.095Z 21 420-439 73 73 73 132 132 132 F F F
85172024-04-17T17:15:52.095Z 22 440-459 68 68 68 123 123 123 F F F
85182024-04-17T17:15:52.095Z 23 460-479 74 74 74 134 134 134 T F T <---
85192024-04-17T17:15:52.095Z 24 480-499 76 76 76 138 138 138 T F T <---
85202024-04-17T17:15:52.095Z 25 500-519 70 70 70 127 127 127 F F F
85212024-04-17T17:15:52.095Z 26 520-539 72 72 72 130 130 130 F F F
85222024-04-17T17:15:52.095Z 27 540-559 76 76 76 137 137 137 F F F
85232024-04-17T17:15:52.095Z 28 560-579 76 76 76 138 138 138 T F T <---
85242024-04-17T17:15:52.095Z 29 580-599 76 76 76 136 136 136 T T T
85252024-04-17T17:15:52.096ZMax gen: 77, Max flush: 139
85262024-04-17T17:15:52.096ZError: Difference in extent metadata found!
85272024-04-17T17:15:52.096ZA Difference in extent metadata is expected here
85282024-04-17T17:15:52.096ZOn loop 39
85292024-04-17T17:15:52.096Z
85302024-04-17T17:15:52.099ZVerifying data now
85312024-04-17T17:16:06.680ZNo change after verify
85322024-04-17T17:16:06.680ZLoop: 39 Downstairs dump after verify (and repair):
85332024-04-17T17:16:06.692ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
85342024-04-17T17:16:06.692Z 0 000-019 60 60 60 109 109 109 F F F
85352024-04-17T17:16:06.696Z 1 020-039 77 77 77 139 139 139 F F F
85362024-04-17T17:16:06.696Z 2 040-059 72 72 72 131 131 131 F F F
85372024-04-17T17:16:06.697Z 3 060-079 74 74 74 134 134 134 F F F
85382024-04-17T17:16:06.697Z 4 080-099 78 78 78 140 140 140 F F F
85392024-04-17T17:16:06.697Z 5 100-119 78 78 78 140 140 140 F F F
85402024-04-17T17:16:06.697Z 6 120-139 78 78 78 140 140 140 F F F
85412024-04-17T17:16:06.697Z 7 140-159 68 68 68 124 124 124 F F F
85422024-04-17T17:16:06.697Z 8 160-179 78 78 78 140 140 140 F F F
85432024-04-17T17:16:06.697Z 9 180-199 76 76 76 137 137 137 F F F
85442024-04-17T17:16:06.697Z 10 200-219 78 78 78 140 140 140 F F F
85452024-04-17T17:16:06.697Z 11 220-239 78 78 78 140 140 140 F F F
85462024-04-17T17:16:06.697Z 12 240-259 76 76 76 138 138 138 F F F
85472024-04-17T17:16:06.697Z 13 260-279 76 76 76 138 138 138 F F F
85482024-04-17T17:16:06.697Z 14 280-299 75 75 75 135 135 135 F F F
85492024-04-17T17:16:06.697Z 15 300-319 78 78 78 140 140 140 F F F
85502024-04-17T17:16:06.697Z 16 320-339 72 72 72 130 130 130 F F F
85512024-04-17T17:16:06.697Z 17 340-359 63 63 63 114 114 114 F F F
85522024-04-17T17:16:06.697Z 18 360-379 76 76 76 137 137 137 F F F
85532024-04-17T17:16:06.697Z 19 380-399 76 76 76 137 137 137 F F F
85542024-04-17T17:16:06.697Z 20 400-419 78 78 78 140 140 140 F F F
85552024-04-17T17:16:06.697Z 21 420-439 73 73 73 132 132 132 F F F
85562024-04-17T17:16:06.697Z 22 440-459 68 68 68 123 123 123 F F F
85572024-04-17T17:16:06.697Z 23 460-479 78 78 78 140 140 140 F F F
85582024-04-17T17:16:06.697Z 24 480-499 78 78 78 140 140 140 F F F
85592024-04-17T17:16:06.697Z 25 500-519 70 70 70 127 127 127 F F F
85602024-04-17T17:16:06.698Z 26 520-539 72 72 72 130 130 130 F F F
85612024-04-17T17:16:06.698Z 27 540-559 76 76 76 137 137 137 F F F
85622024-04-17T17:16:06.698Z 28 560-579 78 78 78 140 140 140 F F F
85632024-04-17T17:16:06.698Z 29 580-599 78 78 78 140 140 140 F F F
85642024-04-17T17:16:06.698ZMax gen: 78, Max flush: 140
85652024-04-17T17:16:06.698Z
85662024-04-17T17:16:06.698ZBegin loop 40
85672024-04-17T17:16:06.698ZDownstairs to restart: 0
85682024-04-17T17:16:06.716Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.323492233Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488}
85692024-04-17T17:16:06.716Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.323805122Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488}
85702024-04-17T17:16:06.721Z{"msg":"Upstairs <-> Downstairs Message Version: 6Crucible runtime is spawned
85712024-04-17T17:16:06.721Z","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.323911379Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488}
85722024-04-17T17:16:06.721Z{"msg":"Crucible fe8865b5-45db-4b3a-9c6d-8919e7cd48cd has session id: 4024d048-7439-4c7c-acd2-28e6fe207c8e","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.323951741Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
85732024-04-17T17:16:06.721Z{"msg":"Upstairs opts: Upstairs UUID: fe8865b5-45db-4b3a-9c6d-8919e7cd48cd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.323992334Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
85742024-04-17T17:16:06.721Z{"msg":"Crucible stats registered with UUID: fe8865b5-45db-4b3a-9c6d-8919e7cd48cd","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.324026896Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
85752024-04-17T17:16:06.721Z{"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.324043577Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488}
85762024-04-17T17:16:06.721Z{"msg":"fe8865b5-45db-4b3a-9c6d-8919e7cd48cd active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.324071739Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
85772024-04-17T17:16:06.721Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.324112121Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"client":"0","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
85782024-04-17T17:16:06.721Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.324160944Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"client":"0","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
85792024-04-17T17:16:06.721Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.324188666Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"client":"1","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
85802024-04-17T17:16:06.721Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.32425628Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"client":"1","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
85812024-04-17T17:16:06.721Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.324317334Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"client":"2","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
85822024-04-17T17:16:06.721Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.324357526Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"client":"2","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
85832024-04-17T17:16:06.721Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.324379747Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
85842024-04-17T17:16:06.721Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.32442241Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
85852024-04-17T17:16:06.721Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.324458222Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
85862024-04-17T17:16:06.721Z{"msg":"ds_connection connected from Ok(127.0.0.1:48861)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.324484413Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
85872024-04-17T17:16:06.722Z{"msg":"ds_connection connected from Ok(127.0.0.1:43715)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.324509905Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
85882024-04-17T17:16:06.722Z{"msg":"ds_connection connected from Ok(127.0.0.1:57913)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.324535277Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
85892024-04-17T17:16:06.722Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.325277192Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"client":"2","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
85902024-04-17T17:16:06.722Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.325319144Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"client":"2","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
85912024-04-17T17:16:06.722Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.325349366Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"client":"1","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
85922024-04-17T17:16:06.722Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.325377758Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"client":"1","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
85932024-04-17T17:16:06.722Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.32608393Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"client":"1","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
85942024-04-17T17:16:06.722Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.326125393Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"client":"2","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
85952024-04-17T17:16:06.722Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.326656295Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"client":"2","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
85962024-04-17T17:16:06.722Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.326693367Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"client":"2","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
85972024-04-17T17:16:06.722Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.326708798Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
85982024-04-17T17:16:06.722Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.326721589Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"client":"1","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
85992024-04-17T17:16:06.722Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.32674519Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"client":"1","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
86002024-04-17T17:16:06.722Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.326759451Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
86012024-04-17T17:16:06.722Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.326771392Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"client":"0","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
86022024-04-17T17:16:06.722Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.326798174Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"client":"0","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
86032024-04-17T17:16:06.722Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.327204429Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"client":"0","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
86042024-04-17T17:16:06.728Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.327658936Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"client":"0","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
86052024-04-17T17:16:06.728Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.32771888Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"client":"0","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
86062024-04-17T17:16:06.728Z{"msg":"[0]R flush_numbers[0..12]: [109, 139, 131, 134, 140, 140, 140, 124, 140, 137, 140, 140]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.327753402Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"":"downstairs","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
86072024-04-17T17:16:06.728Z{"msg":"[0]R generation[0..12]: [60, 77, 72, 74, 78, 78, 78, 68, 78, 76, 78, 78]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.327769003Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"":"downstairs","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
86082024-04-17T17:16:06.728Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.327790584Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"":"downstairs","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
86092024-04-17T17:16:06.728Z{"msg":"[1]R flush_numbers[0..12]: [109, 139, 131, 134, 140, 140, 140, 124, 140, 137, 140, 140]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.327805085Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO
86102024-04-17T17:16:06.728Z,"session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
86112024-04-17T17:16:06.728Z{"msg":"[1]R generation[0..12]: [60, 77, 72, 74, 78, 78, 78, 68, 78, 76, 78, 78]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.327828636Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"":"downstairs","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
86122024-04-17T17:16:06.728Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.327859648Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"":"downstairs","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
86132024-04-17T17:16:06.729Z{"msg":"[2]R flush_numbers[0..12]: [109, 139, 131, 134, 140, 140, 140, 124, 140, 137, 140, 140]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.327875659Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"":"downstairs","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
86142024-04-17T17:16:06.729Z{"msg":"[2]R generation[0..12]: [60, 77, 72, 74, 78, 78, 78, 68, 78, 76, 78, 78]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.327902031Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"":"downstairs","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
86152024-04-17T17:16:06.729Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.327917162Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"":"downstairs","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
86162024-04-17T17:16:06.729Z{"msg":"Max found gen is 79","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.327930052Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"":"downstairs","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
86172024-04-17T17:16:06.729Z{"msg":"Generation requested: 80 >= found:79","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.327950754Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"":"downstairs","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
86182024-04-17T17:16:06.729Z{"msg":"Next flush: 141","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.327963975Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"":"downstairs","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
86192024-04-17T17:16:06.729Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.327978425Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"":"downstairs","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
86202024-04-17T17:16:06.729Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.328019778Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
86212024-04-17T17:16:06.729Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.32805091Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"client":"0","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
86222024-04-17T17:16:06.729Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.328069801Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"client":"1","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
86232024-04-17T17:16:06.729Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.328094952Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"client":"2","":"downstairs","session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
86242024-04-17T17:16:06.729Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.328112833Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
86252024-04-17T17:16:06.729Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.328128785Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
86262024-04-17T17:16:06.729Z{"msg":"fe8865b5-45db-4b3a-9c6d-8919e7cd48cd is now active with session: 4024d048-7439-4c7c-acd2-28e6fe207c8e","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.328161796Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
86272024-04-17T17:16:06.729Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.328197189Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488,"session_id":"4024d048-7439-4c7c-acd2-28e6fe207c8e"}
86282024-04-17T17:16:06.729Z{"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:05.32821636Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488}
86292024-04-17T17:16:06.729ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
86302024-04-17T17:16:06.729ZLoading write count information from file "/tmp/test_repair_verify.data"
86312024-04-17T17:16:06.729ZRun Repair workload
86322024-04-17T17:16:06.729Z01/30 Write block 3 len 8192 data: 5 6
86332024-04-17T17:16:06.729Z02/30 Read block 183 len 4096
86342024-04-17T17:16:06.729Z03/30 Write block 94 len 16384 data: 8 8 7 7
86352024-04-17T17:16:06.729Z04/30 Read block 318 len 32768
86362024-04-17T17:16:06.730Z05/30 Write block 50 len 12288 data: 3 4 4
86372024-04-17T17:16:06.730Z06/30 Read block 4 len 24576
86382024-04-17T17:16:06.730Z07/30 Flush
86392024-04-17T17:16:06.735Z08/30 Write block 119 len 36864 data: 6 7 7 6 5 5 5 5 6
86402024-04-17T17:16:06.735Z09/30 Read block 565 len 40960
86412024-04-17T17:16:06.736Z10/30 Write block 323 len 8192 data: 3 4
86422024-04-17T17:16:06.736Z11/30 Write block 85 len 4096 data: 5
86432024-04-17T17:16:06.736Z12/30 Write block 535 len 32768 data: 4 5 4 4 5 5 6 7
86442024-04-17T17:16:06.736Z13/30 Read block 531 len 20480
86452024-04-17T17:16:06.736Z14/30 Read block 19 len 24576
86462024-04-17T17:16:06.739Z15/30 Write block 99 len 36864 data: 12 12 10 8 10 10 9 9 9
86472024-04-17T17:16:06.739Z16/30 Write block 382 len 16384 data: 4 5 5 5
86482024-04-17T17:16:06.739Z17/30 Write block 377 len 24576 data: 6 6 7 5 4 5
86492024-04-17T17:16:06.739Z18/30 Read block 246 len 24576
86502024-04-17T17:16:06.739Z19/30 Read block 336 len 36864
86512024-04-17T17:16:06.744Z20/30 Write block 54 len 16384 data: 8 7 7 9
86522024-04-17T17:16:06.744Z21/30 Read block 10 len 20480
86532024-04-17T17:16:06.744Z22/30 Read block 232 len 32768
86542024-04-17T17:16:06.748Z23/30 Write block 463 len 36864 data: 8 8 9 8 7 6 7 7 8
86552024-04-17T17:16:06.748Z24/30 Write block 215 len 36864 data: 8 7 6 6 7 8 6 6 8
86562024-04-17T17:16:06.748Z25/30 Read block 429 len 12288
86572024-04-17T17:16:06.748Z26/30 Write block 188 len 32768 data: 5 5 4 5 6 6 4 5
86582024-04-17T17:16:06.748Z27/30 Write block 468 len 4096 data: 7
86592024-04-17T17:16:06.752Z28/30 Write block 178 len 24576 data: 7 7 5 4 4 4
86602024-04-17T17:16:06.752Z29/30 Write block 39 len 20480 data: 6 7 10 8 7
86612024-04-17T17:16:06.752Z30/30 Write block 337 len 4096 data: 11
86622024-04-17T17:16:06.753Z----------------------------------------------------------------
86632024-04-17T17:16:06.753Z Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:23
86642024-04-17T17:16:06.753ZStates: Active Active Active
86652024-04-17T17:16:06.753ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
86662024-04-17T17:16:06.753Z 8 Acked 1007 Write 9 Done Done Done false
86672024-04-17T17:16:06.753Z 9 Acked 1008 Read 10 Done Done Done false
86682024-04-17T17:16:06.753Z 10 Acked 1009 Write 2 Done Done Done false
86692024-04-17T17:16:06.753Z 11 Acked 1010 Write 1 Done Done Done false
86702024-04-17T17:16:06.753Z 12 Acked 1011 Write 8 Sent Done Done false
86712024-04-17T17:16:06.753Z 13 Acked 1012 Read 5 Sent Done Done false
86722024-04-17T17:16:06.753Z 14 Acked 1013 Read 6 Sent Done Done false
86732024-04-17T17:16:06.753Z 15 Acked 1014 Write 9 Sent Done Done false
86742024-04-17T17:16:06.753Z 16 Acked 1015 Write 4 Sent Done Done false
86752024-04-17T17:16:06.753Z 17 Acked 1016 Write 6 Sent Done Done false
86762024-04-17T17:16:06.753Z 18 Acked 1017 Read 6 Sent Done Done false
86772024-04-17T17:16:06.753Z 19 Acked 1018 Read 9 Sent Done Done false
86782024-04-17T17:16:06.753Z 20 Acked 1019 Write 4 Sent Done Done false
86792024-04-17T17:16:06.753Z 21 Acked 1020 Read 5 Sent Done Done false
86802024-04-17T17:16:06.753Z 22 Acked 1021 Read 8 Sent Done Done false
86812024-04-17T17:16:06.753Z 23 Acked 1022 Write 9 Sent Done Done false
86822024-04-17T17:16:06.753Z 24 Acked 1023 Write 9 Sent Done Done false
86832024-04-17T17:16:06.753Z 25 Acked 1024 Read 3 Sent Done Done false
86842024-04-17T17:16:06.753Z 26 Acked 1025 Write 8 Sent Done Done false
86852024-04-17T17:16:06.754Z 27 Acked 1026 Write 1 Sent Done Done false
86862024-04-17T17:16:06.754Z 28 Acked 1027 Write 6 Sent Done Done false
86872024-04-17T17:16:06.754Z 29 Acked 1028 Write 5 Sent Done Done false
86882024-04-17T17:16:06.754Z 30 Acked 1029 Write 1 Sent Sent Sent false
86892024-04-17T17:16:06.754Z STATES DS:0 DS:1 DS:2 TOTAL
86902024-04-17T17:16:06.754Z New 0 0 0 0
86912024-04-17T17:16:06.754Z Sent 19 1 1 21
86922024-04-17T17:16:06.754Z Done 4 22 22 48
86932024-04-17T17:16:06.754Z Skipped 0 0 0 0
86942024-04-17T17:16:06.754Z Error 0 0 0 0
86952024-04-17T17:16:06.754ZLast Flush: 1006 1006 1006
86962024-04-17T17:16:06.754ZDownstairs last five completed: 1006 1005 1004 1003 1002
86972024-04-17T17:16:06.754ZUpstairs last five completed: 30 29 28 27 26
86982024-04-17T17:16:06.754Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:16:05.359323705Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1488}
86992024-04-17T17:16:06.754ZWrote out file "/tmp/test_repair_verify.data"
87002024-04-17T17:16:06.758Z
87012024-04-17T17:16:06.761ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
87022024-04-17T17:16:06.780ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
87032024-04-17T17:16:06.780Z 0 000-019 80 80 80 141 141 141 F F F
87042024-04-17T17:16:06.781Z 1 020-039 77 77 77 139 139 139 F T T <---
87052024-04-17T17:16:06.781Z 2 040-059 80 80 80 141 141 141 F T T <---
87062024-04-17T17:16:06.781Z 3 060-079 74 74 74 134 134 134 F F F
87072024-04-17T17:16:06.781Z 4 080-099 80 80 80 141 141 141 T T T
87082024-04-17T17:16:06.781Z 5 100-119 78 78 78 140 140 140 T T T
87092024-04-17T17:16:06.781Z 6 120-139 78 78 78 140 140 140 T T T
87102024-04-17T17:16:06.784Z 7 140-159 68 68 68 124 124 124 F F F
87112024-04-17T17:16:06.785Z 8 160-179 78 78 78 140 140 140 F T T <---
87122024-04-17T17:16:06.785Z 9 180-199 76 76 76 137 137 137 F T T <---
87132024-04-17T17:16:06.785Z 10 200-219 78 78 78 140 140 140 F T T <---
87142024-04-17T17:16:06.785Z 11 220-239 78 78 78 140 140 140 F T T <---
87152024-04-17T17:16:06.785Z 12 240-259 76 76 76 138 138 138 F F F
87162024-04-17T17:16:06.785Z 13 260-279 76 76 76 138 138 138 F F F
87172024-04-17T17:16:06.785Z 14 280-299 75 75 75 135 135 135 F F F
87182024-04-17T17:16:06.785Z 15 300-319 78 78 78 140 140 140 F F F
87192024-04-17T17:16:06.785Z 16 320-339 72 72 72 130 130 130 T T T
87202024-04-17T17:16:06.785Z 17 340-359 63 63 63 114 114 114 F F F
87212024-04-17T17:16:06.785Z 18 360-379 76 76 76 137 137 137 F T T <---
87222024-04-17T17:16:06.785Z 19 380-399 76 76 76 137 137 137 F T T <---
87232024-04-17T17:16:06.785Z 20 400-419 78 78 78 140 140 140 F F F
87242024-04-17T17:16:06.785Z 21 420-439 73 73 73 132 132 132 F F F
87252024-04-17T17:16:06.785Z 22 440-459 68 68 68 123 123 123 F F F
87262024-04-17T17:16:06.785Z 23 460-479 78 78 78 140 140 140 F T T <---
87272024-04-17T17:16:06.785Z 24 480-499 78 78 78 140 140 140 F F F
87282024-04-17T17:16:06.785Z 25 500-519 70 70 70 127 127 127 F F F
87292024-04-17T17:16:06.785Z 26 520-539 72 72 72 130 130 130 F T T <---
87302024-04-17T17:16:06.785Z 27 540-559 76 76 76 137 137 137 F T T <---
87312024-04-17T17:16:06.785Z 28 560-579 78 78 78 140 140 140 F F F
87322024-04-17T17:16:06.785Z 29 580-599 78 78 78 140 140 140 F F F
87332024-04-17T17:16:06.786ZMax gen: 80, Max flush: 141
87342024-04-17T17:16:06.786ZError: Difference in extent metadata found!
87352024-04-17T17:16:06.786ZA Difference in extent metadata is expected here
87362024-04-17T17:16:06.786ZOn loop 40
87372024-04-17T17:16:06.786Z
87382024-04-17T17:16:06.789ZVerifying data now
87392024-04-17T17:16:21.561ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
87402024-04-17T17:16:21.565Z102,103c102,107
87412024-04-17T17:16:21.565Z< 11,
87422024-04-17T17:16:21.565Z< 11,
87432024-04-17T17:16:21.565Z---
87442024-04-17T17:16:21.565Z> 12,
87452024-04-17T17:16:21.565Z> 12,
87462024-04-17T17:16:21.565Z> 10,
87472024-04-17T17:16:21.565Z> 8,
87482024-04-17T17:16:21.565Z> 10,
87492024-04-17T17:16:21.565Z> 10,
87502024-04-17T17:16:21.565Z105d108
87512024-04-17T17:16:21.566Z< 7,
87522024-04-17T17:16:21.566Z108,110d110
87532024-04-17T17:16:21.566Z< 8,
87542024-04-17T17:16:21.566Z< 8,
87552024-04-17T17:16:21.566Z< 8,
87562024-04-17T17:16:21.566Z339a340
87572024-04-17T17:16:21.566Z> 11,
87582024-04-17T17:16:21.566Z342d342
87592024-04-17T17:16:21.566Z< 10,
87602024-04-17T17:16:21.566Zdiff found after verify
87612024-04-17T17:16:21.569ZLoop: 40 Downstairs dump after verify (and repair):
87622024-04-17T17:16:21.583ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
87632024-04-17T17:16:21.583Z 0 000-019 80 80 80 141 141 141 F F F
87642024-04-17T17:16:21.583Z 1 020-039 81 81 81 142 142 142 F F F
87652024-04-17T17:16:21.583Z 2 040-059 81 81 81 142 142 142 F F F
87662024-04-17T17:16:21.584Z 3 060-079 74 74 74 134 134 134 F F F
87672024-04-17T17:16:21.584Z 4 080-099 81 81 81 142 142 142 F F F
87682024-04-17T17:16:21.584Z 5 100-119 81 81 81 142 142 142 F F F
87692024-04-17T17:16:21.584Z 6 120-139 81 81 81 142 142 142 F F F
87702024-04-17T17:16:21.584Z 7 140-159 68 68 68 124 124 124 F F F
87712024-04-17T17:16:21.584Z 8 160-179 81 81 81 142 142 142 F F F
87722024-04-17T17:16:21.584Z 9 180-199 81 81 81 142 142 142 F F F
87732024-04-17T17:16:21.584Z 10 200-219 81 81 81 142 142 142 F F F
87742024-04-17T17:16:21.584Z 11 220-239 81 81 81 142 142 142 F F F
87752024-04-17T17:16:21.587Z 12 240-259 76 76 76 138 138 138 F F F
87762024-04-17T17:16:21.587Z 13 260-279 76 76 76 138 138 138 F F F
87772024-04-17T17:16:21.587Z 14 280-299 75 75 75 135 135 135 F F F
87782024-04-17T17:16:21.587Z 15 300-319 78 78 78 140 140 140 F F F
87792024-04-17T17:16:21.587Z 16 320-339 81 81 81 142 142 142 F F F
87802024-04-17T17:16:21.588Z 17 340-359 63 63 63 114 114 114 F F F
87812024-04-17T17:16:21.588Z 18 360-379 81 81 81 142 142 142 F F F
87822024-04-17T17:16:21.588Z 19 380-399 81 81 81 142 142 142 F F F
87832024-04-17T17:16:21.588Z 20 400-419 78 78 78 140 140 140 F F F
87842024-04-17T17:16:21.588Z 21 420-439 73 73 73 132 132 132 F F F
87852024-04-17T17:16:21.588Z 22 440-459 68 68 68 123 123 123 F F F
87862024-04-17T17:16:21.588Z 23 460-479 81 81 81 142 142 142 F F F
87872024-04-17T17:16:21.588Z 24 480-499 78 78 78 140 140 140 F F F
87882024-04-17T17:16:21.588Z 25 500-519 70 70 70 127 127 127 F F F
87892024-04-17T17:16:21.588Z 26 520-539 81 81 81 142 142 142 F F F
87902024-04-17T17:16:21.588Z 27 540-559 81 81 81 142 142 142 F F F
87912024-04-17T17:16:21.588Z 28 560-579 78 78 78 140 140 140 F F F
87922024-04-17T17:16:21.588Z 29 580-599 78 78 78 140 140 140 F F F
87932024-04-17T17:16:21.588ZMax gen: 81, Max flush: 142
87942024-04-17T17:16:21.588Z
87952024-04-17T17:16:21.588ZBegin loop 41
87962024-04-17T17:16:21.588ZDownstairs to restart: 0
87972024-04-17T17:16:21.606Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.214326104Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499}
87982024-04-17T17:16:21.613Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.214634972Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499}
87992024-04-17T17:16:21.613Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30Crucible runtime is spawned
88002024-04-17T17:16:21.613Z,"time":"2024-04-17T17:16:20.214675725Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499}
88012024-04-17T17:16:21.613Z{"msg":"Crucible 284dc99e-5310-49ee-b3ec-b39a15012246 has session id: 3241bce3-0432-4d83-a565-2c146202f55a","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.214711067Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88022024-04-17T17:16:21.613Z{"msg":"Upstairs opts: Upstairs UUID: 284dc99e-5310-49ee-b3ec-b39a15012246, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.214779111Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88032024-04-17T17:16:21.613Z{"msg":"Crucible stats registered with UUID: 284dc99e-5310-49ee-b3ec-b39a15012246","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.214821414Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88042024-04-17T17:16:21.613Z{"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.214849585Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499}
88052024-04-17T17:16:21.613Z{"msg":"284dc99e-5310-49ee-b3ec-b39a15012246 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.214880367Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88062024-04-17T17:16:21.613Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.214906249Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"client":"0","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88072024-04-17T17:16:21.614Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.214935701Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"client":"0","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88082024-04-17T17:16:21.614Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.214981504Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"client":"1","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88092024-04-17T17:16:21.614Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.215027126Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"client":"1","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88102024-04-17T17:16:21.614Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.215111151Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"client":"2","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88112024-04-17T17:16:21.614Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.215174375Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"client":"2","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88122024-04-17T17:16:21.614Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.215228799Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"":"io task","client":"2","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88132024-04-17T17:16:21.614Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.215302143Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"":"io task","client":"1","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88142024-04-17T17:16:21.614Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.215372007Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"":"io task","client":"0","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88152024-04-17T17:16:21.614Z{"msg":"ds_connection connected from Ok(127.0.0.1:38824)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.215442652Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"":"io task","client":"2","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88162024-04-17T17:16:21.614Z{"msg":"ds_connection connected from Ok(127.0.0.1:64271)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.215480474Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"":"io task","client":"1","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88172024-04-17T17:16:21.614Z{"msg":"ds_connection connected from Ok(127.0.0.1:51170)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.215496485Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"":"io task","client":"0","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88182024-04-17T17:16:21.614Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.216062879Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"client":"2","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88192024-04-17T17:16:21.614Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.216097431Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"client":"2","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88202024-04-17T17:16:21.614Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.216113602Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"client":"1","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88212024-04-17T17:16:21.614Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.216147644Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"client":"1","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88222024-04-17T17:16:21.614Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.21658079Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"client":"0","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88232024-04-17T17:16:21.614Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.216633564Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"client":"0","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88242024-04-17T17:16:21.614Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.216673626Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"client":"1","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88252024-04-17T17:16:21.614Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.216813504Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"client":"2","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88262024-04-17T17:16:21.614Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.217179427Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"client":"0","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88272024-04-17T17:16:21.614Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.217357168Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"client":"2","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88282024-04-17T17:16:21.614Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.21739312Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"client":"2","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88292024-04-17T17:16:21.614Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.217408711Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88302024-04-17T17:16:21.615Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.217428032Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"client":"1","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88312024-04-17T17:16:21.615Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.217441543Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"client":"1","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88322024-04-17T17:16:21.615Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.217454343Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88332024-04-17T17:16:21.615Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.217614133Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"client":"0","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88342024-04-17T17:16:21.615Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.217644905Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"client":"0","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88352024-04-17T17:16:21.615Z{"msg":"[0]R flush_numbers[0..12]: [141, 142, 142, 134, 142, 142, 142, 124, 142, 142, 142, 142]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.217659646Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"":"downstairs","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88362024-04-17T17:16:21.615Z{"msg":"[0]R generation[0..12]: [80, 81, 81, 74, 81, 81, 81, 68, 81, 81, 81, 81]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.217674837Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"":"downstairs","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88372024-04-17T17:16:21.615Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.217696278Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"":"downstairs","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88382024-04-17T17:16:21.615Z{"msg":"[1]R flush_numbers[0..12]: [141, 142, 142, 134, 142, 142, 142, 124, 142, 142, 142, 142]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.217710549Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"":"downstairs","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88392024-04-17T17:16:21.615Z{"msg":"[1]R generation[0..12]: [80, 81, 81, 74, 81, 81, 81, 68, 81, 81, 81, 81]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.21773055Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"":"downstairs","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88402024-04-17T17:16:21.615Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.217745001Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"":"downstairs","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88412024-04-17T17:16:21.615Z{"msg":"[2]R flush_numbers[0..12]: [141, 142, 142, 134, 142, 142, 142, 124, 142, 142, 142, 142]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.217765242Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"":"downstairs","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88422024-04-17T17:16:21.615Z{"msg":"[2]R generation[0..12]: [80, 81, 81, 74, 81, 81, 81, 68, 81, 81, 81, 81]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.217778863Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"":"downstairs","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88432024-04-17T17:16:21.615Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
88442024-04-17T17:16:21.615Z2024-04-17T17:16:20.217799244Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"":"downstairs","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88452024-04-17T17:16:21.615Z{"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.217815215Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"":"downstairs","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88462024-04-17T17:16:21.615Z{"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.217834786Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"":"downstairs","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88472024-04-17T17:16:21.615Z{"msg":"Next flush: 143","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.217847467Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"":"downstairs","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88482024-04-17T17:16:21.615Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.217866588Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"":"downstairs","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88492024-04-17T17:16:21.615Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.217880179Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88502024-04-17T17:16:21.615Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.21789838Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"client":"0","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88512024-04-17T17:16:21.615Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.217911831Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"client":"1","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88522024-04-17T17:16:21.615Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.217931652Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"client":"2","":"downstairs","session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88532024-04-17T17:16:21.615Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.217945303Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88542024-04-17T17:16:21.616Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.217957754Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88552024-04-17T17:16:21.616Z{"msg":"284dc99e-5310-49ee-b3ec-b39a15012246 is now active with session: 3241bce3-0432-4d83-a565-2c146202f55a","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.217970245Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88562024-04-17T17:16:21.616Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.217981875Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499,"session_id":"3241bce3-0432-4d83-a565-2c146202f55a"}
88572024-04-17T17:16:21.616Z{"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:20.218000547Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499}
88582024-04-17T17:16:21.616ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
88592024-04-17T17:16:21.619ZLoading write count information from file "/tmp/test_repair_verify.data"
88602024-04-17T17:16:21.619ZRun Repair workload
88612024-04-17T17:16:21.620Z01/30 Write block 405 len 36864 data: 8 9 8 6 8 9 11 10 11
88622024-04-17T17:16:21.620Z02/30 Read block 305 len 40960
88632024-04-17T17:16:21.620Z03/30 Write block 446 len 4096 data: 6
88642024-04-17T17:16:21.620Z04/30 Read block 110 len 12288
88652024-04-17T17:16:21.620Z05/30 Write block 256 len 32768 data: 8 8 7 8 6 7 6 6
88662024-04-17T17:16:21.620Z06/30 Flush
88672024-04-17T17:16:21.623Z07/30 Write block 221 len 20480 data: 7 7 9 9 9
88682024-04-17T17:16:21.623Z08/30 Read block 480 len 28672
88692024-04-17T17:16:21.623Z09/30 Read block 263 len 12288
88702024-04-17T17:16:21.623Z10/30 Write block 454 len 28672 data: 6 6 5 5 7 7 8
88712024-04-17T17:16:21.626Z11/30 Write block 211 len 20480 data: 5 6 6 7 9
88722024-04-17T17:16:21.626Z12/30 Read block 258 len 20480
88732024-04-17T17:16:21.627Z13/30 Write block 75 len 16384 data: 4 4 4 2
88742024-04-17T17:16:21.627Z14/30 Read block 506 len 28672
88752024-04-17T17:16:21.627Z15/30 Write block 495 len 8192 data: 7 6
88762024-04-17T17:16:21.630Z16/30 Write block 313 len 4096 data: 9
88772024-04-17T17:16:21.630Z17/30 Write block 246 len 24576 data: 9 8 6 5 6 7
88782024-04-17T17:16:21.630Z18/30 Flush
88792024-04-17T17:16:21.634Z19/30 Write block 268 len 36864 data: 4 6 6 7 7 8 8 10 10
88802024-04-17T17:16:21.634Z20/30 Read block 546 len 20480
88812024-04-17T17:16:21.637Z21/30 Write block 330 len 32768 data: 5 6 6 6 7 8 9 11
88822024-04-17T17:16:21.637Z22/30 Write block 587 len 8192 data: 10 6
88832024-04-17T17:16:21.637Z23/30 Read block 278 len 28672
88842024-04-17T17:16:21.637Z24/30 Flush
88852024-04-17T17:16:21.640Z25/30 Write block 395 len 32768 data: 5 5 6 7 7 7 7 6
88862024-04-17T17:16:21.641Z26/30 Read block 296 len 12288
88872024-04-17T17:16:21.641Z27/30 Write block 194 len 12288 data: 5 6 5
88882024-04-17T17:16:21.647Z28/30 Write block 119 len 36864 data: 7 8 8 7 6 6 6 6 7
88892024-04-17T17:16:21.647Z29/30 Write block 350 len 4096 data: 8
88902024-04-17T17:16:21.647Z30/30 Read block 179 len 8192
88912024-04-17T17:16:21.647Z----------------------------------------------------------------
88922024-04-17T17:16:21.647Z Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:24
88932024-04-17T17:16:21.647ZStates: Active Active Active
88942024-04-17T17:16:21.647ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
88952024-04-17T17:16:21.647Z 7 Acked 1006 Write 5 Done Done Done false
88962024-04-17T17:16:21.647Z 8 Acked 1007 Read 7 Sent Done Done false
88972024-04-17T17:16:21.647Z 9 Acked 1008 Read 3 Sent Done Done false
88982024-04-17T17:16:21.647Z 10 Acked 1009 Write 7 Sent Done Done false
88992024-04-17T17:16:21.648Z 11 Acked 1010 Write 5 Sent Done Done false
89002024-04-17T17:16:21.648Z 12 Acked 1011 Read 5 Sent Done Done false
89012024-04-17T17:16:21.648Z 13 Acked 1012 Write 4 Sent Done Done false
89022024-04-17T17:16:21.648Z 14 Acked 1013 Read 7 Sent Done Done false
89032024-04-17T17:16:21.648Z 15 Acked 1014 Write 2 Sent Done Done false
89042024-04-17T17:16:21.648Z 16 Acked 1015 Write 1 Sent Done Done false
89052024-04-17T17:16:21.648Z 17 Acked 1016 Write 6 Sent Done Done false
89062024-04-17T17:16:21.648Z 18 Acked 1017 Flush 0 Sent Done Done false
89072024-04-17T17:16:21.648Z 19 Acked 1018 Write 9 Sent Done Done false
89082024-04-17T17:16:21.648Z 20 Acked 1019 Read 5 Sent Done Done false
89092024-04-17T17:16:21.648Z 21 Acked 1020 Write 8 Sent Done Done false
89102024-04-17T17:16:21.648Z 22 Acked 1021 Write 2 Sent Done Done false
89112024-04-17T17:16:21.648Z 23 Acked 1022 Read 7 Sent Done Done false
89122024-04-17T17:16:21.648Z 24 Acked 1023 Flush 0 Sent Done Done false
89132024-04-17T17:16:21.648Z 25 Acked 1024 Write 8 Sent Done Done false
89142024-04-17T17:16:21.648Z 26 Acked 1025 Read 3 Sent Done Done false
89152024-04-17T17:16:21.648Z 27 Acked 1026 Write 3 Sent Done Done false
89162024-04-17T17:16:21.648Z 28 Acked 1027 Write 9 Sent Done Done false
89172024-04-17T17:16:21.648Z 29 Acked 1028 Write 1 Sent Done Done false
89182024-04-17T17:16:21.648Z 30 Acked 1029 Read 2 Sent Sent Done false
89192024-04-17T17:16:21.648Z STATES DS:0 DS:1 DS:2 TOTAL
89202024-04-17T17:16:21.648Z New 0 0 0 0
89212024-04-17T17:16:21.648Z Sent 23 1 0 24
89222024-04-17T17:16:21.649Z Done 1 23 24 48
89232024-04-17T17:16:21.649Z Skipped 0 0 0 0
89242024-04-17T17:16:21.649Z Error 0 0 0 0
89252024-04-17T17:16:21.649ZLast Flush: 1005 1023 1023
89262024-04-17T17:16:21.649ZDownstairs last five completed: 1005 1004 1003 1002 1001
89272024-04-17T17:16:21.649ZUpstairs last five completed: 30 29 28 27 26
89282024-04-17T17:16:21.649Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:16:20.251489266Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1499}
89292024-04-17T17:16:21.649ZWrote out file "/tmp/test_repair_verify.data"
89302024-04-17T17:16:21.653Z
89312024-04-17T17:16:21.653ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
89322024-04-17T17:16:21.667ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
89332024-04-17T17:16:21.667Z 0 000-019 80 80 80 141 141 141 F F F
89342024-04-17T17:16:21.671Z 1 020-039 81 81 81 142 142 142 F F F
89352024-04-17T17:16:21.671Z 2 040-059 81 81 81 142 142 142 F F F
89362024-04-17T17:16:21.671Z 3 060-079 74 82 82 134 144 144 F F F <---
89372024-04-17T17:16:21.671Z 4 080-099 81 81 81 142 142 142 F F F
89382024-04-17T17:16:21.671Z 5 100-119 81 81 81 142 142 142 F T T <---
89392024-04-17T17:16:21.671Z 6 120-139 81 81 81 142 142 142 F T T <---
89402024-04-17T17:16:21.671Z 7 140-159 68 68 68 124 124 124 F F F
89412024-04-17T17:16:21.671Z 8 160-179 81 81 81 142 142 142 F F F
89422024-04-17T17:16:21.671Z 9 180-199 81 81 81 142 142 142 F T T <---
89432024-04-17T17:16:21.671Z 10 200-219 81 82 82 142 144 144 F F F <---
89442024-04-17T17:16:21.671Z 11 220-239 81 82 82 142 144 144 T F F <---
89452024-04-17T17:16:21.671Z 12 240-259 82 82 82 143 144 144 F F F <---
89462024-04-17T17:16:21.671Z 13 260-279 82 82 82 143 145 145 F F F <---
89472024-04-17T17:16:21.671Z 14 280-299 75 75 75 135 135 135 F F F
89482024-04-17T17:16:21.671Z 15 300-319 78 82 82 140 144 144 F F F <---
89492024-04-17T17:16:21.671Z 16 320-339 81 82 82 142 145 145 F F F <---
89502024-04-17T17:16:21.671Z 17 340-359 63 63 63 114 114 114 F T T <---
89512024-04-17T17:16:21.671Z 18 360-379 81 81 81 142 142 142 F F F
89522024-04-17T17:16:21.671Z 19 380-399 81 81 81 142 142 142 F T T <---
89532024-04-17T17:16:21.671Z 20 400-419 82 82 82 143 143 143 F T T <---
89542024-04-17T17:16:21.671Z 21 420-439 73 73 73 132 132 132 F F F
89552024-04-17T17:16:21.672Z 22 440-459 82 82 82 143 144 144 F F F <---
89562024-04-17T17:16:21.672Z 23 460-479 81 82 82 142 144 144 F F F <---
89572024-04-17T17:16:21.672Z 24 480-499 78 82 82 140 144 144 F F F <---
89582024-04-17T17:16:21.672Z 25 500-519 70 70 70 127 127 127 F F F
89592024-04-17T17:16:21.672Z 26 520-539 81 81 81 142 142 142 F F F
89602024-04-17T17:16:21.672Z 27 540-559 81 81 81 142 142 142 F F F
89612024-04-17T17:16:21.672Z 28 560-579 78 78 78 140 140 140 F F F
89622024-04-17T17:16:21.672Z 29 580-599 78 82 82 140 145 145 F F F <---
89632024-04-17T17:16:21.672ZMax gen: 82, Max flush: 145
89642024-04-17T17:16:21.672ZError: Difference in extent metadata found!
89652024-04-17T17:16:21.675ZA Difference in extent metadata is expected here
89662024-04-17T17:16:21.676ZOn loop 41
89672024-04-17T17:16:21.676Z
89682024-04-17T17:16:21.676ZVerifying data now
89692024-04-17T17:16:26.445ZNo change after verify
89702024-04-17T17:16:26.445ZLoop: 41 Downstairs dump after verify (and repair):
89712024-04-17T17:16:26.459ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
89722024-04-17T17:16:26.459Z 0 000-019 80 80 80 141 141 141 F F F
89732024-04-17T17:16:26.459Z 1 020-039 81 81 81 142 142 142 F F F
89742024-04-17T17:16:26.459Z 2 040-059 81 81 81 142 142 142 F F F
89752024-04-17T17:16:26.459Z 3 060-079 82 82 82 144 144 144 F F F
89762024-04-17T17:16:26.459Z 4 080-099 81 81 81 142 142 142 F F F
89772024-04-17T17:16:26.463Z 5 100-119 83 83 83 146 146 146 F F F
89782024-04-17T17:16:26.463Z 6 120-139 83 83 83 146 146 146 F F F
89792024-04-17T17:16:26.463Z 7 140-159 68 68 68 124 124 124 F F F
89802024-04-17T17:16:26.463Z 8 160-179 81 81 81 142 142 142 F F F
89812024-04-17T17:16:26.463Z 9 180-199 83 83 83 146 146 146 F F F
89822024-04-17T17:16:26.463Z 10 200-219 82 82 82 144 144 144 F F F
89832024-04-17T17:16:26.463Z 11 220-239 82 82 82 144 144 144 F F F
89842024-04-17T17:16:26.463Z 12 240-259 82 82 82 144 144 144 F F F
89852024-04-17T17:16:26.464Z 13 260-279 82 82 82 145 145 145 F F F
89862024-04-17T17:16:26.464Z 14 280-299 75 75 75 135 135 135 F F F
89872024-04-17T17:16:26.464Z 15 300-319 82 82 82 144 144 144 F F F
89882024-04-17T17:16:26.464Z 16 320-339 82 82 82 145 145 145 F F F
89892024-04-17T17:16:26.464Z 17 340-359 83 83 83 146 146 146 F F F
89902024-04-17T17:16:26.464Z 18 360-379 81 81 81 142 142 142 F F F
89912024-04-17T17:16:26.464Z 19 380-399 83 83 83 146 146 146 F F F
89922024-04-17T17:16:26.464Z 20 400-419 83 83 83 146 146 146 F F F
89932024-04-17T17:16:26.464Z 21 420-439 73 73 73 132 132 132 F F F
89942024-04-17T17:16:26.464Z 22 440-459 82 82 82 144 144 144 F F F
89952024-04-17T17:16:26.464Z 23 460-479 82 82 82 144 144 144 F F F
89962024-04-17T17:16:26.464Z 24 480-499 82 82 82 144 144 144 F F F
89972024-04-17T17:16:26.464Z 25 500-519 70 70 70 127 127 127 F F F
89982024-04-17T17:16:26.464Z 26 520-539 81 81 81 142 142 142 F F F
89992024-04-17T17:16:26.464Z 27 540-559 81 81 81 142 142 142 F F F
90002024-04-17T17:16:26.464Z 28 560-579 78 78 78 140 140 140 F F F
90012024-04-17T17:16:26.464Z 29 580-599 82 82 82 145 145 145 F F F
90022024-04-17T17:16:26.464ZMax gen: 83, Max flush: 146
90032024-04-17T17:16:26.465Z
90042024-04-17T17:16:26.465ZBegin loop 42
90052024-04-17T17:16:26.465ZDownstairs to restart: 0
90062024-04-17T17:16:26.482Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.089413928Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507}
90072024-04-17T17:16:26.482Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.089713226Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507}
90082024-04-17T17:16:26.482ZCrucible runtime is spawned
90092024-04-17T17:16:26.489Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.08977451Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507}
90102024-04-17T17:16:26.489Z{"msg":"Crucible 885b2b83-d3ae-4a74-a2cd-06c78a4f6b86 has session id: 8f3900cf-043d-46f1-bb06-650b9b837487","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.089805812Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90112024-04-17T17:16:26.489Z{"msg":"Upstairs opts: Upstairs UUID: 885b2b83-d3ae-4a74-a2cd-06c78a4f6b86, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.089869806Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90122024-04-17T17:16:26.489Z{"msg":"Crucible stats registered with UUID: 885b2b83-d3ae-4a74-a2cd-06c78a4f6b86","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.089917149Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90132024-04-17T17:16:26.489Z{"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.089961591Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507}
90142024-04-17T17:16:26.489Z{"msg":"885b2b83-d3ae-4a74-a2cd-06c78a4f6b86 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.090023165Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90152024-04-17T17:16:26.489Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.090050746Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"client":"0","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90162024-04-17T17:16:26.489Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.090067367Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"client":"0","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90172024-04-17T17:16:26.489Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.0900991Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"client":"1","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90182024-04-17T17:16:26.489Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.090120941Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"client":"1","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90192024-04-17T17:16:26.489Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.090142262Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"client":"2","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90202024-04-17T17:16:26.489Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.090166454Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"client":"2","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90212024-04-17T17:16:26.489Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.090201786Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"":"io task","client":"0","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90222024-04-17T17:16:26.490Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.090233528Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"":"io task","client":"2","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90232024-04-17T17:16:26.490Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.090257199Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"":"io task","client":"1","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90242024-04-17T17:16:26.490Z{"msg":"ds_connection connected from Ok(127.0.0.1:42910)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.090447681Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"":"io task","client":"0","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90252024-04-17T17:16:26.490Z{"msg":"ds_connection connected from Ok(127.0.0.1:32897)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.090501754Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"":"io task","client":"1","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90262024-04-17T17:16:26.490Z{"msg":"ds_connection connected from Ok(127.0.0.1:57925)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.090542807Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"":"io task","client":"2","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90272024-04-17T17:16:26.490Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.09125241Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"client":"2","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90282024-04-17T17:16:26.490Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.091322434Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"client":"2","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90292024-04-17T17:16:26.490Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.091372827Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"client":"1","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90302024-04-17T17:16:26.490Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.09141876Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"client":"1","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90312024-04-17T17:16:26.490Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.091460112Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"client":"0","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90322024-04-17T17:16:26.490Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.091485644Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"client":"0","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90332024-04-17T17:16:26.490Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.091767021Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"client":"1","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90342024-04-17T17:16:26.490Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.091906079Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"client":"0","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90352024-04-17T17:16:26.490Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.091943921Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"client":"2","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90362024-04-17T17:16:26.490Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.092230089Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"client":"1","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90372024-04-17T17:16:26.490Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.092291742Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"client":"1","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90382024-04-17T17:16:26.490Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.092319454Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90392024-04-17T17:16:26.490Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.092467503Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"client":"2","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90402024-04-17T17:16:26.490Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.092520046Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"client":"2","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90412024-04-17T17:16:26.490Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.092537397Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90422024-04-17T17:16:26.491Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.092553248Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"client":"0","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90432024-04-17T17:16:26.491Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.092574259Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"client":"0","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90442024-04-17T17:16:26.491Z{"msg":"[0]R flush_numbers[0..12]: [141, 142, 142, 144, 142, 146, 146, 124, 142, 146, 144, 144]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.092599321Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"":"downstairs","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90452024-04-17T17:16:26.491Z{"msg":"[0]R generation[0..12]: [80, 81, 81, 82, 81, 83, 83, 68, 81, 83, 82, 82]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.092614502Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"":"downstairs","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90462024-04-17T17:16:26.491Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.092637323Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"":"downstairs","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90472024-04-17T17:16:26.491Z{"msg":"[1]R flush_numbers[0..12]: [141, 142, 142, 144, 142, 146, 146, 124, 142, 146, 144, 144]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.092651624Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"":"downstairs","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90482024-04-17T17:16:26.491Z{"msg":"[1]R generation[0..12]: [80, 81, 81, 82, 81, 83, 83, 68, 81, 83, 82, 82]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.092671776Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"":"downstairs","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90492024-04-17T17:16:26.491Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.092685886Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"":"downstairs","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90502024-04-17T17:16:26.491Z{"msg":"[2]R flush_numbers[0..12]: [141, 142, 142, 144, 142, 146, 146, 124, 142, 146, 144, 144]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.092706118Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"":"downstairs","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90512024-04-17T17:16:26.491Z{"msg":"[2]R generation[0..12]: [80, 81, 81, 82, 81, 83, 83, 68, 81, 83, 82, 82]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.092719938Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"":"downstairs","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90522024-04-17T17:16:26.491Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.09274077Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"":"downstairs","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90532024-04-17T17:16:26.491Z{"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.09275375Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"":"downstairs","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90542024-04-17T17:16:26.491Z{"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.092766781Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"":"downstairs","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90552024-04-17T17:16:26.491Z{"msg":"Next flush: 147","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.092785982Z","hostname":"Wait for a query_work_queue command to finish before sending IO
90562024-04-17T17:16:26.491Zw-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"":"downstairs","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90572024-04-17T17:16:26.491Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.092802263Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"":"downstairs","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90582024-04-17T17:16:26.491Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.092815224Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90592024-04-17T17:16:26.492Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.092833665Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"client":"0","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90602024-04-17T17:16:26.492Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.092846906Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"client":"1","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90612024-04-17T17:16:26.492Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.092860527Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"client":"2","":"downstairs","session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90622024-04-17T17:16:26.492Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.092880418Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90632024-04-17T17:16:26.492Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.092892679Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90642024-04-17T17:16:26.492Z{"msg":"885b2b83-d3ae-4a74-a2cd-06c78a4f6b86 is now active with session: 8f3900cf-043d-46f1-bb06-650b9b837487","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.09290525Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90652024-04-17T17:16:26.492Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.092930921Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507,"session_id":"8f3900cf-043d-46f1-bb06-650b9b837487"}
90662024-04-17T17:16:26.492Z{"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:25.092947302Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507}
90672024-04-17T17:16:26.492ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
90682024-04-17T17:16:26.492ZLoading write count information from file "/tmp/test_repair_verify.data"
90692024-04-17T17:16:26.492ZRun Repair workload
90702024-04-17T17:16:26.496Z01/30 Write block 83 len 4096 data: 6
90712024-04-17T17:16:26.496Z02/30 Read block 358 len 40960
90722024-04-17T17:16:26.496Z03/30 Read block 567 len 8192
90732024-04-17T17:16:26.496Z04/30 Flush
90742024-04-17T17:16:26.496Z05/30 Write block 209 len 12288 data: 5 5 6
90752024-04-17T17:16:26.497Z06/30 Read block 283 len 40960
90762024-04-17T17:16:26.497Z07/30 Read block 42 len 12288
90772024-04-17T17:16:26.497Z08/30 Read block 373 len 12288
90782024-04-17T17:16:26.497Z09/30 Write block 325 len 4096 data: 5
90792024-04-17T17:16:26.497Z10/30 Flush
90802024-04-17T17:16:26.497Z11/30 Write block 497 len 8192 data: 6 8
90812024-04-17T17:16:26.497Z12/30 Read block 546 len 36864
90822024-04-17T17:16:26.500Z13/30 Write block 75 len 20480 data: 5 5 5 3 2
90832024-04-17T17:16:26.500Z14/30 Flush
90842024-04-17T17:16:26.505Z15/30 Write block 87 len 32768 data: 8 7 10 11 12 12 11 9
90852024-04-17T17:16:26.505Z16/30 Read block 145 len 20480
90862024-04-17T17:16:26.505Z17/30 Read block 213 len 4096
90872024-04-17T17:16:26.505Z18/30 Read block 433 len 40960
90882024-04-17T17:16:26.505Z19/30 Read block 457 len 32768
90892024-04-17T17:16:26.505Z20/30 Read block 567 len 36864
90902024-04-17T17:16:26.508Z21/30 Read block 61 len 4096
90912024-04-17T17:16:26.508Z22/30 Read block 195 len 32768
90922024-04-17T17:16:26.509Z23/30 Write block 265 len 40960 data: 3 3 3 5 7 7 8 8 9 9
90932024-04-17T17:16:26.509Z24/30 Write block 63 len 24576 data: 9 10 8 7 7 6
90942024-04-17T17:16:26.512Z25/30 Write block 220 len 8192 data: 9 8
90952024-04-17T17:16:26.512Z26/30 Write block 278 len 4096 data: 5
90962024-04-17T17:16:26.512Z27/30 Read block 411 len 24576
90972024-04-17T17:16:26.512Z28/30 Write block 313 len 40960 data: 10 10 9 8 9 9 8 6 5 3
90982024-04-17T17:16:26.516Z29/30 Write block 562 len 12288 data: 8 7 8
90992024-04-17T17:16:26.516Z30/30 Write block 235 len 36864 data: 9 6 5 4 6 6 9 7 9
91002024-04-17T17:16:26.516Z----------------------------------------------------------------
91012024-04-17T17:16:26.516Z Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:30
91022024-04-17T17:16:26.516ZStates: Active Active Active
91032024-04-17T17:16:26.517ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
91042024-04-17T17:16:26.517Z 1 Acked 1000 Write 1 Sent Done Done false
91052024-04-17T17:16:26.517Z 2 Acked 1001 Read 10 Sent Done Done false
91062024-04-17T17:16:26.517Z 3 Acked 1002 Read 2 Sent Done Done false
91072024-04-17T17:16:26.517Z 4 Acked 1003 Flush 0 Sent Done Done false
91082024-04-17T17:16:26.517Z 5 Acked 1004 Write 3 Sent Done Done false
91092024-04-17T17:16:26.517Z 6 Acked 1005 Read 10 Sent Done Done false
91102024-04-17T17:16:26.517Z 7 Acked 1006 Read 3 Sent Done Done false
91112024-04-17T17:16:26.517Z 8 Acked 1007 Read 3 Sent Done Done false
91122024-04-17T17:16:26.517Z 9 Acked 1008 Write 1 Sent Done Done false
91132024-04-17T17:16:26.517Z 10 Acked 1009 Flush 0 Sent Done Done false
91142024-04-17T17:16:26.517Z 11 Acked 1010 Write 2 Sent Done Done false
91152024-04-17T17:16:26.517Z 12 Acked 1011 Read 9 Sent Done Done false
91162024-04-17T17:16:26.517Z 13 Acked 1012 Write 5 Sent Done Done false
91172024-04-17T17:16:26.517Z 14 Acked 1013 Flush 0 Sent Done Done false
91182024-04-17T17:16:26.518Z 15 Acked 1014 Write 8 Sent Done Done false
91192024-04-17T17:16:26.518Z 16 Acked 1015 Read 5 Sent Done Done false
91202024-04-17T17:16:26.518Z 17 Acked 1016 Read 1 Sent Done Done false
91212024-04-17T17:16:26.518Z 18 Acked 1017 Read 10 Sent Done Done false
91222024-04-17T17:16:26.518Z 19 Acked 1018 Read 8 Sent Done Done false
91232024-04-17T17:16:26.518Z 20 Acked 1019 Read 9 Sent Done Done false
91242024-04-17T17:16:26.518Z 21 Acked 1020 Read 1 Sent Done Done false
91252024-04-17T17:16:26.518Z 22 Acked 1021 Read 8 Sent Done Done false
91262024-04-17T17:16:26.518Z 23 Acked 1022 Write 10 Sent Done Done false
91272024-04-17T17:16:26.518Z 24 Acked 1023 Write 6 Sent Done Done false
91282024-04-17T17:16:26.518Z 25 Acked 1024 Write 2 Sent Done Done false
91292024-04-17T17:16:26.518Z 26 Acked 1025 Write 1 Sent Done Done false
91302024-04-17T17:16:26.518Z 27 Acked 1026 Read 6 Sent Done Done false
91312024-04-17T17:16:26.518Z 28 Acked 1027 Write 10 Sent Done Done false
91322024-04-17T17:16:26.518Z 29 Acked 1028 Write 3 Sent Done Done false
91332024-04-17T17:16:26.518Z 30 Acked 1029 Write 9 Sent Sent Sent false
91342024-04-17T17:16:26.518Z STATES DS:0 DS:1 DS:2 TOTAL
91352024-04-17T17:16:26.518Z New 0 0 0 0
91362024-04-17T17:16:26.518Z Sent 30 1 1 32
91372024-04-17T17:16:26.518Z Done 0 29 29 58
91382024-04-17T17:16:26.518Z Skipped 0 0 0 0
91392024-04-17T17:16:26.519Z Error 0 0 0 0
91402024-04-17T17:16:26.519ZLast Flush: 0 1013 1013
91412024-04-17T17:16:26.519ZDownstairs last five completed:
91422024-04-17T17:16:26.519ZUpstairs last five completed: 30 29 28 27 26
91432024-04-17T17:16:26.519Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:16:25.122970842Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1507}
91442024-04-17T17:16:26.519ZWrote out file "/tmp/test_repair_verify.data"
91452024-04-17T17:16:26.522Z
91462024-04-17T17:16:26.522ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
91472024-04-17T17:16:26.539ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
91482024-04-17T17:16:26.539Z 0 000-019 80 80 80 141 141 141 F F F
91492024-04-17T17:16:26.539Z 1 020-039 81 81 81 142 142 142 F F F
91502024-04-17T17:16:26.539Z 2 040-059 81 81 81 142 142 142 F F F
91512024-04-17T17:16:26.539Z 3 060-079 82 84 84 144 149 149 F T T <---
91522024-04-17T17:16:26.539Z 4 080-099 81 84 84 142 147 147 F T T <---
91532024-04-17T17:16:26.539Z 5 100-119 83 83 83 146 146 146 F F F
91542024-04-17T17:16:26.539Z 6 120-139 83 83 83 146 146 146 F F F
91552024-04-17T17:16:26.539Z 7 140-159 68 68 68 124 124 124 F F F
91562024-04-17T17:16:26.539Z 8 160-179 81 81 81 142 142 142 F F F
91572024-04-17T17:16:26.543Z 9 180-199 83 83 83 146 146 146 F F F
91582024-04-17T17:16:26.543Z 10 200-219 82 84 84 144 148 148 F F F <---
91592024-04-17T17:16:26.544Z 11 220-239 82 82 82 144 144 144 F T T <---
91602024-04-17T17:16:26.544Z 12 240-259 82 82 82 144 144 144 F T T <---
91612024-04-17T17:16:26.544Z 13 260-279 82 82 82 145 145 145 F T T <---
91622024-04-17T17:16:26.544Z 14 280-299 75 75 75 135 135 135 F F F
91632024-04-17T17:16:26.544Z 15 300-319 82 82 82 144 144 144 F T T <---
91642024-04-17T17:16:26.544Z 16 320-339 82 84 84 145 148 148 F T T <---
91652024-04-17T17:16:26.544Z 17 340-359 83 83 83 146 146 146 F F F
91662024-04-17T17:16:26.544Z 18 360-379 81 81 81 142 142 142 F F F
91672024-04-17T17:16:26.544Z 19 380-399 83 83 83 146 146 146 F F F
91682024-04-17T17:16:26.544Z 20 400-419 83 83 83 146 146 146 F F F
91692024-04-17T17:16:26.544Z 21 420-439 73 73 73 132 132 132 F F F
91702024-04-17T17:16:26.544Z 22 440-459 82 82 82 144 144 144 F F F
91712024-04-17T17:16:26.544Z 23 460-479 82 82 82 144 144 144 F F F
91722024-04-17T17:16:26.544Z 24 480-499 82 84 84 144 149 149 F F F <---
91732024-04-17T17:16:26.544Z 25 500-519 70 70 70 127 127 127 F F F
91742024-04-17T17:16:26.544Z 26 520-539 81 81 81 142 142 142 F F F
91752024-04-17T17:16:26.544Z 27 540-559 81 81 81 142 142 142 F F F
91762024-04-17T17:16:26.544Z 28 560-579 78 78 78 140 140 140 F T T <---
91772024-04-17T17:16:26.544Z 29 580-599 82 82 82 145 145 145 F F F
91782024-04-17T17:16:26.544ZMax gen: 84, Max flush: 149
91792024-04-17T17:16:26.544ZError: Difference in extent metadata found!
91802024-04-17T17:16:26.545ZA Difference in extent metadata is expected here
91812024-04-17T17:16:26.545ZOn loop 42
91822024-04-17T17:16:26.545Z
91832024-04-17T17:16:26.556ZVerifying data now
91842024-04-17T17:16:31.039ZNo change after verify
91852024-04-17T17:16:31.039ZLoop: 42 Downstairs dump after verify (and repair):
91862024-04-17T17:16:31.054ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
91872024-04-17T17:16:31.055Z 0 000-019 80 80 80 141 141 141 F F F
91882024-04-17T17:16:31.058Z 1 020-039 81 81 81 142 142 142 F F F
91892024-04-17T17:16:31.059Z 2 040-059 81 81 81 142 142 142 F F F
91902024-04-17T17:16:31.059Z 3 060-079 85 85 85 150 150 150 F F F
91912024-04-17T17:16:31.059Z 4 080-099 85 85 85 150 150 150 F F F
91922024-04-17T17:16:31.059Z 5 100-119 83 83 83 146 146 146 F F F
91932024-04-17T17:16:31.059Z 6 120-139 83 83 83 146 146 146 F F F
91942024-04-17T17:16:31.059Z 7 140-159 68 68 68 124 124 124 F F F
91952024-04-17T17:16:31.059Z 8 160-179 81 81 81 142 142 142 F F F
91962024-04-17T17:16:31.059Z 9 180-199 83 83 83 146 146 146 F F F
91972024-04-17T17:16:31.059Z 10 200-219 84 84 84 148 148 148 F F F
91982024-04-17T17:16:31.059Z 11 220-239 85 85 85 150 150 150 F F F
91992024-04-17T17:16:31.059Z 12 240-259 85 85 85 150 150 150 F F F
92002024-04-17T17:16:31.059Z 13 260-279 85 85 85 150 150 150 F F F
92012024-04-17T17:16:31.059Z 14 280-299 75 75 75 135 135 135 F F F
92022024-04-17T17:16:31.059Z 15 300-319 85 85 85 150 150 150 F F F
92032024-04-17T17:16:31.059Z 16 320-339 85 85 85 150 150 150 F F F
92042024-04-17T17:16:31.059Z 17 340-359 83 83 83 146 146 146 F F F
92052024-04-17T17:16:31.059Z 18 360-379 81 81 81 142 142 142 F F F
92062024-04-17T17:16:31.059Z 19 380-399 83 83 83 146 146 146 F F F
92072024-04-17T17:16:31.060Z 20 400-419 83 83 83 146 146 146 F F F
92082024-04-17T17:16:31.060Z 21 420-439 73 73 73 132 132 132 F F F
92092024-04-17T17:16:31.060Z 22 440-459 82 82 82 144 144 144 F F F
92102024-04-17T17:16:31.060Z 23 460-479 82 82 82 144 144 144 F F F
92112024-04-17T17:16:31.060Z 24 480-499 84 84 84 149 149 149 F F F
92122024-04-17T17:16:31.060Z 25 500-519 70 70 70 127 127 127 F F F
92132024-04-17T17:16:31.060Z 26 520-539 81 81 81 142 142 142 F F F
92142024-04-17T17:16:31.060Z 27 540-559 81 81 81 142 142 142 F F F
92152024-04-17T17:16:31.060Z 28 560-579 85 85 85 150 150 150 F F F
92162024-04-17T17:16:31.060Z 29 580-599 82 82 82 145 145 145 F F F
92172024-04-17T17:16:31.060ZMax gen: 85, Max flush: 150
92182024-04-17T17:16:31.064Z
92192024-04-17T17:16:31.064ZBegin loop 43
92202024-04-17T17:16:31.064ZDownstairs to restart: 1
92212024-04-17T17:16:31.082Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.68639506Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515}
92222024-04-17T17:16:31.082Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.686663316Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515}
92232024-04-17T17:16:31.082Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.686683667Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515}
92242024-04-17T17:16:31.082Z{"msg":"Crucible 38dbaa92-9977-4c20-a9db-0dd3bd9e59f6 has session id: a84a89d0-6e86-4017-92f2-8502673fbd86","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.686704679Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92252024-04-17T17:16:31.090Z{"msg":"Upstairs opts: Upstairs UUID: 38dbaa92-9977-4c20-a9db-0dd3bd9e59f6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
92262024-04-17T17:16:31.090Z,"time":"2024-04-17T17:16:29.686795984Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92272024-04-17T17:16:31.090Z{"msg":"Crucible stats registered with UUID: 38dbaa92-9977-4c20-a9db-0dd3bd9e59f6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.686875819Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92282024-04-17T17:16:31.090Z{"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.686910981Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515}
92292024-04-17T17:16:31.090Z{"msg":"38dbaa92-9977-4c20-a9db-0dd3bd9e59f6 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.687029948Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92302024-04-17T17:16:31.090Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.687066931Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"client":"0","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92312024-04-17T17:16:31.090Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.687094202Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"client":"0","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92322024-04-17T17:16:31.090Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.687141085Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"client":"1","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92332024-04-17T17:16:31.090Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.687167147Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"client":"1","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92342024-04-17T17:16:31.090Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.687202119Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"client":"2","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92352024-04-17T17:16:31.090Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.687228851Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"client":"2","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92362024-04-17T17:16:31.091Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.687251522Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"":"io task","client":"0","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92372024-04-17T17:16:31.091Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.687324866Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92382024-04-17T17:16:31.091Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.687356078Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"":"io task","client":"1","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92392024-04-17T17:16:31.091Z{"msg":"ds_connection connected from Ok(127.0.0.1:41048)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.68738219Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"":"io task","client":"0","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92402024-04-17T17:16:31.091Z{"msg":"ds_connection connected from Ok(127.0.0.1:55160)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.687621254Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92412024-04-17T17:16:31.091Z{"msg":"ds_connection connected from Ok(127.0.0.1:57039)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.687687708Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"":"io task","client":"1","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92422024-04-17T17:16:31.091Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.688138596Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"client":"0","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92432024-04-17T17:16:31.091Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.688167197Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"client":"0","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92442024-04-17T17:16:31.091Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.688182648Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"client":"2","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92452024-04-17T17:16:31.091Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.688196439Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"client":"2","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92462024-04-17T17:16:31.091Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.688301776Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"client":"1","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92472024-04-17T17:16:31.091Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.68837302Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"client":"1","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92482024-04-17T17:16:31.091Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.688790205Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"client":"2","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92492024-04-17T17:16:31.092Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.688840768Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"client":"1","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92502024-04-17T17:16:31.092Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.68887482Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"client":"0","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92512024-04-17T17:16:31.092Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.68936003Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"client":"0","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92522024-04-17T17:16:31.092Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.689407132Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"client":"0","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92532024-04-17T17:16:31.092Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.689435204Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92542024-04-17T17:16:31.092Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.689472307Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"client":"2","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92552024-04-17T17:16:31.092Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.689503518Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"client":"2","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92562024-04-17T17:16:31.092Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.689517989Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92572024-04-17T17:16:31.092Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.68953703Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"client":"1","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92582024-04-17T17:16:31.092Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.689551301Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"client":"1","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92592024-04-17T17:16:31.092Z{"msg":"[0]R flush_numbers[0..12]: [141, 142, 142, 150, 150, 146, 146, 124, 142, 146, 148, 150]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.689565892Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"":"downstairs","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92602024-04-17T17:16:31.092Z{"msg":"[0]R generation[0..12]: [80, 81, 81, 85, 85, 83, 83, 68, 81, 83, 84, 85]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.689579483Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"":"downstairs","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92612024-04-17T17:16:31.092Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.689600184Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"":"downstairs","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92622024-04-17T17:16:31.092Z{"msg":"[1]R flush_numbers[0..12]: [141, 142, 142, 150, 150, 146, 146, 124, 142, 146, 148, 150]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.689614235Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"":"downstairs","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92632024-04-17T17:16:31.093Z{"msg":"[1]R generation[0..12]: [80, 81, 81, 85, 85, 83, 83, 68, 81, 83, 84, 85]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.689634276Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"":"downstairs","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92642024-04-17T17:16:31.093ZWait for a query_work_queue command to finish before sending IO
92652024-04-17T17:16:31.093Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.689654658Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"":"downstairs","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92662024-04-17T17:16:31.093Z{"msg":"[2]R flush_numbers[0..12]: [141, 142, 142, 150, 150, 146, 146, 124, 142, 146, 148, 150]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.689675559Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"":"downstairs","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92672024-04-17T17:16:31.093Z{"msg":"[2]R generation[0..12]: [80, 81, 81, 85, 85, 83, 83, 68, 81, 83, 84, 85]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.68968956Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"":"downstairs","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92682024-04-17T17:16:31.093Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.68970351Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"":"downstairs","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92692024-04-17T17:16:31.093Z{"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.689716011Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"":"downstairs","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92702024-04-17T17:16:31.093Z{"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.689728962Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"":"downstairs","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92712024-04-17T17:16:31.093Z{"msg":"Next flush: 151","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.689748713Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"":"downstairs","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92722024-04-17T17:16:31.093Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.689769594Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"":"downstairs","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92732024-04-17T17:16:31.093Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.689783515Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92742024-04-17T17:16:31.093Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.689807947Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"client":"0","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92752024-04-17T17:16:31.093Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.689837798Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"client":"1","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92762024-04-17T17:16:31.093Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.68985255Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"client":"2","":"downstairs","session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92772024-04-17T17:16:31.093Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.689872391Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92782024-04-17T17:16:31.093Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.689886212Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92792024-04-17T17:16:31.093Z{"msg":"38dbaa92-9977-4c20-a9db-0dd3bd9e59f6 is now active with session: a84a89d0-6e86-4017-92f2-8502673fbd86","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.689917983Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92802024-04-17T17:16:31.093Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.689939255Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515,"session_id":"a84a89d0-6e86-4017-92f2-8502673fbd86"}
92812024-04-17T17:16:31.094Z{"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:29.689956786Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515}
92822024-04-17T17:16:31.094ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
92832024-04-17T17:16:31.094ZLoading write count information from file "/tmp/test_repair_verify.data"
92842024-04-17T17:16:31.094ZRun Repair workload
92852024-04-17T17:16:31.094Z01/30 Write block 196 len 8192 data: 6 6
92862024-04-17T17:16:31.094Z02/30 Read block 109 len 16384
92872024-04-17T17:16:31.094Z03/30 Read block 325 len 4096
92882024-04-17T17:16:31.094Z04/30 Flush
92892024-04-17T17:16:31.098Z05/30 Write block 273 len 24576 data: 10 10 11 11 5 6
92902024-04-17T17:16:31.098Z06/30 Read block 66 len 32768
92912024-04-17T17:16:31.098Z07/30 Read block 134 len 40960
92922024-04-17T17:16:31.098Z08/30 Read block 105 len 16384
92932024-04-17T17:16:31.098Z09/30 Read block 430 len 24576
92942024-04-17T17:16:31.098Z10/30 Read block 435 len 4096
92952024-04-17T17:16:31.098Z11/30 Read block 541 len 32768
92962024-04-17T17:16:31.098Z12/30 Write block 290 len 24576 data: 6 5 4 4 3 3
92972024-04-17T17:16:31.098Z13/30 Flush
92982024-04-17T17:16:31.098Z14/30 Write block 342 len 36864 data: 11 11 10 11 10 9 8 8 9
92992024-04-17T17:16:31.098Z15/30 Write block 233 len 16384 data: 9 9 10 7
93002024-04-17T17:16:31.099Z16/30 Read block 123 len 8192
93012024-04-17T17:16:31.102Z17/30 Read block 119 len 32768
93022024-04-17T17:16:31.102Z18/30 Read block 160 len 20480
93032024-04-17T17:16:31.102Z19/30 Read block 43 len 40960
93042024-04-17T17:16:31.102Z20/30 Read block 133 len 32768
93052024-04-17T17:16:31.102Z21/30 Read block 587 len 32768
93062024-04-17T17:16:31.105Z22/30 Write block 486 len 36864 data: 13 10 10 8 8 5 5 5 5
93072024-04-17T17:16:31.106Z23/30 Read block 490 len 36864
93082024-04-17T17:16:31.106Z24/30 Flush
93092024-04-17T17:16:31.109Z25/30 Flush
93102024-04-17T17:16:31.109Z26/30 Write block 260 len 32768 data: 7 8 7 7 6 4 4 4
93112024-04-17T17:16:31.113Z27/30 Read block 101 len 32768
93122024-04-17T17:16:31.113Z28/30 Write block 539 len 28672 data: 6 6 7 8 9 8 6
93132024-04-17T17:16:31.113Z29/30 Read block 82 len 36864
93142024-04-17T17:16:31.113Z30/30 Read block 596 len 8192
93152024-04-17T17:16:31.113Z----------------------------------------------------------------
93162024-04-17T17:16:31.113Z Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:30
93172024-04-17T17:16:31.114ZStates: Active Active Active
93182024-04-17T17:16:31.114ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
93192024-04-17T17:16:31.114Z 1 Acked 1000 Write 2 Done Done Done false
93202024-04-17T17:16:31.114Z 2 Acked 1001 Read 4 Done Done Done false
93212024-04-17T17:16:31.114Z 3 Acked 1002 Read 1 Done Sent Done false
93222024-04-17T17:16:31.114Z 4 Acked 1003 Flush 0 Done Sent Done false
93232024-04-17T17:16:31.114Z 5 Acked 1004 Write 6 Done Sent Done false
93242024-04-17T17:16:31.114Z 6 Acked 1005 Read 8 Done Sent Done false
93252024-04-17T17:16:31.114Z 7 Acked 1006 Read 10 Done Sent Done false
93262024-04-17T17:16:31.114Z 8 Acked 1007 Read 4 Done Sent Done false
93272024-04-17T17:16:31.114Z 9 Acked 1008 Read 6 Done Sent Done false
93282024-04-17T17:16:31.114Z 10 Acked 1009 Read 1 Done Sent Done false
93292024-04-17T17:16:31.114Z 11 Acked 1010 Read 8 Done Sent Done false
93302024-04-17T17:16:31.114Z 12 Acked 1011 Write 6 Done Sent Done false
93312024-04-17T17:16:31.114Z 13 Acked 1012 Flush 0 Done Sent Done false
93322024-04-17T17:16:31.114Z 14 Acked 1013 Write 9 Done Sent Done false
93332024-04-17T17:16:31.114Z 15 Acked 1014 Write 4 Done Sent Done false
93342024-04-17T17:16:31.114Z 16 Acked 1015 Read 2 Done Sent Done false
93352024-04-17T17:16:31.114Z 17 Acked 1016 Read 8 Done Sent Done false
93362024-04-17T17:16:31.114Z 18 Acked 1017 Read 5 Done Sent Done false
93372024-04-17T17:16:31.114Z 19 Acked 1018 Read 10 Done Sent Done false
93382024-04-17T17:16:31.114Z 20 Acked 1019 Read 8 Done Sent Done false
93392024-04-17T17:16:31.115Z 21 Acked 1020 Read 8 Done Sent Done false
93402024-04-17T17:16:31.115Z 22 Acked 1021 Write 9 Done Sent Done false
93412024-04-17T17:16:31.115Z 23 Acked 1022 Read 9 Done Sent Done false
93422024-04-17T17:16:31.115Z 24 Acked 1023 Flush 0 Done Sent Done false
93432024-04-17T17:16:31.115Z 25 Acked 1024 Flush 0 Done Sent Done false
93442024-04-17T17:16:31.115Z 26 Acked 1025 Write 8 Done Sent Done false
93452024-04-17T17:16:31.115Z 27 Acked 1026 Read 8 Done Sent Done false
93462024-04-17T17:16:31.115Z 28 Acked 1027 Write 7 Done Sent Done false
93472024-04-17T17:16:31.115Z 29 Acked 1028 Read 9 Done Sent Done false
93482024-04-17T17:16:31.115Z 30 Acked 1029 Read 2 Done Sent Sent false
93492024-04-17T17:16:31.115Z STATES DS:0 DS:1 DS:2 TOTAL
93502024-04-17T17:16:31.115Z New 0 0 0 0
93512024-04-17T17:16:31.115Z Sent 0 28 1 29
93522024-04-17T17:16:31.115Z Done 30 2 29 61
93532024-04-17T17:16:31.115Z Skipped 0 0 0 0
93542024-04-17T17:16:31.115Z Error 0 0 0 0
93552024-04-17T17:16:31.115ZLast Flush: 1024 0 1024
93562024-04-17T17:16:31.115ZDownstairs last five completed:
93572024-04-17T17:16:31.115ZUpstairs last five completed: 30 29 28 27 26
93582024-04-17T17:16:31.119Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:16:29.721274884Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1515}
93592024-04-17T17:16:31.119ZWrote out file "/tmp/test_repair_verify.data"
93602024-04-17T17:16:31.119Z
93612024-04-17T17:16:31.123ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
93622024-04-17T17:16:31.136ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
93632024-04-17T17:16:31.136Z 0 000-019 80 80 80 141 141 141 F F F
93642024-04-17T17:16:31.136Z 1 020-039 81 81 81 142 142 142 F F F
93652024-04-17T17:16:31.136Z 2 040-059 81 81 81 142 142 142 F F F
93662024-04-17T17:16:31.136Z 3 060-079 85 85 85 150 150 150 F F F
93672024-04-17T17:16:31.136Z 4 080-099 85 85 85 150 150 150 F F F
93682024-04-17T17:16:31.136Z 5 100-119 83 83 83 146 146 146 F F F
93692024-04-17T17:16:31.140Z 6 120-139 83 83 83 146 146 146 F F F
93702024-04-17T17:16:31.140Z 7 140-159 68 68 68 124 124 124 F F F
93712024-04-17T17:16:31.140Z 8 160-179 81 81 81 142 142 142 F F F
93722024-04-17T17:16:31.140Z 9 180-199 86 83 86 151 146 151 F T F <---
93732024-04-17T17:16:31.140Z 10 200-219 84 84 84 148 148 148 F F F
93742024-04-17T17:16:31.140Z 11 220-239 86 85 86 153 150 153 F F F <---
93752024-04-17T17:16:31.140Z 12 240-259 85 85 85 150 150 150 F F F
93762024-04-17T17:16:31.140Z 13 260-279 86 85 86 152 150 152 T F T <---
93772024-04-17T17:16:31.140Z 14 280-299 86 75 86 152 135 152 F F F <---
93782024-04-17T17:16:31.140Z 15 300-319 85 85 85 150 150 150 F F F
93792024-04-17T17:16:31.140Z 16 320-339 85 85 85 150 150 150 F F F
93802024-04-17T17:16:31.140Z 17 340-359 86 83 86 153 146 153 F F F <---
93812024-04-17T17:16:31.140Z 18 360-379 81 81 81 142 142 142 F F F
93822024-04-17T17:16:31.140Z 19 380-399 83 83 83 146 146 146 F F F
93832024-04-17T17:16:31.140Z 20 400-419 83 83 83 146 146 146 F F F
93842024-04-17T17:16:31.141Z 21 420-439 73 73 73 132 132 132 F F F
93852024-04-17T17:16:31.141Z 22 440-459 82 82 82 144 144 144 F F F
93862024-04-17T17:16:31.141Z 23 460-479 82 82 82 144 144 144 F F F
93872024-04-17T17:16:31.141Z 24 480-499 86 84 86 153 149 153 F F F <---
93882024-04-17T17:16:31.141Z 25 500-519 70 70 70 127 127 127 F F F
93892024-04-17T17:16:31.141Z 26 520-539 81 81 81 142 142 142 T F T <---
93902024-04-17T17:16:31.141Z 27 540-559 81 81 81 142 142 142 T F T <---
93912024-04-17T17:16:31.141Z 28 560-579 85 85 85 150 150 150 F F F
93922024-04-17T17:16:31.141Z 29 580-599 82 82 82 145 145 145 F F F
93932024-04-17T17:16:31.141ZMax gen: 86, Max flush: 153
93942024-04-17T17:16:31.141ZError: Difference in extent metadata found!
93952024-04-17T17:16:31.141ZA Difference in extent metadata is expected here
93962024-04-17T17:16:31.141ZOn loop 43
93972024-04-17T17:16:31.141Z
93982024-04-17T17:16:31.145ZVerifying data now
93992024-04-17T17:16:35.543ZNo change after verify
94002024-04-17T17:16:35.543ZLoop: 43 Downstairs dump after verify (and repair):
94012024-04-17T17:16:35.558ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
94022024-04-17T17:16:35.558Z 0 000-019 80 80 80 141 141 141 F F F
94032024-04-17T17:16:35.562Z 1 020-039 81 81 81 142 142 142 F F F
94042024-04-17T17:16:35.562Z 2 040-059 81 81 81 142 142 142 F F F
94052024-04-17T17:16:35.562Z 3 060-079 85 85 85 150 150 150 F F F
94062024-04-17T17:16:35.562Z 4 080-099 85 85 85 150 150 150 F F F
94072024-04-17T17:16:35.562Z 5 100-119 83 83 83 146 146 146 F F F
94082024-04-17T17:16:35.562Z 6 120-139 83 83 83 146 146 146 F F F
94092024-04-17T17:16:35.562Z 7 140-159 68 68 68 124 124 124 F F F
94102024-04-17T17:16:35.562Z 8 160-179 81 81 81 142 142 142 F F F
94112024-04-17T17:16:35.562Z 9 180-199 86 86 86 151 151 151 F F F
94122024-04-17T17:16:35.562Z 10 200-219 84 84 84 148 148 148 F F F
94132024-04-17T17:16:35.563Z 11 220-239 86 86 86 153 153 153 F F F
94142024-04-17T17:16:35.563Z 12 240-259 85 85 85 150 150 150 F F F
94152024-04-17T17:16:35.563Z 13 260-279 87 87 87 154 154 154 F F F
94162024-04-17T17:16:35.563Z 14 280-299 86 86 86 152 152 152 F F F
94172024-04-17T17:16:35.563Z 15 300-319 85 85 85 150 150 150 F F F
94182024-04-17T17:16:35.563Z 16 320-339 85 85 85 150 150 150 F F F
94192024-04-17T17:16:35.563Z 17 340-359 86 86 86 153 153 153 F F F
94202024-04-17T17:16:35.563Z 18 360-379 81 81 81 142 142 142 F F F
94212024-04-17T17:16:35.563Z 19 380-399 83 83 83 146 146 146 F F F
94222024-04-17T17:16:35.563Z 20 400-419 83 83 83 146 146 146 F F F
94232024-04-17T17:16:35.563Z 21 420-439 73 73 73 132 132 132 F F F
94242024-04-17T17:16:35.563Z 22 440-459 82 82 82 144 144 144 F F F
94252024-04-17T17:16:35.563Z 23 460-479 82 82 82 144 144 144 F F F
94262024-04-17T17:16:35.563Z 24 480-499 86 86 86 153 153 153 F F F
94272024-04-17T17:16:35.563Z 25 500-519 70 70 70 127 127 127 F F F
94282024-04-17T17:16:35.563Z 26 520-539 87 87 87 154 154 154 F F F
94292024-04-17T17:16:35.563Z 27 540-559 87 87 87 154 154 154 F F F
94302024-04-17T17:16:35.563Z 28 560-579 85 85 85 150 150 150 F F F
94312024-04-17T17:16:35.563Z 29 580-599 82 82 82 145 145 145 F F F
94322024-04-17T17:16:35.563ZMax gen: 87, Max flush: 154
94332024-04-17T17:16:35.563Z
94342024-04-17T17:16:35.563ZBegin loop 44
94352024-04-17T17:16:35.563ZDownstairs to restart: 2
94362024-04-17T17:16:35.581Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.189435457Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523}
94372024-04-17T17:16:35.581Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.189708183Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523}
94382024-04-17T17:16:35.588Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30Crucible runtime is spawned
94392024-04-17T17:16:35.588Z,"time":"2024-04-17T17:16:34.189742555Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523}
94402024-04-17T17:16:35.588Z{"msg":"Crucible dfdc9fbf-6237-45e6-a846-44f1f6e7843b has session id: fd1ea0bc-41af-46ae-8c88-d17dab53bb97","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.189788558Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94412024-04-17T17:16:35.588Z{"msg":"Upstairs opts: Upstairs UUID: dfdc9fbf-6237-45e6-a846-44f1f6e7843b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.189843601Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94422024-04-17T17:16:35.588Z{"msg":"Crucible stats registered with UUID: dfdc9fbf-6237-45e6-a846-44f1f6e7843b","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.189863793Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94432024-04-17T17:16:35.589Z{"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.189880354Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523}
94442024-04-17T17:16:35.589Z{"msg":"dfdc9fbf-6237-45e6-a846-44f1f6e7843b active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.189904265Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94452024-04-17T17:16:35.589Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.189919776Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"client":"0","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94462024-04-17T17:16:35.589Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.189937407Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"client":"0","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94472024-04-17T17:16:35.589Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.18997677Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"client":"1","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94482024-04-17T17:16:35.589Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.190002761Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"client":"1","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94492024-04-17T17:16:35.589Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.190029673Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"client":"2","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94502024-04-17T17:16:35.589Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.190051604Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"client":"2","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94512024-04-17T17:16:35.589Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.190070885Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94522024-04-17T17:16:35.589Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.190115508Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94532024-04-17T17:16:35.589Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.19014209Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94542024-04-17T17:16:35.589Z{"msg":"ds_connection connected from Ok(127.0.0.1:54591)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.190284348Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94552024-04-17T17:16:35.589Z{"msg":"ds_connection connected from Ok(127.0.0.1:62728)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.190337791Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94562024-04-17T17:16:35.589Z{"msg":"ds_connection connected from Ok(127.0.0.1:64756)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.190367763Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94572024-04-17T17:16:35.589Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.191094217Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"client":"0","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94582024-04-17T17:16:35.589Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.19113098Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"client":"0","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94592024-04-17T17:16:35.589Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.191147171Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"client":"1","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94602024-04-17T17:16:35.589Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.191161741Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"client":"1","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94612024-04-17T17:16:35.589Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.191332192Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"client":"2","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94622024-04-17T17:16:35.589Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.191374684Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"client":"2","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94632024-04-17T17:16:35.589Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.191693284Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"client":"1","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94642024-04-17T17:16:35.589Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.191810651Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"client":"0","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94652024-04-17T17:16:35.590Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.192071477Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"client":"2","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94662024-04-17T17:16:35.590Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.192240767Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"client":"1","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94672024-04-17T17:16:35.590Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.192326392Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"client":"1","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94682024-04-17T17:16:35.590Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.192352974Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94692024-04-17T17:16:35.590Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.192366965Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"client":"0","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94702024-04-17T17:16:35.590Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.192380615Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"client":"0","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94712024-04-17T17:16:35.590Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.192403717Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94722024-04-17T17:16:35.590Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.192599659Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"client":"2","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94732024-04-17T17:16:35.590Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.192638881Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"client":"2","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94742024-04-17T17:16:35.590Z{"msg":"[0]R flush_numbers[0..12]: [141, 142, 142, 150, 150, 146, 146, 124, 142, 151, 148, 153]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.192655722Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"":"downstairs","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94752024-04-17T17:16:35.590Z{"msg":"[0]R generation[0..12]: [80, 81, 81, 85, 85, 83, 83, 68, 81, 86, 84, 86]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.192676673Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"":"downstairs","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94762024-04-17T17:16:35.590Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.192690994Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"":"downstairs","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94772024-04-17T17:16:35.590Z{"msg":"[1]R flush_numbers[0..12]: [141, 142, 142, 150, 150, 146, 146, 124, 142, 151, 148, 153]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.192711746Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"":"downstairs","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94782024-04-17T17:16:35.590Z{"msg":"[1]R generation[0..12]: [80, 81, 81, 85, 85, 83, 83, 68, 81, 86, 84, 86]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.192725836Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"":"downstairs","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94792024-04-17T17:16:35.590Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.192740667Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"":"downstairs","Wait for a query_work_queue command to finish before sending IO
94802024-04-17T17:16:35.590Z":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94812024-04-17T17:16:35.590Z{"msg":"[2]R flush_numbers[0..12]: [141, 142, 142, 150, 150, 146, 146, 124, 142, 151, 148, 153]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.192766859Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"":"downstairs","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94822024-04-17T17:16:35.590Z{"msg":"[2]R generation[0..12]: [80, 81, 81, 85, 85, 83, 83, 68, 81, 86, 84, 86]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.192780319Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"":"downstairs","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94832024-04-17T17:16:35.590Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.192801801Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"":"downstairs","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94842024-04-17T17:16:35.590Z{"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.192815412Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"":"downstairs","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94852024-04-17T17:16:35.590Z{"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.192845714Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"":"downstairs","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94862024-04-17T17:16:35.590Z{"msg":"Next flush: 155","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.192861444Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"":"downstairs","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94872024-04-17T17:16:35.590Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.192885506Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"":"downstairs","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94882024-04-17T17:16:35.590Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.192899337Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94892024-04-17T17:16:35.591Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.192911778Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"client":"0","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94902024-04-17T17:16:35.591Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.192937299Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"client":"1","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94912024-04-17T17:16:35.591Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.19295206Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"client":"2","":"downstairs","session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94922024-04-17T17:16:35.591Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.192981282Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94932024-04-17T17:16:35.591Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.193017914Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94942024-04-17T17:16:35.591Z{"msg":"dfdc9fbf-6237-45e6-a846-44f1f6e7843b is now active with session: fd1ea0bc-41af-46ae-8c88-d17dab53bb97","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.193033075Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94952024-04-17T17:16:35.591Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.193057816Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523,"session_id":"fd1ea0bc-41af-46ae-8c88-d17dab53bb97"}
94962024-04-17T17:16:35.591Z{"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:34.193075937Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523}
94972024-04-17T17:16:35.591ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
94982024-04-17T17:16:35.595ZLoading write count information from file "/tmp/test_repair_verify.data"
94992024-04-17T17:16:35.595ZRun Repair workload
95002024-04-17T17:16:35.595Z01/30 Write block 173 len 24576 data: 6 4 7 6 5 8
95012024-04-17T17:16:35.595Z02/30 Read block 42 len 4096
95022024-04-17T17:16:35.595Z03/30 Flush
95032024-04-17T17:16:35.595Z04/30 Write block 522 len 24576 data: 7 7 4 5 5 6
95042024-04-17T17:16:35.595Z05/30 Read block 375 len 4096
95052024-04-17T17:16:35.595Z06/30 Read block 109 len 8192
95062024-04-17T17:16:35.595Z07/30 Read block 237 len 36864
95072024-04-17T17:16:35.595Z08/30 Flush
95082024-04-17T17:16:35.614Z09/30 Write block 391 len 4096 data: 6
95092024-04-17T17:16:35.614Z10/30 Read block 512 len 36864
95102024-04-17T17:16:35.614Z11/30 Flush
95112024-04-17T17:16:35.614Z12/30 Write block 60 len 40960 data: 10 9 9 10 11 9 8 8 7 5
95122024-04-17T17:16:35.614Z13/30 Read block 562 len 36864
95132024-04-17T17:16:35.614Z14/30 Flush
95142024-04-17T17:16:35.614Z15/30 Write block 417 len 20480 data: 10 9 10 9 5
95152024-04-17T17:16:35.614Z16/30 Write block 34 len 40960 data: 15 12 9 7 6 7 8 11 9 8
95162024-04-17T17:16:35.615Z17/30 Write block 72 len 4096 data: 6
95172024-04-17T17:16:35.615Z18/30 Read block 521 len 24576
95182024-04-17T17:16:35.615Z19/30 Read block 365 len 20480
95192024-04-17T17:16:35.615Z20/30 Read block 216 len 4096
95202024-04-17T17:16:35.615Z21/30 Write block 34 len 40960 data: 16 13 10 8 7 8 9 12 10 9
95212024-04-17T17:16:35.615Z22/30 Read block 87 len 20480
95222024-04-17T17:16:35.615Z23/30 Read block 398 len 28672
95232024-04-17T17:16:35.615Z24/30 Write block 176 len 16384 data: 7 6 9 8
95242024-04-17T17:16:35.615Z25/30 Flush
95252024-04-17T17:16:35.620Z26/30 Write block 236 len 32768 data: 8 6 5 7 7 10 8 10
95262024-04-17T17:16:35.620Z27/30 Read block 262 len 16384
95272024-04-17T17:16:35.620Z28/30 Read block 154 len 24576
95282024-04-17T17:16:35.620Z29/30 Read block 587 len 24576
95292024-04-17T17:16:35.620Z30/30 Write block 19 len 36864 data: 5 6 9 10 9 9 9 9 5
95302024-04-17T17:16:35.620Z----------------------------------------------------------------
95312024-04-17T17:16:35.620Z Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30
95322024-04-17T17:16:35.620ZStates: Active Active Active
95332024-04-17T17:16:35.620ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
95342024-04-17T17:16:35.620Z 1 Acked 1000 Write 6 Done Done Done false
95352024-04-17T17:16:35.620Z 2 Acked 1001 Read 1 Done Done Done false
95362024-04-17T17:16:35.620Z 3 Acked 1002 Flush 0 Done Done Sent false
95372024-04-17T17:16:35.620Z 4 Acked 1003 Write 6 Done Done Sent false
95382024-04-17T17:16:35.620Z 5 Acked 1004 Read 1 Done Done Sent false
95392024-04-17T17:16:35.620Z 6 Acked 1005 Read 2 Done Done Sent false
95402024-04-17T17:16:35.620Z 7 Acked 1006 Read 9 Done Done Sent false
95412024-04-17T17:16:35.621Z 8 Acked 1007 Flush 0 Done Done Sent false
95422024-04-17T17:16:35.621Z 9 Acked 1008 Write 1 Done Done Sent false
95432024-04-17T17:16:35.621Z 10 Acked 1009 Read 9 Done Done Sent false
95442024-04-17T17:16:35.621Z 11 Acked 1010 Flush 0 Done Done Sent false
95452024-04-17T17:16:35.621Z 12 Acked 1011 Write 10 Done Done Sent false
95462024-04-17T17:16:35.621Z 13 Acked 1012 Read 9 Done Done Sent false
95472024-04-17T17:16:35.621Z 14 Acked 1013 Flush 0 Done Done Sent false
95482024-04-17T17:16:35.621Z 15 Acked 1014 Write 5 Done Done Sent false
95492024-04-17T17:16:35.621Z 16 Acked 1015 Write 10 Done Done Sent false
95502024-04-17T17:16:35.621Z 17 Acked 1016 Write 1 Done Done Sent false
95512024-04-17T17:16:35.621Z 18 Acked 1017 Read 6 Done Done Sent false
95522024-04-17T17:16:35.621Z 19 Acked 1018 Read 5 Done Done Sent false
95532024-04-17T17:16:35.621Z 20 Acked 1019 Read 1 Done Done Sent false
95542024-04-17T17:16:35.621Z 21 Acked 1020 Write 10 Done Done Sent false
95552024-04-17T17:16:35.621Z 22 Acked 1021 Read 5 Done Done Sent false
95562024-04-17T17:16:35.621Z 23 Acked 1022 Read 7 Done Done Sent false
95572024-04-17T17:16:35.621Z 24 Acked 1023 Write 4 Done Done Sent false
95582024-04-17T17:16:35.621Z 25 Acked 1024 Flush 0 Done Done Sent false
95592024-04-17T17:16:35.621Z 26 Acked 1025 Write 8 Done Done Sent false
95602024-04-17T17:16:35.621Z 27 Acked 1026 Read 4 Done Done Sent false
95612024-04-17T17:16:35.621Z 28 Acked 1027 Read 6 Done Done Sent false
95622024-04-17T17:16:35.621Z 29 Acked 1028 Read 6 Done Done Sent false
95632024-04-17T17:16:35.621Z 30 Acked 1029 Write 9 Sent Sent Sent false
95642024-04-17T17:16:35.622Z STATES DS:0 DS:1 DS:2 TOTAL
95652024-04-17T17:16:35.622Z New 0 0 0 0
95662024-04-17T17:16:35.622Z Sent 1 1 28 30
95672024-04-17T17:16:35.622Z Done 29 29 2 60
95682024-04-17T17:16:35.622Z Skipped 0 0 0 0
95692024-04-17T17:16:35.622Z Error 0 0 0 0
95702024-04-17T17:16:35.622ZLast Flush: 1024 1024 0
95712024-04-17T17:16:35.622ZDownstairs last five completed:
95722024-04-17T17:16:35.622ZUpstairs last five completed: 30 29 28 27 26
95732024-04-17T17:16:35.626Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:16:34.228026826Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1523}
95742024-04-17T17:16:35.626ZWrote out file "/tmp/test_repair_verify.data"
95752024-04-17T17:16:35.626Z
95762024-04-17T17:16:35.629ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
95772024-04-17T17:16:35.643ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
95782024-04-17T17:16:35.643Z 0 000-019 80 80 80 141 141 141 T T F <---
95792024-04-17T17:16:35.647Z 1 020-039 88 88 81 159 159 142 T T F <---
95802024-04-17T17:16:35.647Z 2 040-059 88 88 81 159 159 142 F F F <---
95812024-04-17T17:16:35.647Z 3 060-079 88 88 85 159 159 150 F F F <---
95822024-04-17T17:16:35.647Z 4 080-099 85 85 85 150 150 150 F F F
95832024-04-17T17:16:35.647Z 5 100-119 83 83 83 146 146 146 F F F
95842024-04-17T17:16:35.647Z 6 120-139 83 83 83 146 146 146 F F F
95852024-04-17T17:16:35.647Z 7 140-159 68 68 68 124 124 124 F F F
95862024-04-17T17:16:35.647Z 8 160-179 88 88 81 159 159 142 F F T <---
95872024-04-17T17:16:35.647Z 9 180-199 86 86 86 151 151 151 F F F
95882024-04-17T17:16:35.647Z 10 200-219 84 84 84 148 148 148 F F F
95892024-04-17T17:16:35.647Z 11 220-239 86 86 86 153 153 153 T T F <---
95902024-04-17T17:16:35.647Z 12 240-259 85 85 85 150 150 150 T T F <---
95912024-04-17T17:16:35.647Z 13 260-279 87 87 87 154 154 154 F F F
95922024-04-17T17:16:35.647Z 14 280-299 86 86 86 152 152 152 F F F
95932024-04-17T17:16:35.647Z 15 300-319 85 85 85 150 150 150 F F F
95942024-04-17T17:16:35.647Z 16 320-339 85 85 85 150 150 150 F F F
95952024-04-17T17:16:35.647Z 17 340-359 86 86 86 153 153 153 F F F
95962024-04-17T17:16:35.647Z 18 360-379 81 81 81 142 142 142 F F F
95972024-04-17T17:16:35.647Z 19 380-399 88 88 83 157 157 146 F F F <---
95982024-04-17T17:16:35.648Z 20 400-419 88 88 83 159 159 146 F F F <---
95992024-04-17T17:16:35.648Z 21 420-439 88 88 73 159 159 132 F F F <---
96002024-04-17T17:16:35.648Z 22 440-459 82 82 82 144 144 144 F F F
96012024-04-17T17:16:35.648Z 23 460-479 82 82 82 144 144 144 F F F
96022024-04-17T17:16:35.648Z 24 480-499 86 86 86 153 153 153 F F F
96032024-04-17T17:16:35.648Z 25 500-519 70 70 70 127 127 127 F F F
96042024-04-17T17:16:35.648Z 26 520-539 88 88 87 156 156 154 F F F <---
96052024-04-17T17:16:35.648Z 27 540-559 87 87 87 154 154 154 F F F
96062024-04-17T17:16:35.648Z 28 560-579 85 85 85 150 150 150 F F F
96072024-04-17T17:16:35.648Z 29 580-599 82 82 82 145 145 145 F F F
96082024-04-17T17:16:35.648ZMax gen: 88, Max flush: 159
96092024-04-17T17:16:35.648ZError: Difference in extent metadata found!
96102024-04-17T17:16:35.648ZA Difference in extent metadata is expected here
96112024-04-17T17:16:35.648ZOn loop 44
96122024-04-17T17:16:35.648Z
96132024-04-17T17:16:35.651ZVerifying data now
96142024-04-17T17:16:40.174ZNo change after verify
96152024-04-17T17:16:40.175ZLoop: 44 Downstairs dump after verify (and repair):
96162024-04-17T17:16:40.189ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
96172024-04-17T17:16:40.189Z 0 000-019 89 89 89 160 160 160 F F F
96182024-04-17T17:16:40.193Z 1 020-039 89 89 89 160 160 160 F F F
96192024-04-17T17:16:40.193Z 2 040-059 88 88 88 159 159 159 F F F
96202024-04-17T17:16:40.193Z 3 060-079 88 88 88 159 159 159 F F F
96212024-04-17T17:16:40.193Z 4 080-099 85 85 85 150 150 150 F F F
96222024-04-17T17:16:40.193Z 5 100-119 83 83 83 146 146 146 F F F
96232024-04-17T17:16:40.194Z 6 120-139 83 83 83 146 146 146 F F F
96242024-04-17T17:16:40.194Z 7 140-159 68 68 68 124 124 124 F F F
96252024-04-17T17:16:40.194Z 8 160-179 88 88 88 159 159 159 F F F
96262024-04-17T17:16:40.194Z 9 180-199 86 86 86 151 151 151 F F F
96272024-04-17T17:16:40.194Z 10 200-219 84 84 84 148 148 148 F F F
96282024-04-17T17:16:40.194Z 11 220-239 89 89 89 160 160 160 F F F
96292024-04-17T17:16:40.194Z 12 240-259 89 89 89 160 160 160 F F F
96302024-04-17T17:16:40.194Z 13 260-279 87 87 87 154 154 154 F F F
96312024-04-17T17:16:40.194Z 14 280-299 86 86 86 152 152 152 F F F
96322024-04-17T17:16:40.194Z 15 300-319 85 85 85 150 150 150 F F F
96332024-04-17T17:16:40.194Z 16 320-339 85 85 85 150 150 150 F F F
96342024-04-17T17:16:40.194Z 17 340-359 86 86 86 153 153 153 F F F
96352024-04-17T17:16:40.194Z 18 360-379 81 81 81 142 142 142 F F F
96362024-04-17T17:16:40.194Z 19 380-399 88 88 88 157 157 157 F F F
96372024-04-17T17:16:40.194Z 20 400-419 88 88 88 159 159 159 F F F
96382024-04-17T17:16:40.194Z 21 420-439 88 88 88 159 159 159 F F F
96392024-04-17T17:16:40.194Z 22 440-459 82 82 82 144 144 144 F F F
96402024-04-17T17:16:40.194Z 23 460-479 82 82 82 144 144 144 F F F
96412024-04-17T17:16:40.194Z 24 480-499 86 86 86 153 153 153 F F F
96422024-04-17T17:16:40.194Z 25 500-519 70 70 70 127 127 127 F F F
96432024-04-17T17:16:40.194Z 26 520-539 88 88 88 156 156 156 F F F
96442024-04-17T17:16:40.194Z 27 540-559 87 87 87 154 154 154 F F F
96452024-04-17T17:16:40.195Z 28 560-579 85 85 85 150 150 150 F F F
96462024-04-17T17:16:40.195Z 29 580-599 82 82 82 145 145 145 F F F
96472024-04-17T17:16:40.195ZMax gen: 89, Max flush: 160
96482024-04-17T17:16:40.195Z
96492024-04-17T17:16:40.195ZBegin loop 45
96502024-04-17T17:16:40.195ZDownstairs to restart: 1
96512024-04-17T17:16:40.213Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.819791571Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531}
96522024-04-17T17:16:40.223Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.820094349Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531}
96532024-04-17T17:16:40.223ZCrucible runtime is spawned
96542024-04-17T17:16:40.223Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.820178064Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531}
96552024-04-17T17:16:40.223Z{"msg":"Crucible b6d3c2be-289f-4103-bc36-f673b2796ec3 has session id: 2c4e0693-ee76-431e-a8d5-0040e2880e83","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.820209866Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96562024-04-17T17:16:40.223Z{"msg":"Upstairs opts: Upstairs UUID: b6d3c2be-289f-4103-bc36-f673b2796ec3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.82027268Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96572024-04-17T17:16:40.223Z{"msg":"Crucible stats registered with UUID: b6d3c2be-289f-4103-bc36-f673b2796ec3","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.820337313Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96582024-04-17T17:16:40.223Z{"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.820373486Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531}
96592024-04-17T17:16:40.223Z{"msg":"b6d3c2be-289f-4103-bc36-f673b2796ec3 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.820428599Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96602024-04-17T17:16:40.223Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.820463021Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"client":"0","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96612024-04-17T17:16:40.223Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.820492573Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"client":"0","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96622024-04-17T17:16:40.223Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.820517655Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"client":"1","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96632024-04-17T17:16:40.223Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.820540856Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"client":"1","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96642024-04-17T17:16:40.223Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.820583528Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"client":"2","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96652024-04-17T17:16:40.223Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.82061126Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"client":"2","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96662024-04-17T17:16:40.223Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.820651013Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"":"io task","client":"2","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96672024-04-17T17:16:40.223Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.820680994Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"":"io task","client":"0","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96682024-04-17T17:16:40.224Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.820713436Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"":"io task","client":"1","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96692024-04-17T17:16:40.224Z{"msg":"ds_connection connected from Ok(127.0.0.1:38147)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.820728527Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"":"io task","client":"2","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96702024-04-17T17:16:40.224Z{"msg":"ds_connection connected from Ok(127.0.0.1:52454)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.820898968Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"":"io task","client":"1","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96712024-04-17T17:16:40.224Z{"msg":"ds_connection connected from Ok(127.0.0.1:45224)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.820951381Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"":"io task","client":"0","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96722024-04-17T17:16:40.224Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.821442Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"client":"2","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96732024-04-17T17:16:40.224Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.821488983Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"client":"2","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96742024-04-17T17:16:40.224Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.821540326Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"client":"1","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96752024-04-17T17:16:40.224Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.821585249Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"client":"1","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96762024-04-17T17:16:40.224Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.821628142Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"client":"0","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96772024-04-17T17:16:40.224Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.821645323Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"client":"0","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96782024-04-17T17:16:40.224Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.822042757Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"client":"0","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96792024-04-17T17:16:40.224Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.822077539Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"client":"2","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96802024-04-17T17:16:40.224Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.822156684Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"client":"1","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96812024-04-17T17:16:40.224Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.822688636Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"client":"2","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96822024-04-17T17:16:40.224Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.822740059Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"client":"2","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96832024-04-17T17:16:40.224Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.8227625Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96842024-04-17T17:16:40.224Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.822793302Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"client":"1","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96852024-04-17T17:16:40.224Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.822812343Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"client":"1","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96862024-04-17T17:16:40.224Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.822830565Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96872024-04-17T17:16:40.224Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.822857996Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"client":"0","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96882024-04-17T17:16:40.224Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.822873287Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"client":"0","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96892024-04-17T17:16:40.224Z{"msg":"[0]R flush_numbers[0..12]: [160, 160, 159, 159, 150, 146, 146, 124, 159, 151, 148, 160]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.822905119Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"":"downstairs","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96902024-04-17T17:16:40.224Z{"msg":"[0]R generation[0..12]: [89, 89, 88, 88, 85, 83, 83, 68, 88, 86, 84, 89]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.82292421Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"":"downstairs","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96912024-04-17T17:16:40.224Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.822954462Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"":"downstairs","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96922024-04-17T17:16:40.225Z{"msg":"[1]R flush_numbers[0..12]: [160, 160, 159, 159, 150, 146, 146, 124, 159, 151, 148, 160]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.822970083Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"":"downstairs","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96932024-04-17T17:16:40.225Z{"msg":"[1]R generation[0..12]: [89, 89, 88, 88, 85, 83, 83, 68, 88, 86, 84, 89]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.823012636Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"":"downstairs","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96942024-04-17T17:16:40.225Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.823029577Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"":"downstairs","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96952024-04-17T17:16:40.225Z{"msg":"[2]R flush_numbers[0..12]: [160, 160, 159, 159, 150, 146, 146, 124, 159, 151, 148, 160]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.823043268Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"":"downstairs","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96962024-04-17T17:16:40.225Z{"msg":"[2]R generation[0..12]: [89, 89, 88, 88, 85, 83, 83, 68, 88, 86, 84, 89]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.823059829Z","hostnameWait for a query_work_queue command to finish before sending IO
96972024-04-17T17:16:40.225Z":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"":"downstairs","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96982024-04-17T17:16:40.225Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.823090771Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"":"downstairs","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
96992024-04-17T17:16:40.225Z{"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.823106401Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"":"downstairs","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
97002024-04-17T17:16:40.225Z{"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.823136193Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"":"downstairs","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
97012024-04-17T17:16:40.225Z{"msg":"Next flush: 161","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.823150664Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"":"downstairs","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
97022024-04-17T17:16:40.225Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.823169975Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"":"downstairs","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
97032024-04-17T17:16:40.225Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.823197207Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
97042024-04-17T17:16:40.225Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.823210048Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"client":"0","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
97052024-04-17T17:16:40.225Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.823227159Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"client":"1","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
97062024-04-17T17:16:40.225Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.823255901Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"client":"2","":"downstairs","session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
97072024-04-17T17:16:40.225Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.823270281Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
97082024-04-17T17:16:40.225Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.823287272Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
97092024-04-17T17:16:40.225Z{"msg":"b6d3c2be-289f-4103-bc36-f673b2796ec3 is now active with session: 2c4e0693-ee76-431e-a8d5-0040e2880e83","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.823315354Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
97102024-04-17T17:16:40.225Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.823329325Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531,"session_id":"2c4e0693-ee76-431e-a8d5-0040e2880e83"}
97112024-04-17T17:16:40.225Z{"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:38.823347076Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531}
97122024-04-17T17:16:40.225ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
97132024-04-17T17:16:40.225ZLoading write count information from file "/tmp/test_repair_verify.data"
97142024-04-17T17:16:40.225ZRun Repair workload
97152024-04-17T17:16:40.225Z01/30 Write block 492 len 4096 data: 6
97162024-04-17T17:16:40.230Z02/30 Read block 283 len 24576
97172024-04-17T17:16:40.230Z03/30 Read block 58 len 20480
97182024-04-17T17:16:40.230Z04/30 Read block 300 len 4096
97192024-04-17T17:16:40.230Z05/30 Read block 398 len 12288
97202024-04-17T17:16:40.230Z06/30 Read block 358 len 12288
97212024-04-17T17:16:40.230Z07/30 Read block 246 len 12288
97222024-04-17T17:16:40.230Z08/30 Write block 253 len 40960 data: 8 7 5 9 9 8 9 8 9 8
97232024-04-17T17:16:40.230Z09/30 Read block 327 len 32768
97242024-04-17T17:16:40.230Z10/30 Write block 269 len 24576 data: 8 8 9 9 11 11
97252024-04-17T17:16:40.230Z11/30 Read block 162 len 36864
97262024-04-17T17:16:40.230Z12/30 Write block 526 len 28672 data: 6 7 7 8 8 8 6
97272024-04-17T17:16:40.230Z13/30 Write block 436 len 16384 data: 8 8 9 10
97282024-04-17T17:16:40.230Z14/30 Write block 230 len 12288 data: 10 8 9
97292024-04-17T17:16:40.230Z15/30 Read block 470 len 20480
97302024-04-17T17:16:40.230Z16/30 Read block 396 len 32768
97312024-04-17T17:16:40.234Z17/30 Write block 3 len 32768 data: 6 7 7 6 5 6 7 7
97322024-04-17T17:16:40.234Z18/30 Read block 461 len 12288
97332024-04-17T17:16:40.234Z19/30 Read block 2 len 8192
97342024-04-17T17:16:40.234Z20/30 Write block 154 len 32768 data: 6 7 9 10 8 8 7 8
97352024-04-17T17:16:40.234Z21/30 Read block 261 len 12288
97362024-04-17T17:16:40.239Z22/30 Read block 499 len 32768
97372024-04-17T17:16:40.239Z23/30 Read block 206 len 20480
97382024-04-17T17:16:40.239Z24/30 Flush
97392024-04-17T17:16:40.244Z25/30 Write block 473 len 32768 data: 6 6 4 5 9 11 11 13
97402024-04-17T17:16:40.244Z26/30 Read block 245 len 24576
97412024-04-17T17:16:40.244Z27/30 Read block 417 len 28672
97422024-04-17T17:16:40.244Z28/30 Write block 297 len 16384 data: 3 3 4 4
97432024-04-17T17:16:40.249Z29/30 Read block 146 len 28672
97442024-04-17T17:16:40.249Z30/30 Read block 304 len 20480
97452024-04-17T17:16:40.249Z----------------------------------------------------------------
97462024-04-17T17:16:40.249Z Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:30
97472024-04-17T17:16:40.250ZStates: Active Active Active
97482024-04-17T17:16:40.250ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
97492024-04-17T17:16:40.250Z 1 Acked 1000 Write 1 Done Done Done false
97502024-04-17T17:16:40.250Z 2 Acked 1001 Read 6 Done Done Done false
97512024-04-17T17:16:40.250Z 3 Acked 1002 Read 5 Done Done Done false
97522024-04-17T17:16:40.250Z 4 Acked 1003 Read 1 Done Done Done false
97532024-04-17T17:16:40.250Z 5 Acked 1004 Read 3 Done Done Done false
97542024-04-17T17:16:40.250Z 6 Acked 1005 Read 3 Done Done Done false
97552024-04-17T17:16:40.250Z 7 Acked 1006 Read 3 Done Done Done false
97562024-04-17T17:16:40.250Z 8 Acked 1007 Write 10 Done Done Done false
97572024-04-17T17:16:40.250Z 9 Acked 1008 Read 8 Done Sent Done false
97582024-04-17T17:16:40.250Z 10 Acked 1009 Write 6 Done Sent Done false
97592024-04-17T17:16:40.250Z 11 Acked 1010 Read 9 Done Sent Done false
97602024-04-17T17:16:40.250Z 12 Acked 1011 Write 7 Done Sent Done false
97612024-04-17T17:16:40.250Z 13 Acked 1012 Write 4 Done Sent Done false
97622024-04-17T17:16:40.250Z 14 Acked 1013 Write 3 Done Sent Done false
97632024-04-17T17:16:40.250Z 15 Acked 1014 Read 5 Done Sent Done false
97642024-04-17T17:16:40.250Z 16 Acked 1015 Read 8 Done Sent Done false
97652024-04-17T17:16:40.250Z 17 Acked 1016 Write 8 Done Sent Done false
97662024-04-17T17:16:40.250Z 18 Acked 1017 Read 3 Done Sent Done false
97672024-04-17T17:16:40.250Z 19 Acked 1018 Read 2 Done Sent Done false
97682024-04-17T17:16:40.250Z 20 Acked 1019 Write 8 Done Sent Done false
97692024-04-17T17:16:40.250Z 21 Acked 1020 Read 3 Done Sent Done false
97702024-04-17T17:16:40.251Z 22 Acked 1021 Read 8 Done Sent Done false
97712024-04-17T17:16:40.251Z 23 Acked 1022 Read 5 Done Sent Done false
97722024-04-17T17:16:40.251Z 24 Acked 1023 Flush 0 Done Sent Done false
97732024-04-17T17:16:40.251Z 25 Acked 1024 Write 8 Done Sent Done false
97742024-04-17T17:16:40.251Z 26 Acked 1025 Read 6 Done Sent Done false
97752024-04-17T17:16:40.251Z 27 Acked 1026 Read 7 Done Sent Done false
97762024-04-17T17:16:40.251Z 28 Acked 1027 Write 4 Done Sent Done false
97772024-04-17T17:16:40.251Z 29 Acked 1028 Read 7 Done Sent Done false
97782024-04-17T17:16:40.251Z 30 Acked 1029 Read 5 Sent Sent Done false
97792024-04-17T17:16:40.251Z STATES DS:0 DS:1 DS:2 TOTAL
97802024-04-17T17:16:40.251Z New 0 0 0 0
97812024-04-17T17:16:40.251Z Sent 1 22 0 23
97822024-04-17T17:16:40.251Z Done 29 8 30 67
97832024-04-17T17:16:40.251Z Skipped 0 0 0 0
97842024-04-17T17:16:40.251Z Error 0 0 0 0
97852024-04-17T17:16:40.251ZLast Flush: 1023 0 1023
97862024-04-17T17:16:40.251ZDownstairs last five completed:
97872024-04-17T17:16:40.251ZUpstairs last five completed: 30 29 28 27 26
97882024-04-17T17:16:40.251Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:16:38.85327642Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1531}
97892024-04-17T17:16:40.251ZWrote out file "/tmp/test_repair_verify.data"
97902024-04-17T17:16:40.255Z
97912024-04-17T17:16:40.255ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
97922024-04-17T17:16:40.269ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
97932024-04-17T17:16:40.269Z 0 000-019 90 89 90 161 160 161 F F F <---
97942024-04-17T17:16:40.273Z 1 020-039 89 89 89 160 160 160 F F F
97952024-04-17T17:16:40.273Z 2 040-059 88 88 88 159 159 159 F F F
97962024-04-17T17:16:40.273Z 3 060-079 88 88 88 159 159 159 F F F
97972024-04-17T17:16:40.273Z 4 080-099 85 85 85 150 150 150 F F F
97982024-04-17T17:16:40.273Z 5 100-119 83 83 83 146 146 146 F F F
97992024-04-17T17:16:40.273Z 6 120-139 83 83 83 146 146 146 F F F
98002024-04-17T17:16:40.273Z 7 140-159 90 68 90 161 124 161 F F F <---
98012024-04-17T17:16:40.273Z 8 160-179 90 88 90 161 159 161 F F F <---
98022024-04-17T17:16:40.273Z 9 180-199 86 86 86 151 151 151 F F F
98032024-04-17T17:16:40.273Z 10 200-219 84 84 84 148 148 148 F F F
98042024-04-17T17:16:40.273Z 11 220-239 90 89 90 161 160 161 F F F <---
98052024-04-17T17:16:40.273Z 12 240-259 90 89 90 161 160 161 F T F <---
98062024-04-17T17:16:40.273Z 13 260-279 90 87 90 161 154 161 F T F <---
98072024-04-17T17:16:40.273Z 14 280-299 86 86 86 152 152 152 T F T <---
98082024-04-17T17:16:40.273Z 15 300-319 85 85 85 150 150 150 T F T <---
98092024-04-17T17:16:40.273Z 16 320-339 85 85 85 150 150 150 F F F
98102024-04-17T17:16:40.273Z 17 340-359 86 86 86 153 153 153 F F F
98112024-04-17T17:16:40.273Z 18 360-379 81 81 81 142 142 142 F F F
98122024-04-17T17:16:40.273Z 19 380-399 88 88 88 157 157 157 F F F
98132024-04-17T17:16:40.274Z 20 400-419 88 88 88 159 159 159 F F F
98142024-04-17T17:16:40.274Z 21 420-439 90 88 90 161 159 161 F F F <---
98152024-04-17T17:16:40.274Z 22 440-459 82 82 82 144 144 144 F F F
98162024-04-17T17:16:40.274Z 23 460-479 82 82 82 144 144 144 T F T <---
98172024-04-17T17:16:40.274Z 24 480-499 90 86 90 161 153 161 T T T <---
98182024-04-17T17:16:40.274Z 25 500-519 70 70 70 127 127 127 F F F
98192024-04-17T17:16:40.274Z 26 520-539 90 88 90 161 156 161 F F F <---
98202024-04-17T17:16:40.274Z 27 540-559 87 87 87 154 154 154 F F F
98212024-04-17T17:16:40.274Z 28 560-579 85 85 85 150 150 150 F F F
98222024-04-17T17:16:40.274Z 29 580-599 82 82 82 145 145 145 F F F
98232024-04-17T17:16:40.274ZMax gen: 90, Max flush: 161
98242024-04-17T17:16:40.274ZError: Difference in extent metadata found!
98252024-04-17T17:16:40.274ZA Difference in extent metadata is expected here
98262024-04-17T17:16:40.274ZOn loop 45
98272024-04-17T17:16:40.274Z
98282024-04-17T17:16:40.278ZVerifying data now
98292024-04-17T17:16:50.829ZNo change after verify
98302024-04-17T17:16:50.829ZLoop: 45 Downstairs dump after verify (and repair):
98312024-04-17T17:16:50.844ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
98322024-04-17T17:16:50.844Z 0 000-019 90 90 90 161 161 161 F F F
98332024-04-17T17:16:50.844Z 1 020-039 89 89 89 160 160 160 F F F
98342024-04-17T17:16:50.844Z 2 040-059 88 88 88 159 159 159 F F F
98352024-04-17T17:16:50.844Z 3 060-079 88 88 88 159 159 159 F F F
98362024-04-17T17:16:50.844Z 4 080-099 85 85 85 150 150 150 F F F
98372024-04-17T17:16:50.844Z 5 100-119 83 83 83 146 146 146 F F F
98382024-04-17T17:16:50.844Z 6 120-139 83 83 83 146 146 146 F F F
98392024-04-17T17:16:50.844Z 7 140-159 90 90 90 161 161 161 F F F
98402024-04-17T17:16:50.844Z 8 160-179 90 90 90 161 161 161 F F F
98412024-04-17T17:16:50.848Z 9 180-199 86 86 86 151 151 151 F F F
98422024-04-17T17:16:50.848Z 10 200-219 84 84 84 148 148 148 F F F
98432024-04-17T17:16:50.848Z 11 220-239 90 90 90 161 161 161 F F F
98442024-04-17T17:16:50.848Z 12 240-259 90 90 90 161 161 161 F F F
98452024-04-17T17:16:50.848Z 13 260-279 90 90 90 161 161 161 F F F
98462024-04-17T17:16:50.848Z 14 280-299 91 91 91 162 162 162 F F F
98472024-04-17T17:16:50.848Z 15 300-319 91 91 91 162 162 162 F F F
98482024-04-17T17:16:50.848Z 16 320-339 85 85 85 150 150 150 F F F
98492024-04-17T17:16:50.848Z 17 340-359 86 86 86 153 153 153 F F F
98502024-04-17T17:16:50.848Z 18 360-379 81 81 81 142 142 142 F F F
98512024-04-17T17:16:50.848Z 19 380-399 88 88 88 157 157 157 F F F
98522024-04-17T17:16:50.848Z 20 400-419 88 88 88 159 159 159 F F F
98532024-04-17T17:16:50.849Z 21 420-439 90 90 90 161 161 161 F F F
98542024-04-17T17:16:50.849Z 22 440-459 82 82 82 144 144 144 F F F
98552024-04-17T17:16:50.849Z 23 460-479 91 91 91 162 162 162 F F F
98562024-04-17T17:16:50.849Z 24 480-499 91 91 91 162 162 162 F F F
98572024-04-17T17:16:50.849Z 25 500-519 70 70 70 127 127 127 F F F
98582024-04-17T17:16:50.849Z 26 520-539 90 90 90 161 161 161 F F F
98592024-04-17T17:16:50.849Z 27 540-559 87 87 87 154 154 154 F F F
98602024-04-17T17:16:50.849Z 28 560-579 85 85 85 150 150 150 F F F
98612024-04-17T17:16:50.849Z 29 580-599 82 82 82 145 145 145 F F F
98622024-04-17T17:16:50.849ZMax gen: 91, Max flush: 162
98632024-04-17T17:16:50.849Z
98642024-04-17T17:16:50.849ZBegin loop 46
98652024-04-17T17:16:50.849ZDownstairs to restart: 1
98662024-04-17T17:16:50.868Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.475941777Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539}
98672024-04-17T17:16:50.868Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.476220784Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539}
98682024-04-17T17:16:50.868Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.476240275Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539}
98692024-04-17T17:16:50.875Z{"msg":"Crucible 59ed0faf-f47b-4c1a-b2ef-97e8671d801e has session id: ba787a05-04b6-4f7d-8681-e0b2cac26528","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.476275817Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
98702024-04-17T17:16:50.875Z{Crucible runtime is spawned
98712024-04-17T17:16:50.875Z"msg":"Upstairs opts: Upstairs UUID: 59ed0faf-f47b-4c1a-b2ef-97e8671d801e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.476359212Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
98722024-04-17T17:16:50.875Z{"msg":"Crucible stats registered with UUID: 59ed0faf-f47b-4c1a-b2ef-97e8671d801e","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.476416456Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
98732024-04-17T17:16:50.875Z{"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.476442167Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539}
98742024-04-17T17:16:50.875Z{"msg":"59ed0faf-f47b-4c1a-b2ef-97e8671d801e active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.476471779Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
98752024-04-17T17:16:50.875Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.47648651Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"client":"0","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
98762024-04-17T17:16:50.875Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.476516472Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"client":"0","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
98772024-04-17T17:16:50.875Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.476541983Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"client":"1","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
98782024-04-17T17:16:50.875Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.476565635Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"client":"1","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
98792024-04-17T17:16:50.875Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.476601087Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"client":"2","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
98802024-04-17T17:16:50.875Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.476628009Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"client":"2","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
98812024-04-17T17:16:50.875Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.476677112Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"":"io task","client":"2","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
98822024-04-17T17:16:50.875Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.476708413Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"":"io task","client":"0","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
98832024-04-17T17:16:50.876Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.476733525Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"":"io task","client":"1","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
98842024-04-17T17:16:50.876Z{"msg":"ds_connection connected from Ok(127.0.0.1:64656)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.476791019Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"":"io task","client":"2","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
98852024-04-17T17:16:50.876Z{"msg":"ds_connection connected from Ok(127.0.0.1:50350)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.476829241Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"":"io task","client":"1","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
98862024-04-17T17:16:50.876Z{"msg":"ds_connection connected from Ok(127.0.0.1:44381)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.476919276Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"":"io task","client":"0","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
98872024-04-17T17:16:50.876Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.477695713Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"client":"2","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
98882024-04-17T17:16:50.876Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.477732285Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"client":"2","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
98892024-04-17T17:16:50.876Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.477790049Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"client":"0","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
98902024-04-17T17:16:50.876Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.477835622Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"client":"0","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
98912024-04-17T17:16:50.876Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.477852923Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"client":"1","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
98922024-04-17T17:16:50.876Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.477915816Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"client":"1","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
98932024-04-17T17:16:50.876Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.478313671Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"client":"2","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
98942024-04-17T17:16:50.876Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.478356893Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"client":"1","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
98952024-04-17T17:16:50.876Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.478385675Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"client":"0","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
98962024-04-17T17:16:50.876Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.478825222Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"client":"2","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
98972024-04-17T17:16:50.876Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.478877885Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"client":"2","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
98982024-04-17T17:16:50.876Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.478922868Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
98992024-04-17T17:16:50.876Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.478951239Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"client":"1","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
99002024-04-17T17:16:50.876Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.478988121Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"client":"1","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
99012024-04-17T17:16:50.876Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.479013413Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
99022024-04-17T17:16:50.876Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.479045035Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"client":"0","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
99032024-04-17T17:16:50.876Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.479079327Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"client":"0","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
99042024-04-17T17:16:50.876Z{"msg":"[0]R flush_numbers[0..12]: [161, 160, 159, 159, 150, 146, 146, 161, 161, 151, 148, 161]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.479116469Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"":"downstairs","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
99052024-04-17T17:16:50.876Z{"msg":"[0]R generation[0..12]: [90, 89, 88, 88, 85, 83, 83, 90, 90, 86, 84, 90]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.479150851Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"":"downstairs","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
99062024-04-17T17:16:50.877Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.479186544Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"":"downstairs","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
99072024-04-17T17:16:50.877Z{"msg":"[1]R flush_numbers[0..12]: [161, 160, 159, 159, 150, 146, 146, 161, 161, 151, 148, 161]","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO
99082024-04-17T17:16:50.877Zlevel":30,"time":"2024-04-17T17:16:49.479226566Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"":"downstairs","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
99092024-04-17T17:16:50.877Z{"msg":"[1]R generation[0..12]: [90, 89, 88, 88, 85, 83, 83, 90, 90, 86, 84, 90]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.479251148Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"":"downstairs","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
99102024-04-17T17:16:50.877Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.47928834Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"":"downstairs","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
99112024-04-17T17:16:50.877Z{"msg":"[2]R flush_numbers[0..12]: [161, 160, 159, 159, 150, 146, 146, 161, 161, 151, 148, 161]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.479317392Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"":"downstairs","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
99122024-04-17T17:16:50.877Z{"msg":"[2]R generation[0..12]: [90, 89, 88, 88, 85, 83, 83, 90, 90, 86, 84, 90]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.479342873Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"":"downstairs","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
99132024-04-17T17:16:50.877Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.479369735Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"":"downstairs","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
99142024-04-17T17:16:50.877Z{"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.479403837Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"":"downstairs","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
99152024-04-17T17:16:50.877Z{"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.479439799Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"":"downstairs","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
99162024-04-17T17:16:50.877Z{"msg":"Next flush: 163","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.479483672Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"":"downstairs","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
99172024-04-17T17:16:50.877Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.479511613Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"":"downstairs","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
99182024-04-17T17:16:50.877ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
99192024-04-17T17:16:50.877Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.479550636Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
99202024-04-17T17:16:50.877Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.479583498Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"client":"0","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
99212024-04-17T17:16:50.877Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.479600279Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"client":"1","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
99222024-04-17T17:16:50.877Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.47962075Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"client":"2","":"downstairs","session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
99232024-04-17T17:16:50.877Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.479634311Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
99242024-04-17T17:16:50.877Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.479653842Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
99252024-04-17T17:16:50.877Z{"msg":"59ed0faf-f47b-4c1a-b2ef-97e8671d801e is now active with session: ba787a05-04b6-4f7d-8681-e0b2cac26528","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.479666863Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
99262024-04-17T17:16:50.877Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.479679503Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539,"session_id":"ba787a05-04b6-4f7d-8681-e0b2cac26528"}
99272024-04-17T17:16:50.877Z{"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:49.479691534Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539}
99282024-04-17T17:16:50.878ZLoading write count information from file "/tmp/test_repair_verify.data"
99292024-04-17T17:16:50.878ZRun Repair workload
99302024-04-17T17:16:50.881Z01/30 Write block 358 len 36864 data: 6 6 6 6 5 4 5 4 5
99312024-04-17T17:16:50.882Z02/30 Write block 83 len 20480 data: 7 5 6 5 9
99322024-04-17T17:16:50.882Z03/30 Write block 102 len 36864 data: 8 10 10 9 9 9 8 7 7
99332024-04-17T17:16:50.882Z04/30 Read block 51 len 4096
99342024-04-17T17:16:50.882Z05/30 Read block 74 len 32768
99352024-04-17T17:16:50.882Z06/30 Write block 125 len 36864 data: 7 7 8 6 7 9 10 11 8
99362024-04-17T17:16:50.882Z07/30 Read block 366 len 16384
99372024-04-17T17:16:50.882Z08/30 Read block 489 len 40960
99382024-04-17T17:16:50.882Z09/30 Read block 159 len 32768
99392024-04-17T17:16:50.885Z10/30 Write block 98 len 40960 data: 9 12 12 10 9 11 11 10 10 10
99402024-04-17T17:16:50.886Z11/30 Read block 98 len 8192
99412024-04-17T17:16:50.886Z12/30 Write block 362 len 32768 data: 6 5 6 5 6 5 7 7
99422024-04-17T17:16:50.889Z13/30 Write block 76 len 32768 data: 6 6 4 3 4 4 6 8
99432024-04-17T17:16:50.889Z14/30 Flush
99442024-04-17T17:16:50.893Z15/30 Write block 539 len 28672 data: 7 7 8 9 10 9 7
99452024-04-17T17:16:50.893Z16/30 Write block 34 len 4096 data: 17
99462024-04-17T17:16:50.894Z17/30 Read block 500 len 4096
99472024-04-17T17:16:50.894Z18/30 Read block 497 len 36864
99482024-04-17T17:16:50.894Z19/30 Read block 268 len 32768
99492024-04-17T17:16:50.894Z20/30 Flush
99502024-04-17T17:16:50.897Z21/30 Write block 475 len 40960 data: 5 6 10 12 12 14 13 13 15 15
99512024-04-17T17:16:50.897Z22/30 Read block 277 len 16384
99522024-04-17T17:16:50.901Z23/30 Write block 336 len 24576 data: 10 12 11 11 9 9
99532024-04-17T17:16:50.901Z24/30 Read block 25 len 12288
99542024-04-17T17:16:50.901Z25/30 Write block 13 len 8192 data: 7 9
99552024-04-17T17:16:50.904Z26/30 Write block 187 len 12288 data: 5 6 6
99562024-04-17T17:16:50.904Z27/30 Write block 145 len 40960 data: 7 6 4 4 5 5 4 5 4 7
99572024-04-17T17:16:50.905Z28/30 Read block 341 len 32768
99582024-04-17T17:16:50.905Z29/30 Read block 300 len 24576
99592024-04-17T17:16:50.912Z30/30 Read block 430 len 4096
99602024-04-17T17:16:50.912Z----------------------------------------------------------------
99612024-04-17T17:16:50.912Z Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:30
99622024-04-17T17:16:50.912ZStates: Active Active Active
99632024-04-17T17:16:50.912ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
99642024-04-17T17:16:50.912Z 1 Acked 1000 Write 9 Done Sent Done false
99652024-04-17T17:16:50.912Z 2 Acked 1001 Write 5 Done Sent Done false
99662024-04-17T17:16:50.912Z 3 Acked 1002 Write 9 Done Sent Done false
99672024-04-17T17:16:50.912Z 4 Acked 1003 Read 1 Done Sent Done false
99682024-04-17T17:16:50.912Z 5 Acked 1004 Read 8 Done Sent Done false
99692024-04-17T17:16:50.912Z 6 Acked 1005 Write 9 Done Sent Done false
99702024-04-17T17:16:50.912Z 7 Acked 1006 Read 4 Done Sent Done false
99712024-04-17T17:16:50.912Z 8 Acked 1007 Read 10 Done Sent Done false
99722024-04-17T17:16:50.912Z 9 Acked 1008 Read 8 Done Sent Done false
99732024-04-17T17:16:50.912Z 10 Acked 1009 Write 10 Done Sent Done false
99742024-04-17T17:16:50.912Z 11 Acked 1010 Read 2 Done Sent Done false
99752024-04-17T17:16:50.912Z 12 Acked 1011 Write 8 Done Sent Done false
99762024-04-17T17:16:50.913Z 13 Acked 1012 Write 8 Done Sent Done false
99772024-04-17T17:16:50.913Z 14 Acked 1013 Flush 0 Done Sent Done false
99782024-04-17T17:16:50.913Z 15 Acked 1014 Write 7 Done Sent Done false
99792024-04-17T17:16:50.913Z 16 Acked 1015 Write 1 Done Sent Done false
99802024-04-17T17:16:50.913Z 17 Acked 1016 Read 1 Done Sent Done false
99812024-04-17T17:16:50.913Z 18 Acked 1017 Read 9 Done Sent Done false
99822024-04-17T17:16:50.913Z 19 Acked 1018 Read 8 Done Sent Done false
99832024-04-17T17:16:50.913Z 20 Acked 1019 Flush 0 Done Sent Done false
99842024-04-17T17:16:50.913Z 21 Acked 1020 Write 10 Done Sent Done false
99852024-04-17T17:16:50.913Z 22 Acked 1021 Read 4 Done Sent Done false
99862024-04-17T17:16:50.913Z 23 Acked 1022 Write 6 Done Sent Done false
99872024-04-17T17:16:50.913Z 24 Acked 1023 Read 3 Done Sent Done false
99882024-04-17T17:16:50.913Z 25 Acked 1024 Write 2 Done Sent Done false
99892024-04-17T17:16:50.913Z 26 Acked 1025 Write 3 Done Sent Done false
99902024-04-17T17:16:50.913Z 27 Acked 1026 Write 10 Done Sent Done false
99912024-04-17T17:16:50.913Z 28 Acked 1027 Read 8 Done Sent Done false
99922024-04-17T17:16:50.913Z 29 Acked 1028 Read 6 Done Sent Done false
99932024-04-17T17:16:50.913Z 30 Acked 1029 Read 1 Done Sent Sent false
99942024-04-17T17:16:50.913Z STATES DS:0 DS:1 DS:2 TOTAL
99952024-04-17T17:16:50.913Z New 0 0 0 0
99962024-04-17T17:16:50.913Z Sent 0 30 1 31
99972024-04-17T17:16:50.913Z Done 30 0 29 59
99982024-04-17T17:16:50.913Z Skipped 0 0 0 0
99992024-04-17T17:16:50.913Z Error 0 0 0 0
100002024-04-17T17:16:50.914ZLast Flush: 1019 0 1019
100012024-04-17T17:16:50.914ZDownstairs last five completed:
100022024-04-17T17:16:50.914ZUpstairs last five completed: 30 29 28 27 26
100032024-04-17T17:16:50.914Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:16:49.51393884Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1539}
100042024-04-17T17:16:50.914ZWrote out file "/tmp/test_repair_verify.data"
100052024-04-17T17:16:50.917Z
100062024-04-17T17:16:50.917ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
100072024-04-17T17:16:50.930ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
100082024-04-17T17:16:50.930Z 0 000-019 90 90 90 161 161 161 T F T <---
100092024-04-17T17:16:50.934Z 1 020-039 92 89 92 164 160 164 F F F <---
100102024-04-17T17:16:50.935Z 2 040-059 88 88 88 159 159 159 F F F
100112024-04-17T17:16:50.935Z 3 060-079 92 88 92 163 159 163 F F F <---
100122024-04-17T17:16:50.935Z 4 080-099 92 85 92 163 150 163 F F F <---
100132024-04-17T17:16:50.935Z 5 100-119 92 83 92 163 146 163 F F F <---
100142024-04-17T17:16:50.935Z 6 120-139 92 83 92 163 146 163 F F F <---
100152024-04-17T17:16:50.935Z 7 140-159 90 90 90 161 161 161 T F T <---
100162024-04-17T17:16:50.935Z 8 160-179 90 90 90 161 161 161 F F F
100172024-04-17T17:16:50.935Z 9 180-199 86 86 86 151 151 151 T F T <---
100182024-04-17T17:16:50.935Z 10 200-219 84 84 84 148 148 148 F F F
100192024-04-17T17:16:50.935Z 11 220-239 90 90 90 161 161 161 F F F
100202024-04-17T17:16:50.935Z 12 240-259 90 90 90 161 161 161 F F F
100212024-04-17T17:16:50.935Z 13 260-279 90 90 90 161 161 161 F F F
100222024-04-17T17:16:50.935Z 14 280-299 91 91 91 162 162 162 F F F
100232024-04-17T17:16:50.935Z 15 300-319 91 91 91 162 162 162 F F F
100242024-04-17T17:16:50.935Z 16 320-339 85 85 85 150 150 150 T F T <---
100252024-04-17T17:16:50.935Z 17 340-359 92 86 92 163 153 163 T F T <---
100262024-04-17T17:16:50.935Z 18 360-379 92 81 92 163 142 163 F F F <---
100272024-04-17T17:16:50.935Z 19 380-399 88 88 88 157 157 157 F F F
100282024-04-17T17:16:50.935Z 20 400-419 88 88 88 159 159 159 F F F
100292024-04-17T17:16:50.935Z 21 420-439 90 90 90 161 161 161 F F F
100302024-04-17T17:16:50.935Z 22 440-459 82 82 82 144 144 144 F F F
100312024-04-17T17:16:50.935Z 23 460-479 91 91 91 162 162 162 T F T <---
100322024-04-17T17:16:50.936Z 24 480-499 91 91 91 162 162 162 T F T <---
100332024-04-17T17:16:50.936Z 25 500-519 70 70 70 127 127 127 F F F
100342024-04-17T17:16:50.936Z 26 520-539 92 90 92 164 161 164 F F F <---
100352024-04-17T17:16:50.936Z 27 540-559 92 87 92 164 154 164 F F F <---
100362024-04-17T17:16:50.936Z 28 560-579 85 85 85 150 150 150 F F F
100372024-04-17T17:16:50.936Z 29 580-599 82 82 82 145 145 145 F F F
100382024-04-17T17:16:50.936ZMax gen: 92, Max flush: 164
100392024-04-17T17:16:50.936ZError: Difference in extent metadata found!
100402024-04-17T17:16:50.936ZA Difference in extent metadata is expected here
100412024-04-17T17:16:50.936ZOn loop 46
100422024-04-17T17:16:50.936Z
100432024-04-17T17:16:50.940ZVerifying data now
100442024-04-17T17:16:55.633ZNo change after verify
100452024-04-17T17:16:55.633ZLoop: 46 Downstairs dump after verify (and repair):
100462024-04-17T17:16:55.645ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
100472024-04-17T17:16:55.645Z 0 000-019 93 93 93 165 165 165 F F F
100482024-04-17T17:16:55.649Z 1 020-039 92 92 92 164 164 164 F F F
100492024-04-17T17:16:55.649Z 2 040-059 88 88 88 159 159 159 F F F
100502024-04-17T17:16:55.649Z 3 060-079 92 92 92 163 163 163 F F F
100512024-04-17T17:16:55.649Z 4 080-099 92 92 92 163 163 163 F F F
100522024-04-17T17:16:55.649Z 5 100-119 92 92 92 163 163 163 F F F
100532024-04-17T17:16:55.649Z 6 120-139 92 92 92 163 163 163 F F F
100542024-04-17T17:16:55.649Z 7 140-159 93 93 93 165 165 165 F F F
100552024-04-17T17:16:55.649Z 8 160-179 90 90 90 161 161 161 F F F
100562024-04-17T17:16:55.649Z 9 180-199 93 93 93 165 165 165 F F F
100572024-04-17T17:16:55.649Z 10 200-219 84 84 84 148 148 148 F F F
100582024-04-17T17:16:55.649Z 11 220-239 90 90 90 161 161 161 F F F
100592024-04-17T17:16:55.649Z 12 240-259 90 90 90 161 161 161 F F F
100602024-04-17T17:16:55.649Z 13 260-279 90 90 90 161 161 161 F F F
100612024-04-17T17:16:55.649Z 14 280-299 91 91 91 162 162 162 F F F
100622024-04-17T17:16:55.649Z 15 300-319 91 91 91 162 162 162 F F F
100632024-04-17T17:16:55.649Z 16 320-339 93 93 93 165 165 165 F F F
100642024-04-17T17:16:55.649Z 17 340-359 93 93 93 165 165 165 F F F
100652024-04-17T17:16:55.650Z 18 360-379 92 92 92 163 163 163 F F F
100662024-04-17T17:16:55.650Z 19 380-399 88 88 88 157 157 157 F F F
100672024-04-17T17:16:55.650Z 20 400-419 88 88 88 159 159 159 F F F
100682024-04-17T17:16:55.650Z 21 420-439 90 90 90 161 161 161 F F F
100692024-04-17T17:16:55.650Z 22 440-459 82 82 82 144 144 144 F F F
100702024-04-17T17:16:55.650Z 23 460-479 93 93 93 165 165 165 F F F
100712024-04-17T17:16:55.650Z 24 480-499 93 93 93 165 165 165 F F F
100722024-04-17T17:16:55.650Z 25 500-519 70 70 70 127 127 127 F F F
100732024-04-17T17:16:55.650Z 26 520-539 92 92 92 164 164 164 F F F
100742024-04-17T17:16:55.650Z 27 540-559 92 92 92 164 164 164 F F F
100752024-04-17T17:16:55.650Z 28 560-579 85 85 85 150 150 150 F F F
100762024-04-17T17:16:55.650Z 29 580-599 82 82 82 145 145 145 F F F
100772024-04-17T17:16:55.650ZMax gen: 93, Max flush: 165
100782024-04-17T17:16:55.654Z
100792024-04-17T17:16:55.654ZBegin loop 47
100802024-04-17T17:16:55.654ZDownstairs to restart: 2
100812024-04-17T17:16:55.671Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.279342098Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547}
100822024-04-17T17:16:55.671ZCrucible runtime is spawned
100832024-04-17T17:16:55.678Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.279682988Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547}
100842024-04-17T17:16:55.678Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.279725501Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547}
100852024-04-17T17:16:55.678Z{"msg":"Crucible b99b438e-8956-4350-b467-f1312ace1f13 has session id: 55067bcb-1068-4ff8-b9e6-d9a3f48073b2","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.279752453Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
100862024-04-17T17:16:55.678Z{"msg":"Upstairs opts: Upstairs UUID: b99b438e-8956-4350-b467-f1312ace1f13, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.279801966Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
100872024-04-17T17:16:55.678Z{"msg":"Crucible stats registered with UUID: b99b438e-8956-4350-b467-f1312ace1f13","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.279844058Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
100882024-04-17T17:16:55.678Z{"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.279889201Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547}
100892024-04-17T17:16:55.679Z{"msg":"b99b438e-8956-4350-b467-f1312ace1f13 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.279911902Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
100902024-04-17T17:16:55.679Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.279944584Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"client":"0","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
100912024-04-17T17:16:55.679Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.279963936Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"client":"0","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
100922024-04-17T17:16:55.679Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.280001838Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"client":"1","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
100932024-04-17T17:16:55.679Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.280025469Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"client":"1","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
100942024-04-17T17:16:55.679Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.280057351Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"client":"2","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
100952024-04-17T17:16:55.679Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.280083503Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"client":"2","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
100962024-04-17T17:16:55.679Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.280125385Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"":"io task","client":"0","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
100972024-04-17T17:16:55.679Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.280179109Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"":"io task","client":"1","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
100982024-04-17T17:16:55.679Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.280213221Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"":"io task","client":"2","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
100992024-04-17T17:16:55.679Z{"msg":"ds_connection connected from Ok(127.0.0.1:35612)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.280244773Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"":"io task","client":"1","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101002024-04-17T17:16:55.679Z{"msg":"ds_connection connected from Ok(127.0.0.1:59837)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.280299376Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"":"io task","client":"0","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101012024-04-17T17:16:55.679Z{"msg":"ds_connection connected from Ok(127.0.0.1:63613)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.280347719Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"":"io task","client":"2","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101022024-04-17T17:16:55.679Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.281039731Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"client":"0","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101032024-04-17T17:16:55.679Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.281077843Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"client":"0","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101042024-04-17T17:16:55.679Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.281106935Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"client":"1","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101052024-04-17T17:16:55.679Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.281134536Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"client":"1","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101062024-04-17T17:16:55.679Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.281634537Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"client":"0","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101072024-04-17T17:16:55.679Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.281671199Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"client":"1","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101082024-04-17T17:16:55.679Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.281736323Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"client":"2","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101092024-04-17T17:16:55.679Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.281769765Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"client":"2","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101102024-04-17T17:16:55.679Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.282259025Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"client":"0","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101112024-04-17T17:16:55.680Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.282294727Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"client":"0","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101122024-04-17T17:16:55.680Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.282309838Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101132024-04-17T17:16:55.680Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.282322339Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"client":"1","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101142024-04-17T17:16:55.680Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.282335349Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"client":"1","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101152024-04-17T17:16:55.680Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.28234881Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101162024-04-17T17:16:55.680Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.282361531Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"client":"2","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101172024-04-17T17:16:55.680Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.282768815Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"client":"2","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101182024-04-17T17:16:55.680Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.282797857Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"client":"2","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101192024-04-17T17:16:55.680Z{"msg":"[0]R flush_numbers[0..12]: [165, 164, 159, 163, 163, 163, 163, 165, 161, 165, 148, 161]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.282812828Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"":"downstairs","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101202024-04-17T17:16:55.680Z{"msg":"[0]R generation[0..12]: [93, 92, 88, 92, 92, 92, 92, 93, 90, 93, 84, 90]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.282834319Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"":"downstairs","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101212024-04-17T17:16:55.680Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.28284892Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"":"downstairs","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101222024-04-17T17:16:55.680Z{"msg":"[1]R flush_numbers[0..12]: [165, 164, 159, 163, 163, 163, 163, 165, 161, 165, 148, 161]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.282863271Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"":"downstairs","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101232024-04-17T17:16:55.680Z{"msg":"[1]R generation[0..12]: [93, 92, 88, 92, 92, 92, 92, 93, 90, 93, 84, 90]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.282876762Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"":"downstairs","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101242024-04-17T17:16:55.680Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.282897613Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"":"downstairs","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101252024-04-17T17:16:55.680Z{"msg":"[2]R flush_numbers[0..12]: [165, 164, 159, 163, 163, 163, 163, 165, 161, 165, 148, 161]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.282911634Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"":"downstairs","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101262024-04-17T17:16:55.680ZWait for a query_work_queue command to finish before sending IO
101272024-04-17T17:16:55.680Z{"msg":"[2]R generation[0..12]: [93, 92, 88, 92, 92, 92, 92, 93, 90, 93, 84, 90]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.282934296Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"":"downstairs","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101282024-04-17T17:16:55.680Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.282948846Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"":"downstairs","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101292024-04-17T17:16:55.680Z{"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.282968068Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"":"downstairs","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101302024-04-17T17:16:55.680Z{"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.282982659Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"":"downstairs","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101312024-04-17T17:16:55.680Z{"msg":"Next flush: 166","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.282995589Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"":"downstairs","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101322024-04-17T17:16:55.680Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.283019021Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"":"downstairs","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101332024-04-17T17:16:55.681Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.283043792Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101342024-04-17T17:16:55.681Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.283056393Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"client":"0","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101352024-04-17T17:16:55.681Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.283072824Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"client":"1","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101362024-04-17T17:16:55.681Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.283089665Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"client":"2","":"downstairs","session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101372024-04-17T17:16:55.681Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.283103046Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101382024-04-17T17:16:55.681Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.283114867Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101392024-04-17T17:16:55.681Z{"msg":"b99b438e-8956-4350-b467-f1312ace1f13 is now active with session: 55067bcb-1068-4ff8-b9e6-d9a3f48073b2","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.283128097Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101402024-04-17T17:16:55.681Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.283140088Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547,"session_id":"55067bcb-1068-4ff8-b9e6-d9a3f48073b2"}
101412024-04-17T17:16:55.681Z{"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:54.283152499Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547}
101422024-04-17T17:16:55.681ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
101432024-04-17T17:16:55.685ZLoading write count information from file "/tmp/test_repair_verify.data"
101442024-04-17T17:16:55.685ZRun Repair workload
101452024-04-17T17:16:55.685Z01/30 Flush
101462024-04-17T17:16:55.685Z02/30 Write block 444 len 20480 data: 8 6 7 5 6
101472024-04-17T17:16:55.685Z03/30 Read block 237 len 40960
101482024-04-17T17:16:55.685Z04/30 Read block 364 len 12288
101492024-04-17T17:16:55.685Z05/30 Read block 443 len 16384
101502024-04-17T17:16:55.685Z06/30 Write block 303 len 8192 data: 5 5
101512024-04-17T17:16:55.685Z07/30 Read block 227 len 8192
101522024-04-17T17:16:55.685Z08/30 Write block 389 len 16384 data: 6 6 7 8
101532024-04-17T17:16:55.685Z09/30 Read block 56 len 24576
101542024-04-17T17:16:55.685Z10/30 Read block 474 len 32768
101552024-04-17T17:16:55.685Z11/30 Write block 474 len 8192 data: 7 6
101562024-04-17T17:16:55.689Z12/30 Write block 476 len 12288 data: 7 11 13
101572024-04-17T17:16:55.689Z13/30 Read block 391 len 36864
101582024-04-17T17:16:55.689Z14/30 Read block 183 len 36864
101592024-04-17T17:16:55.689Z15/30 Read block 173 len 36864
101602024-04-17T17:16:55.689Z16/30 Read block 198 len 16384
101612024-04-17T17:16:55.689Z17/30 Read block 408 len 20480
101622024-04-17T17:16:55.689Z18/30 Read block 554 len 8192
101632024-04-17T17:16:55.693Z19/30 Read block 247 len 4096
101642024-04-17T17:16:55.693Z20/30 Read block 124 len 40960
101652024-04-17T17:16:55.693Z21/30 Write block 586 len 40960 data: 12 11 7 6 6 5 7 5 5 5
101662024-04-17T17:16:55.696Z22/30 Write block 412 len 20480 data: 11 12 12 11 11
101672024-04-17T17:16:55.696Z23/30 Write block 35 len 24576 data: 14 11 9 8 9 10
101682024-04-17T17:16:55.696Z24/30 Flush
101692024-04-17T17:16:55.699Z25/30 Write block 289 len 24576 data: 6 7 6 5 5 4
101702024-04-17T17:16:55.699Z26/30 Write block 431 len 20480 data: 5 4 7 7 8
101712024-04-17T17:16:55.703Z27/30 Read block 72 len 28672
101722024-04-17T17:16:55.703Z28/30 Write block 561 len 28672 data: 9 9 8 9 6 7 7
101732024-04-17T17:16:55.703Z29/30 Read block 345 len 12288
101742024-04-17T17:16:55.703Z30/30 Read block 183 len 28672
101752024-04-17T17:16:55.707Z----------------------------------------------------------------
101762024-04-17T17:16:55.707Z Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:29
101772024-04-17T17:16:55.707ZStates: Active Active Active
101782024-04-17T17:16:55.707ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
101792024-04-17T17:16:55.707Z 2 Acked 1001 Write 5 Done Done Done false
101802024-04-17T17:16:55.707Z 3 Acked 1002 Read 10 Done Done Done false
101812024-04-17T17:16:55.707Z 4 Acked 1003 Read 3 Done Done Sent false
101822024-04-17T17:16:55.707Z 5 Acked 1004 Read 4 Done Done Sent false
101832024-04-17T17:16:55.707Z 6 Acked 1005 Write 2 Done Done Sent false
101842024-04-17T17:16:55.707Z 7 Acked 1006 Read 2 Done Done Sent false
101852024-04-17T17:16:55.707Z 8 Acked 1007 Write 4 Done Done Sent false
101862024-04-17T17:16:55.707Z 9 Acked 1008 Read 6 Done Done Sent false
101872024-04-17T17:16:55.707Z 10 Acked 1009 Read 8 Done Done Sent false
101882024-04-17T17:16:55.707Z 11 Acked 1010 Write 2 Done Done Sent false
101892024-04-17T17:16:55.707Z 12 Acked 1011 Write 3 Done Done Sent false
101902024-04-17T17:16:55.708Z 13 Acked 1012 Read 9 Done Done Sent false
101912024-04-17T17:16:55.708Z 14 Acked 1013 Read 9 Done Done Sent false
101922024-04-17T17:16:55.708Z 15 Acked 1014 Read 9 Done Done Sent false
101932024-04-17T17:16:55.708Z 16 Acked 1015 Read 4 Done Done Sent false
101942024-04-17T17:16:55.708Z 17 Acked 1016 Read 5 Done Done Sent false
101952024-04-17T17:16:55.708Z 18 Acked 1017 Read 2 Done Done Sent false
101962024-04-17T17:16:55.708Z 19 Acked 1018 Read 1 Done Done Sent false
101972024-04-17T17:16:55.708Z 20 Acked 1019 Read 10 Done Done Sent false
101982024-04-17T17:16:55.708Z 21 Acked 1020 Write 10 Done Done Sent false
101992024-04-17T17:16:55.708Z 22 Acked 1021 Write 5 Done Done Sent false
102002024-04-17T17:16:55.708Z 23 Acked 1022 Write 6 Done Done Sent false
102012024-04-17T17:16:55.708Z 24 Acked 1023 Flush 0 Done Done Sent false
102022024-04-17T17:16:55.708Z 25 Acked 1024 Write 6 Done Done Sent false
102032024-04-17T17:16:55.708Z 26 Acked 1025 Write 5 Done Done Sent false
102042024-04-17T17:16:55.708Z 27 Acked 1026 Read 7 Done Done Sent false
102052024-04-17T17:16:55.708Z 28 Acked 1027 Write 7 Done Done Sent false
102062024-04-17T17:16:55.708Z 29 Acked 1028 Read 3 Done Done Sent false
102072024-04-17T17:16:55.708Z 30 Acked 1029 Read 7 Done Done Sent false
102082024-04-17T17:16:55.708Z STATES DS:0 DS:1 DS:2 TOTAL
102092024-04-17T17:16:55.708Z New 0 0 0 0
102102024-04-17T17:16:55.708Z Sent 0 0 27 27
102112024-04-17T17:16:55.708Z Done 29 29 2 60
102122024-04-17T17:16:55.708Z Skipped 0 0 0 0
102132024-04-17T17:16:55.709Z Error 0 0 0 0
102142024-04-17T17:16:55.709ZLast Flush: 1023 1023 1000
102152024-04-17T17:16:55.709ZDownstairs last five completed: 1000
102162024-04-17T17:16:55.709ZUpstairs last five completed: 30 29 28 27 26
102172024-04-17T17:16:55.709Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:16:54.311806555Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1547}
102182024-04-17T17:16:55.709ZWrote out file "/tmp/test_repair_verify.data"
102192024-04-17T17:16:55.712Z
102202024-04-17T17:16:55.712ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
102212024-04-17T17:16:55.726ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
102222024-04-17T17:16:55.726Z 0 000-019 93 93 93 165 165 165 F F F
102232024-04-17T17:16:55.726Z 1 020-039 94 94 92 167 167 164 F F F <---
102242024-04-17T17:16:55.726Z 2 040-059 94 94 88 167 167 159 F F F <---
102252024-04-17T17:16:55.726Z 3 060-079 92 92 92 163 163 163 F F F
102262024-04-17T17:16:55.727Z 4 080-099 92 92 92 163 163 163 F F F
102272024-04-17T17:16:55.727Z 5 100-119 92 92 92 163 163 163 F F F
102282024-04-17T17:16:55.727Z 6 120-139 92 92 92 163 163 163 F F F
102292024-04-17T17:16:55.727Z 7 140-159 93 93 93 165 165 165 F F F
102302024-04-17T17:16:55.727Z 8 160-179 90 90 90 161 161 161 F F F
102312024-04-17T17:16:55.727Z 9 180-199 93 93 93 165 165 165 F F F
102322024-04-17T17:16:55.727Z 10 200-219 84 84 84 148 148 148 F F F
102332024-04-17T17:16:55.731Z 11 220-239 90 90 90 161 161 161 F F F
102342024-04-17T17:16:55.731Z 12 240-259 90 90 90 161 161 161 F F F
102352024-04-17T17:16:55.731Z 13 260-279 90 90 90 161 161 161 F F F
102362024-04-17T17:16:55.731Z 14 280-299 91 91 91 162 162 162 T T F <---
102372024-04-17T17:16:55.731Z 15 300-319 94 94 91 167 167 162 F F F <---
102382024-04-17T17:16:55.731Z 16 320-339 93 93 93 165 165 165 F F F
102392024-04-17T17:16:55.731Z 17 340-359 93 93 93 165 165 165 F F F
102402024-04-17T17:16:55.731Z 18 360-379 92 92 92 163 163 163 F F F
102412024-04-17T17:16:55.731Z 19 380-399 94 94 88 167 167 157 F F F <---
102422024-04-17T17:16:55.731Z 20 400-419 94 94 88 167 167 159 F F F <---
102432024-04-17T17:16:55.731Z 21 420-439 90 90 90 161 161 161 T T F <---
102442024-04-17T17:16:55.731Z 22 440-459 94 94 82 167 167 144 F F T <---
102452024-04-17T17:16:55.731Z 23 460-479 94 94 93 167 167 165 F F F <---
102462024-04-17T17:16:55.731Z 24 480-499 93 93 93 165 165 165 F F F
102472024-04-17T17:16:55.731Z 25 500-519 70 70 70 127 127 127 F F F
102482024-04-17T17:16:55.731Z 26 520-539 92 92 92 164 164 164 F F F
102492024-04-17T17:16:55.731Z 27 540-559 92 92 92 164 164 164 F F F
102502024-04-17T17:16:55.732Z 28 560-579 85 85 85 150 150 150 T T F <---
102512024-04-17T17:16:55.732Z 29 580-599 94 94 82 167 167 145 F F F <---
102522024-04-17T17:16:55.732ZMax gen: 94, Max flush: 167
102532024-04-17T17:16:55.732ZError: Difference in extent metadata found!
102542024-04-17T17:16:55.732ZA Difference in extent metadata is expected here
102552024-04-17T17:16:55.732ZOn loop 47
102562024-04-17T17:16:55.732Z
102572024-04-17T17:16:55.736ZVerifying data now
102582024-04-17T17:17:00.246ZNo change after verify
102592024-04-17T17:17:00.246ZLoop: 47 Downstairs dump after verify (and repair):
102602024-04-17T17:17:00.267ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
102612024-04-17T17:17:00.267Z 0 000-019 93 93 93 165 165 165 F F F
102622024-04-17T17:17:00.682Z 1 020-039 94 94 94 167 167 167 F F F
102632024-04-17T17:17:00.682Z 2 040-059 94 94 94 167 167 167 F F F
102642024-04-17T17:17:00.682Z 3 060-079 92 92 92 163 163 163 F F F
102652024-04-17T17:17:00.682Z 4 080-099 92 92 92 163 163 163 F F F
102662024-04-17T17:17:00.683Z 5 100-119 92 92 92 163 163 163 F F F
102672024-04-17T17:17:00.683Z 6 120-139 92 92 92 163 163 163 F F F
102682024-04-17T17:17:00.683Z 7 140-159 93 93 93 165 165 165 F F F
102692024-04-17T17:17:00.683Z 8 160-179 90 90 90 161 161 161 F F F
102702024-04-17T17:17:00.683Z 9 180-199 93 93 93 165 165 165 F F F
102712024-04-17T17:17:00.683Z 10 200-219 84 84 84 148 148 148 F F F
102722024-04-17T17:17:00.683Z 11 220-239 90 90 90 161 161 161 F F F
102732024-04-17T17:17:00.683Z 12 240-259 90 90 90 161 161 161 F F F
102742024-04-17T17:17:00.683Z 13 260-279 90 90 90 161 161 161 F F F
102752024-04-17T17:17:00.683Z 14 280-299 95 95 95 168 168 168 F F F
102762024-04-17T17:17:00.683Z 15 300-319 94 94 94 167 167 167 F F F
102772024-04-17T17:17:00.684Z 16 320-339 93 93 93 165 165 165 F F F
102782024-04-17T17:17:00.684Z 17 340-359 93 93 93 165 165 165 F F F
102792024-04-17T17:17:00.684Z 18 360-379 92 92 92 163 163 163 F F F
102802024-04-17T17:17:00.684Z 19 380-399 94 94 94 167 167 167 F F F
102812024-04-17T17:17:00.684Z 20 400-419 94 94 94 167 167 167 F F F
102822024-04-17T17:17:00.684Z 21 420-439 95 95 95 168 168 168 F F F
102832024-04-17T17:17:00.685Z 22 440-459 94 94 94 167 167 167 F F F
102842024-04-17T17:17:00.685Z 23 460-479 94 94 94 167 167 167 F F F
102852024-04-17T17:17:00.685Z 24 480-499 93 93 93 165 165 165 F F F
102862024-04-17T17:17:00.685Z 25 500-519 70 70 70 127 127 127 F F F
102872024-04-17T17:17:00.685Z 26 520-539 92 92 92 164 164 164 F F F
102882024-04-17T17:17:00.685Z 27 540-559 92 92 92 164 164 164 F F F
102892024-04-17T17:17:00.685Z 28 560-579 95 95 95 168 168 168 F F F
102902024-04-17T17:17:00.685Z 29 580-599 94 94 94 167 167 167 F F F
102912024-04-17T17:17:00.685ZMax gen: 95, Max flush: 168
102922024-04-17T17:17:00.685Z
102932024-04-17T17:17:00.686ZBegin loop 48
102942024-04-17T17:17:00.686ZDownstairs to restart: 0
102952024-04-17T17:17:00.698Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.891345009Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555}
102962024-04-17T17:17:00.698Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.891676129Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555}
102972024-04-17T17:17:00.698ZCrucible runtime is spawned
102982024-04-17T17:17:00.698Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.891733162Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555}
102992024-04-17T17:17:00.698Z{"msg":"Crucible f9e692f1-b759-40e9-8e25-3da9fd10f1a2 has session id: 3f05e7fa-f7a2-4012-b574-357067314d0c","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.891763655Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103002024-04-17T17:17:00.698Z{"msg":"Upstairs opts: Upstairs UUID: f9e692f1-b759-40e9-8e25-3da9fd10f1a2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.891793716Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103012024-04-17T17:17:00.698Z{"msg":"Crucible stats registered with UUID: f9e692f1-b759-40e9-8e25-3da9fd10f1a2","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.891809357Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103022024-04-17T17:17:00.698Z{"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.891822838Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555}
103032024-04-17T17:17:00.699Z{"msg":"f9e692f1-b759-40e9-8e25-3da9fd10f1a2 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.891922444Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103042024-04-17T17:17:00.699Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.891948466Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"client":"0","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103052024-04-17T17:17:00.699Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.891965597Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"client":"0","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103062024-04-17T17:17:00.699Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.891988808Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"client":"1","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103072024-04-17T17:17:00.699Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.8920157Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"client":"1","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103082024-04-17T17:17:00.699Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.892061202Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"client":"2","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103092024-04-17T17:17:00.699Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.892087294Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"client":"2","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103102024-04-17T17:17:00.699Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.892119826Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103112024-04-17T17:17:00.700Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.892167359Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103122024-04-17T17:17:00.700Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.892197061Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103132024-04-17T17:17:00.700Z{"msg":"ds_connection connected from Ok(127.0.0.1:53904)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.892233123Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103142024-04-17T17:17:00.700Z{"msg":"ds_connection connected from Ok(127.0.0.1:45991)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.892257724Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103152024-04-17T17:17:00.700Z{"msg":"ds_connection connected from Ok(127.0.0.1:56881)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.892293667Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103162024-04-17T17:17:00.700Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.892939686Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"client":"2","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103172024-04-17T17:17:00.700Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.892977618Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"client":"2","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103182024-04-17T17:17:00.700Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.893026921Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"client":"1","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103192024-04-17T17:17:00.700Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.893068054Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"client":"1","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103202024-04-17T17:17:00.700Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.893464247Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"client":"1","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103212024-04-17T17:17:00.700Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.893636818Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"client":"2","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103222024-04-17T17:17:00.700Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.893800188Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"client":"0","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103232024-04-17T17:17:00.700Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.89383524Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"client":"0","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103242024-04-17T17:17:00.700Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.893958888Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"client":"1","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103252024-04-17T17:17:00.700Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.894019791Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"client":"1","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103262024-04-17T17:17:00.700Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.894057903Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103272024-04-17T17:17:00.701Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.894085275Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"client":"2","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103282024-04-17T17:17:00.701Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.894122167Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"client":"2","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103292024-04-17T17:17:00.701Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.894152919Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103302024-04-17T17:17:00.701Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.894398414Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"client":"0","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103312024-04-17T17:17:00.701Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.894790388Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"client":"0","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103322024-04-17T17:17:00.701Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.89482648Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"client":"0","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103332024-04-17T17:17:00.701Z{"msg":"[0]R flush_numbers[0..12]: [165, 167, 167, 163, 163, 163, 163, 165, 161, 165, 148, 161]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.894843281Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"":"downstairs","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103342024-04-17T17:17:00.701Z{"msg":"[0]R generation[0..12]: [93, 94, 94, 92, 92, 92, 92, 93, 90, 93, 84, 90]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.894857942Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"":"downstairs","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103352024-04-17T17:17:00.701Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.894879923Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"":"downstairs","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103362024-04-17T17:17:00.701Z{"msg":"[1]R flush_numbers[0..12]: [165, 167, 167, 163, 163, 163, 163, 165, 161, 165, 148, 161]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.894894284Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"":"downstairs","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103372024-04-17T17:17:00.701Z{"msg":"[1]R generation[0..12]: [93, 94, 94, 92, 92, 92, 92, 93, 90, 93, 84, 90]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.894907775Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"":"downstairs","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103382024-04-17T17:17:00.701Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.894928786Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"":"downstairs","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103392024-04-17T17:17:00.701Z{"msg":"[2]R flush_numbers[0..12]: [165, 167, 167, 163, 163, 163, 163, 165, 161, 165, 148, 161]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.894943467Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"":"downstairs","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103402024-04-17T17:17:00.702Z{"msg":"[2]R generation[0..12]: [93, 94, 94, 92, 92, 92, 92, 93, 90, 93, 84, 90]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.894957348Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"":"downstairs","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103412024-04-17T17:17:00.702Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,Wait for a query_work_queue command to finish before sending IO
103422024-04-17T17:17:00.702Z"time":"2024-04-17T17:16:58.894979489Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"":"downstairs","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103432024-04-17T17:17:00.702Z{"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.89499579Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"":"downstairs","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103442024-04-17T17:17:00.702Z{"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.895009651Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"":"downstairs","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103452024-04-17T17:17:00.702Z{"msg":"Next flush: 169","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.895029572Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"":"downstairs","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103462024-04-17T17:17:00.702Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.895042603Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"":"downstairs","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103472024-04-17T17:17:00.702Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.895055524Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103482024-04-17T17:17:00.702Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.895074195Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"client":"0","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103492024-04-17T17:17:00.702Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.895087516Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"client":"1","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103502024-04-17T17:17:00.702Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.895100407Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"client":"2","":"downstairs","session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103512024-04-17T17:17:00.702Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.895125528Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103522024-04-17T17:17:00.702Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.895138899Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103532024-04-17T17:17:00.703Z{"msg":"f9e692f1-b759-40e9-8e25-3da9fd10f1a2 is now active with session: 3f05e7fa-f7a2-4012-b574-357067314d0c","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.89515142Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103542024-04-17T17:17:00.703Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.89516309Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555,"session_id":"3f05e7fa-f7a2-4012-b574-357067314d0c"}
103552024-04-17T17:17:00.703Z{"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:16:58.895175521Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555}
103562024-04-17T17:17:00.703ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
103572024-04-17T17:17:00.703ZLoading write count information from file "/tmp/test_repair_verify.data"
103582024-04-17T17:17:00.703ZRun Repair workload
103592024-04-17T17:17:00.703Z01/30 Write block 83 len 12288 data: 9 6 7
103602024-04-17T17:17:00.703Z02/30 Flush
103612024-04-17T17:17:00.703Z03/30 Flush
103622024-04-17T17:17:00.703Z04/30 Write block 325 len 28672 data: 6 7 6 6 6 6 7
103632024-04-17T17:17:00.703Z05/30 Read block 49 len 24576
103642024-04-17T17:17:00.703Z06/30 Write block 307 len 16384 data: 5 6 5 7
103652024-04-17T17:17:00.703Z07/30 Write block 135 len 24576 data: 8 9 8 7 7 6
103662024-04-17T17:17:00.703Z08/30 Read block 365 len 36864
103672024-04-17T17:17:00.704Z09/30 Read block 513 len 36864
103682024-04-17T17:17:00.704Z10/30 Read block 161 len 20480
103692024-04-17T17:17:00.704Z11/30 Read block 396 len 28672
103702024-04-17T17:17:00.704Z12/30 Write block 144 len 28672 data: 8 8 7 5 5 6 6
103712024-04-17T17:17:00.704Z13/30 Read block 291 len 32768
103722024-04-17T17:17:00.704Z14/30 Read block 464 len 8192
103732024-04-17T17:17:00.704Z15/30 Write block 234 len 16384 data: 10 11 9 7
103742024-04-17T17:17:00.704Z16/30 Write block 162 len 40960 data: 7 9 6 5 7 8 7 7 6 7
103752024-04-17T17:17:00.704Z17/30 Read block 343 len 8192
103762024-04-17T17:17:00.704Z18/30 Read block 66 len 20480
103772024-04-17T17:17:00.704Z19/30 Read block 519 len 8192
103782024-04-17T17:17:00.704Z20/30 Read block 200 len 28672
103792024-04-17T17:17:00.705Z21/30 Write block 548 len 36864 data: 10 11 9 9 10 8 8 8 8
103802024-04-17T17:17:00.705Z22/30 Read block 48 len 8192
103812024-04-17T17:17:00.705Z23/30 Write block 198 len 40960 data: 7 6 8 10 12 13 13 12 11 11
103822024-04-17T17:17:00.705Z24/30 Flush
103832024-04-17T17:17:00.705Z25/30 Write block 105 len 24576 data: 11 11 11 9 8 8
103842024-04-17T17:17:00.705Z26/30 Write block 405 len 4096 data: 9
103852024-04-17T17:17:00.705Z27/30 Write block 263 len 20480 data: 8 7 5 5 5
103862024-04-17T17:17:00.705Z28/30 Write block 498 len 36864 data: 9 8 9 8 8 8 9 7 6
103872024-04-17T17:17:00.705Z29/30 Read block 33 len 40960
103882024-04-17T17:17:00.705Z30/30 Write block 187 len 16384 data: 6 7 7 5
103892024-04-17T17:17:00.705Z----------------------------------------------------------------
103902024-04-17T17:17:00.705Z Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:27
103912024-04-17T17:17:00.705ZStates: Active Active Active
103922024-04-17T17:17:00.705ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
103932024-04-17T17:17:00.705Z 4 Acked 1003 Write 7 Sent Done Done false
103942024-04-17T17:17:00.705Z 5 Acked 1004 Read 6 Sent Done Done false
103952024-04-17T17:17:00.711Z 6 Acked 1005 Write 4 Sent Done Done false
103962024-04-17T17:17:00.711Z 7 Acked 1006 Write 6 Sent Done Done false
103972024-04-17T17:17:00.711Z 8 Acked 1007 Read 9 Sent Done Done false
103982024-04-17T17:17:00.711Z 9 Acked 1008 Read 9 Sent Done Done false
103992024-04-17T17:17:00.711Z 10 Acked 1009 Read 5 Sent Done Done false
104002024-04-17T17:17:00.711Z 11 Acked 1010 Read 7 Sent Done Done false
104012024-04-17T17:17:00.711Z 12 Acked 1011 Write 7 Sent Done Done false
104022024-04-17T17:17:00.711Z 13 Acked 1012 Read 8 Sent Done Done false
104032024-04-17T17:17:00.711Z 14 Acked 1013 Read 2 Sent Done Done false
104042024-04-17T17:17:00.712Z 15 Acked 1014 Write 4 Sent Done Done false
104052024-04-17T17:17:00.712Z 16 Acked 1015 Write 10 Sent Done Done false
104062024-04-17T17:17:00.712Z 17 Acked 1016 Read 2 Sent Done Done false
104072024-04-17T17:17:00.712Z 18 Acked 1017 Read 5 Sent Done Done false
104082024-04-17T17:17:00.712Z 19 Acked 1018 Read 2 Sent Done Done false
104092024-04-17T17:17:00.712Z 20 Acked 1019 Read 7 Sent Done Done false
104102024-04-17T17:17:00.712Z 21 Acked 1020 Write 9 Sent Done Done false
104112024-04-17T17:17:00.712Z 22 Acked 1021 Read 2 Sent Done Done false
104122024-04-17T17:17:00.712Z 23 Acked 1022 Write 10 Sent Done Done false
104132024-04-17T17:17:00.712Z 24 Acked 1023 Flush 0 Sent Done Done false
104142024-04-17T17:17:00.712Z 25 Acked 1024 Write 6 Sent Done Done false
104152024-04-17T17:17:00.712Z 26 Acked 1025 Write 1 Sent Done Done false
104162024-04-17T17:17:00.712Z 27 Acked 1026 Write 5 Sent Done Done false
104172024-04-17T17:17:00.712Z 28 Acked 1027 Write 9 Sent Done Done false
104182024-04-17T17:17:00.712Z 29 Acked 1028 Read 10 Sent Done Done false
104192024-04-17T17:17:00.712Z 30 Acked 1029 Write 4 Sent Sent Sent false
104202024-04-17T17:17:00.712Z STATES DS:0 DS:1 DS:2 TOTAL
104212024-04-17T17:17:00.712Z New 0 0 0 0
104222024-04-17T17:17:00.712Z Sent 27 1 1 29
104232024-04-17T17:17:00.712Z Done 0 26 26 52
104242024-04-17T17:17:00.712Z Skipped 0 0 0 0
104252024-04-17T17:17:00.712Z Error 0 0 0 0
104262024-04-17T17:17:00.712ZLast Flush: 1002 1023 1023
104272024-04-17T17:17:00.713ZDownstairs last five completed: 1002 1001 1000
104282024-04-17T17:17:00.713ZUpstairs last five completed: 30 29 28 27 26
104292024-04-17T17:17:00.713Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:16:58.929542004Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1555}
104302024-04-17T17:17:00.713ZWrote out file "/tmp/test_repair_verify.data"
104312024-04-17T17:17:00.713Z
104322024-04-17T17:17:00.713ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
104332024-04-17T17:17:00.713ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
104342024-04-17T17:17:00.713Z 0 000-019 93 93 93 165 165 165 F F F
104352024-04-17T17:17:00.713Z 1 020-039 94 94 94 167 167 167 F F F
104362024-04-17T17:17:00.713Z 2 040-059 94 94 94 167 167 167 F F F
104372024-04-17T17:17:00.713Z 3 060-079 92 92 92 163 163 163 F F F
104382024-04-17T17:17:00.713Z 4 080-099 96 96 96 169 169 169 F F F
104392024-04-17T17:17:00.713Z 5 100-119 92 92 92 163 163 163 F T T <---
104402024-04-17T17:17:00.713Z 6 120-139 92 96 96 163 171 171 F F F <---
104412024-04-17T17:17:00.713Z 7 140-159 93 96 96 165 171 171 F F F <---
104422024-04-17T17:17:00.713Z 8 160-179 90 96 96 161 171 171 F F F <---
104432024-04-17T17:17:00.713Z 9 180-199 93 96 96 165 171 171 F T T <---
104442024-04-17T17:17:00.713Z 10 200-219 84 96 96 148 171 171 F F F <---
104452024-04-17T17:17:00.713Z 11 220-239 90 96 96 161 171 171 F F F <---
104462024-04-17T17:17:00.713Z 12 240-259 90 90 90 161 161 161 F F F
104472024-04-17T17:17:00.713Z 13 260-279 90 90 90 161 161 161 F T T <---
104482024-04-17T17:17:00.713Z 14 280-299 95 95 95 168 168 168 F F F
104492024-04-17T17:17:00.713Z 15 300-319 94 96 96 167 171 171 F F F <---
104502024-04-17T17:17:00.713Z 16 320-339 93 96 96 165 171 171 F F F <---
104512024-04-17T17:17:00.714Z 17 340-359 93 93 93 165 165 165 F F F
104522024-04-17T17:17:00.714Z 18 360-379 92 92 92 163 163 163 F F F
104532024-04-17T17:17:00.714Z 19 380-399 94 94 94 167 167 167 F F F
104542024-04-17T17:17:00.714Z 20 400-419 94 94 94 167 167 167 F T T <---
104552024-04-17T17:17:00.714Z 21 420-439 95 95 95 168 168 168 F F F
104562024-04-17T17:17:00.714Z 22 440-459 94 94 94 167 167 167 F F F
104572024-04-17T17:17:00.714Z 23 460-479 94 94 94 167 167 167 F F F
104582024-04-17T17:17:00.714Z 24 480-499 93 93 93 165 165 165 F T T <---
104592024-04-17T17:17:00.714Z 25 500-519 70 70 70 127 127 127 F T T <---
104602024-04-17T17:17:00.714Z 26 520-539 92 92 92 164 164 164 F F F
104612024-04-17T17:17:00.714Z 27 540-559 92 96 96 164 171 171 F F F <---
104622024-04-17T17:17:00.714Z 28 560-579 95 95 95 168 168 168 F F F
104632024-04-17T17:17:00.714Z 29 580-599 94 94 94 167 167 167 F F F
104642024-04-17T17:17:00.714ZMax gen: 96, Max flush: 171
104652024-04-17T17:17:00.714ZError: Difference in extent metadata found!
104662024-04-17T17:17:00.714ZA Difference in extent metadata is expected here
104672024-04-17T17:17:00.714ZOn loop 48
104682024-04-17T17:17:00.714Z
104692024-04-17T17:17:00.714ZVerifying data now
104702024-04-17T17:17:04.999ZNo change after verify
104712024-04-17T17:17:05.000ZLoop: 48 Downstairs dump after verify (and repair):
104722024-04-17T17:17:05.013ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
104732024-04-17T17:17:05.014Z 0 000-019 93 93 93 165 165 165 F F F
104742024-04-17T17:17:05.018Z 1 020-039 94 94 94 167 167 167 F F F
104752024-04-17T17:17:05.018Z 2 040-059 94 94 94 167 167 167 F F F
104762024-04-17T17:17:05.018Z 3 060-079 92 92 92 163 163 163 F F F
104772024-04-17T17:17:05.018Z 4 080-099 96 96 96 169 169 169 F F F
104782024-04-17T17:17:05.018Z 5 100-119 97 97 97 172 172 172 F F F
104792024-04-17T17:17:05.018Z 6 120-139 96 96 96 171 171 171 F F F
104802024-04-17T17:17:05.018Z 7 140-159 96 96 96 171 171 171 F F F
104812024-04-17T17:17:05.018Z 8 160-179 96 96 96 171 171 171 F F F
104822024-04-17T17:17:05.018Z 9 180-199 97 97 97 172 172 172 F F F
104832024-04-17T17:17:05.018Z 10 200-219 96 96 96 171 171 171 F F F
104842024-04-17T17:17:05.018Z 11 220-239 96 96 96 171 171 171 F F F
104852024-04-17T17:17:05.018Z 12 240-259 90 90 90 161 161 161 F F F
104862024-04-17T17:17:05.018Z 13 260-279 97 97 97 172 172 172 F F F
104872024-04-17T17:17:05.018Z 14 280-299 95 95 95 168 168 168 F F F
104882024-04-17T17:17:05.018Z 15 300-319 96 96 96 171 171 171 F F F
104892024-04-17T17:17:05.018Z 16 320-339 96 96 96 171 171 171 F F F
104902024-04-17T17:17:05.018Z 17 340-359 93 93 93 165 165 165 F F F
104912024-04-17T17:17:05.018Z 18 360-379 92 92 92 163 163 163 F F F
104922024-04-17T17:17:05.018Z 19 380-399 94 94 94 167 167 167 F F F
104932024-04-17T17:17:05.019Z 20 400-419 97 97 97 172 172 172 F F F
104942024-04-17T17:17:05.019Z 21 420-439 95 95 95 168 168 168 F F F
104952024-04-17T17:17:05.019Z 22 440-459 94 94 94 167 167 167 F F F
104962024-04-17T17:17:05.019Z 23 460-479 94 94 94 167 167 167 F F F
104972024-04-17T17:17:05.019Z 24 480-499 97 97 97 172 172 172 F F F
104982024-04-17T17:17:05.019Z 25 500-519 97 97 97 172 172 172 F F F
104992024-04-17T17:17:05.019Z 26 520-539 92 92 92 164 164 164 F F F
105002024-04-17T17:17:05.019Z 27 540-559 96 96 96 171 171 171 F F F
105012024-04-17T17:17:05.019Z 28 560-579 95 95 95 168 168 168 F F F
105022024-04-17T17:17:05.019Z 29 580-599 94 94 94 167 167 167 F F F
105032024-04-17T17:17:05.019ZMax gen: 97, Max flush: 172
105042024-04-17T17:17:05.019Z
105052024-04-17T17:17:05.019ZBegin loop 49
105062024-04-17T17:17:05.019ZDownstairs to restart: 2
105072024-04-17T17:17:05.037Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.644809322Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563}
105082024-04-17T17:17:05.037Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.645110611Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563}
105092024-04-17T17:17:05.045Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.645151843Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563}
105102024-04-17T17:17:05.045Z{"msg":"Crucible runtime is spawned
105112024-04-17T17:17:05.045ZCrucible 0ee004ca-25d8-4201-a46d-3a54f29f20d0 has session id: e4002326-4744-4510-8320-6293c3928036","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.645191656Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"session_id":"e4002326-4744-4510-8320-6293c3928036"}
105122024-04-17T17:17:05.045Z{"msg":"Upstairs opts: Upstairs UUID: 0ee004ca-25d8-4201-a46d-3a54f29f20d0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.64526354Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"session_id":"e4002326-4744-4510-8320-6293c3928036"}
105132024-04-17T17:17:05.045Z{"msg":"Crucible stats registered with UUID: 0ee004ca-25d8-4201-a46d-3a54f29f20d0","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.645314883Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"session_id":"e4002326-4744-4510-8320-6293c3928036"}
105142024-04-17T17:17:05.045Z{"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.645338625Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563}
105152024-04-17T17:17:05.045Z{"msg":"0ee004ca-25d8-4201-a46d-3a54f29f20d0 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.645377257Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"session_id":"e4002326-4744-4510-8320-6293c3928036"}
105162024-04-17T17:17:05.045Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.645405758Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"client":"0","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105172024-04-17T17:17:05.045Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.645449751Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"client":"0","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105182024-04-17T17:17:05.045Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.645485983Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"client":"1","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105192024-04-17T17:17:05.045Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.645527226Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"client":"1","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105202024-04-17T17:17:05.045Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.645625422Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"client":"2","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105212024-04-17T17:17:05.045Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.645677145Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"client":"2","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105222024-04-17T17:17:05.045Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.645710917Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"":"io task","client":"2","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105232024-04-17T17:17:05.045Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.645770541Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"":"io task","client":"1","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105242024-04-17T17:17:05.045Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.645802193Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"":"io task","client":"0","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105252024-04-17T17:17:05.045Z{"msg":"ds_connection connected from Ok(127.0.0.1:54206)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.645847385Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"":"io task","client":"0","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105262024-04-17T17:17:05.046Z{"msg":"ds_connection connected from Ok(127.0.0.1:61123)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.645879657Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"":"io task","client":"2","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105272024-04-17T17:17:05.046Z{"msg":"ds_connection connected from Ok(127.0.0.1:38224)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.645936831Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"":"io task","client":"1","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105282024-04-17T17:17:05.046Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.646483554Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"client":"0","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105292024-04-17T17:17:05.046Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.646525346Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"client":"0","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105302024-04-17T17:17:05.046Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.646624932Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"client":"2","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105312024-04-17T17:17:05.046Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.646660134Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"client":"2","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105322024-04-17T17:17:05.046Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.646674795Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"client":"1","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105332024-04-17T17:17:05.046Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.646715448Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"client":"1","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105342024-04-17T17:17:05.046Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.647032197Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"client":"0","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105352024-04-17T17:17:05.046Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.647159795Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"client":"1","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105362024-04-17T17:17:05.046Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.647366557Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"client":"2","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105372024-04-17T17:17:05.046Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.647566629Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"client":"0","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105382024-04-17T17:17:05.046Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.647601852Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"client":"0","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105392024-04-17T17:17:05.046Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.647617583Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"session_id":"e4002326-4744-4510-8320-6293c3928036"}
105402024-04-17T17:17:05.046Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.647636344Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"client":"1","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105412024-04-17T17:17:05.046Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.647650094Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"client":"1","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105422024-04-17T17:17:05.046Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.647670656Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"session_id":"e4002326-4744-4510-8320-6293c3928036"}
105432024-04-17T17:17:05.046Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.647871978Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"client":"2","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105442024-04-17T17:17:05.046Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.64789814Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"client":"2","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105452024-04-17T17:17:05.046Z{"msg":"[0]R flush_numbers[0..12]: [165, 167, 167, 163, 169, 172, 171, 171, 171, 172, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.64791339Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"":"downstairs","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105462024-04-17T17:17:05.046Z{"msg":"[0]R generation[0..12]: [93, 94, 94, 92, 96, 97, 96, 96, 96, 97, 96, 96]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.647927411Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"":"downstairs","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105472024-04-17T17:17:05.046Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.647948783Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"":"downstairs","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105482024-04-17T17:17:05.046Z{"msg":"[1]R flush_numbers[0..12]: [165, 167, 167, 163, 169, 172, 171, 171, 171, 172, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.647963164Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"":"downstairs","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105492024-04-17T17:17:05.047Z{"msg":"[1]R generation[0..12]: [93, 94, 94, 92, 96, 97, 96, 96, 96, 97, 96, 96]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.647976834Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"":"downstairs","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105502024-04-17T17:17:05.047Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.647998926Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"":"downstairs","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105512024-04-17T17:17:05.047Z{"msg":"[2]R flush_numbers[0..12]: [165, 167, 167, 163, 169, 172, 171, 171, 171, 172, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.648013016Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"":"downstairs","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105522024-04-17T17:17:05.047Z{"msg":"[2]R generation[0..12]: [93, 94, 94, 92, 96, 97, 96, 96, 96, 97, 96, 96]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.648026537Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"":"downstairs","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105532024-04-17T17:17:05.047Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.648040318Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"":"downstairs","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105542024-04-17T17:17:05.047Z{"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.64806543Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"":"downstairs","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105552024-04-17T17:17:05.047Z{"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.648078951Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"":"downstairs","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105562024-04-17T17:17:05.047Z{"msg":"Next flush: 173","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.648092021Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"":"downstairs",Wait for a query_work_queue command to finish before sending IO
105572024-04-17T17:17:05.047Z"":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105582024-04-17T17:17:05.047Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.648116203Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"":"downstairs","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105592024-04-17T17:17:05.047Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.648129134Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"session_id":"e4002326-4744-4510-8320-6293c3928036"}
105602024-04-17T17:17:05.047Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.648140844Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"client":"0","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105612024-04-17T17:17:05.047Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.648160916Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"client":"1","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105622024-04-17T17:17:05.047Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.648174116Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"client":"2","":"downstairs","session_id":"e4002326-4744-4510-8320-6293c3928036"}
105632024-04-17T17:17:05.047Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.648186997Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"session_id":"e4002326-4744-4510-8320-6293c3928036"}
105642024-04-17T17:17:05.047Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.648198958Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"session_id":"e4002326-4744-4510-8320-6293c3928036"}
105652024-04-17T17:17:05.047Z{"msg":"0ee004ca-25d8-4201-a46d-3a54f29f20d0 is now active with session: e4002326-4744-4510-8320-6293c3928036","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.648211419Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"session_id":"e4002326-4744-4510-8320-6293c3928036"}
105662024-04-17T17:17:05.047Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.64823089Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563,"session_id":"e4002326-4744-4510-8320-6293c3928036"}
105672024-04-17T17:17:05.047Z{"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:03.648243261Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563}
105682024-04-17T17:17:05.047ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
105692024-04-17T17:17:05.047ZLoading write count information from file "/tmp/test_repair_verify.data"
105702024-04-17T17:17:05.047ZRun Repair workload
105712024-04-17T17:17:05.051Z01/30 Write block 287 len 36864 data: 4 5 7 8 7 6 6 5 4
105722024-04-17T17:17:05.052Z02/30 Write block 224 len 28672 data: 10 10 12 13 12 12 11
105732024-04-17T17:17:05.052Z03/30 Write block 123 len 16384 data: 7 7 8 8
105742024-04-17T17:17:05.052Z04/30 Read block 277 len 12288
105752024-04-17T17:17:05.052Z05/30 Read block 342 len 20480
105762024-04-17T17:17:05.052Z06/30 Write block 201 len 28672 data: 11 13 14 14 13 12 12
105772024-04-17T17:17:05.052Z07/30 Write block 248 len 12288 data: 7 6 7
105782024-04-17T17:17:05.052Z08/30 Write block 367 len 28672 data: 6 8 8 7 9 6 7
105792024-04-17T17:17:05.052Z09/30 Read block 446 len 16384
105802024-04-17T17:17:05.052Z10/30 Read block 97 len 40960
105812024-04-17T17:17:05.056Z11/30 Write block 247 len 8192 data: 9 8
105822024-04-17T17:17:05.056Z12/30 Read block 525 len 16384
105832024-04-17T17:17:05.056Z13/30 Read block 570 len 16384
105842024-04-17T17:17:05.056Z14/30 Write block 512 len 4096 data: 7
105852024-04-17T17:17:05.056Z15/30 Write block 592 len 8192 data: 8 6
105862024-04-17T17:17:05.056Z16/30 Write block 204 len 40960 data: 15 14 13 13 5 6 6 7 7 7
105872024-04-17T17:17:05.056Z17/30 Read block 237 len 24576
105882024-04-17T17:17:05.060Z18/30 Read block 582 len 8192
105892024-04-17T17:17:05.060Z19/30 Flush
105902024-04-17T17:17:05.064Z20/30 Write block 63 len 28672 data: 11 12 10 9 9 8 6
105912024-04-17T17:17:05.064Z21/30 Read block 18 len 8192
105922024-04-17T17:17:05.064Z22/30 Read block 295 len 40960
105932024-04-17T17:17:05.064Z23/30 Read block 242 len 16384
105942024-04-17T17:17:05.064Z24/30 Flush
105952024-04-17T17:17:05.070Z25/30 Write block 244 len 24576 data: 9 9 10 10 9 7
105962024-04-17T17:17:05.070Z26/30 Read block 69 len 40960
105972024-04-17T17:17:05.077Z27/30 Write block 380 len 36864 data: 6 5 6 6 6 6 5 6 5
105982024-04-17T17:17:05.078Z28/30 Write block 45 len 40960 data: 8 7 5 4 5 4 5 5 5 9
105992024-04-17T17:17:05.078Z29/30 Read block 74 len 4096
106002024-04-17T17:17:05.078Z30/30 Read block 80 len 28672
106012024-04-17T17:17:05.078Z----------------------------------------------------------------
106022024-04-17T17:17:05.078Z Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30
106032024-04-17T17:17:05.078ZStates: Active Active Active
106042024-04-17T17:17:05.078ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
106052024-04-17T17:17:05.078Z 1 Acked 1000 Write 9 Done Done Done false
106062024-04-17T17:17:05.078Z 2 Acked 1001 Write 7 Done Done Done false
106072024-04-17T17:17:05.078Z 3 Acked 1002 Write 4 Done Done Done false
106082024-04-17T17:17:05.078Z 4 Acked 1003 Read 3 Done Done Done false
106092024-04-17T17:17:05.079Z 5 Acked 1004 Read 5 Done Done Sent false
106102024-04-17T17:17:05.079Z 6 Acked 1005 Write 7 Done Done Sent false
106112024-04-17T17:17:05.079Z 7 Acked 1006 Write 3 Done Done Sent false
106122024-04-17T17:17:05.079Z 8 Acked 1007 Write 7 Done Done Sent false
106132024-04-17T17:17:05.079Z 9 Acked 1008 Read 4 Done Done Sent false
106142024-04-17T17:17:05.079Z 10 Acked 1009 Read 10 Done Done Sent false
106152024-04-17T17:17:05.079Z 11 Acked 1010 Write 2 Done Done Sent false
106162024-04-17T17:17:05.080Z 12 Acked 1011 Read 4 Done Done Sent false
106172024-04-17T17:17:05.080Z 13 Acked 1012 Read 4 Done Done Sent false
106182024-04-17T17:17:05.080Z 14 Acked 1013 Write 1 Done Done Sent false
106192024-04-17T17:17:05.080Z 15 Acked 1014 Write 2 Done Done Sent false
106202024-04-17T17:17:05.080Z 16 Acked 1015 Write 10 Done Done Sent false
106212024-04-17T17:17:05.081Z 17 Acked 1016 Read 6 Done Done Sent false
106222024-04-17T17:17:05.081Z 18 Acked 1017 Read 2 Done Done Sent false
106232024-04-17T17:17:05.081Z 19 Acked 1018 Flush 0 Done Done Sent false
106242024-04-17T17:17:05.081Z 20 Acked 1019 Write 7 Done Done Sent false
106252024-04-17T17:17:05.081Z 21 Acked 1020 Read 2 Done Done Sent false
106262024-04-17T17:17:05.082Z 22 Acked 1021 Read 10 Done Done Sent false
106272024-04-17T17:17:05.082Z 23 Acked 1022 Read 4 Done Done Sent false
106282024-04-17T17:17:05.082Z 24 Acked 1023 Flush 0 Done Done Sent false
106292024-04-17T17:17:05.082Z 25 Acked 1024 Write 6 Done Done Sent false
106302024-04-17T17:17:05.082Z 26 Acked 1025 Read 10 Done Done Sent false
106312024-04-17T17:17:05.082Z 27 Acked 1026 Write 9 Done Done Sent false
106322024-04-17T17:17:05.082Z 28 Acked 1027 Write 10 Done Done Sent false
106332024-04-17T17:17:05.082Z 29 Acked 1028 Read 1 Done Done Sent false
106342024-04-17T17:17:05.083Z 30 Acked 1029 Read 7 Done Sent Sent false
106352024-04-17T17:17:05.083Z STATES DS:0 DS:1 DS:2 TOTAL
106362024-04-17T17:17:05.083Z New 0 0 0 0
106372024-04-17T17:17:05.083Z Sent 0 1 26 27
106382024-04-17T17:17:05.083Z Done 30 29 4 63
106392024-04-17T17:17:05.083Z Skipped 0 0 0 0
106402024-04-17T17:17:05.084Z Error 0 0 0 0
106412024-04-17T17:17:05.084ZLast Flush: 1023 1023 0
106422024-04-17T17:17:05.084ZDownstairs last five completed:
106432024-04-17T17:17:05.084ZUpstairs last five completed: 30 29 28 27 26
106442024-04-17T17:17:05.084Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:17:03.68107694Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1563}
106452024-04-17T17:17:05.084ZWrote out file "/tmp/test_repair_verify.data"
106462024-04-17T17:17:05.098Z
106472024-04-17T17:17:05.099ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
106482024-04-17T17:17:05.113ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
106492024-04-17T17:17:05.113Z 0 000-019 93 93 93 165 165 165 F F F
106502024-04-17T17:17:05.113Z 1 020-039 94 94 94 167 167 167 F F F
106512024-04-17T17:17:05.113Z 2 040-059 94 94 94 167 167 167 T T F <---
106522024-04-17T17:17:05.113Z 3 060-079 98 98 92 174 174 163 F F F <---
106532024-04-17T17:17:05.114Z 4 080-099 96 96 96 169 169 169 F F F
106542024-04-17T17:17:05.114Z 5 100-119 97 97 97 172 172 172 F F F
106552024-04-17T17:17:05.114Z 6 120-139 98 98 96 173 173 171 F F T <---
106562024-04-17T17:17:05.114Z 7 140-159 96 96 96 171 171 171 F F F
106572024-04-17T17:17:05.114Z 8 160-179 96 96 96 171 171 171 F F F
106582024-04-17T17:17:05.114Z 9 180-199 97 97 97 172 172 172 F F F
106592024-04-17T17:17:05.114Z 10 200-219 98 98 96 173 173 171 F F F <---
106602024-04-17T17:17:05.114Z 11 220-239 98 98 96 173 173 171 F F T <---
106612024-04-17T17:17:05.115Z 12 240-259 98 98 90 173 173 161 T T F <---
106622024-04-17T17:17:05.115Z 13 260-279 97 97 97 172 172 172 F F F
106632024-04-17T17:17:05.115Z 14 280-299 98 98 95 173 173 168 F F T <---
106642024-04-17T17:17:05.115Z 15 300-319 96 96 96 171 171 171 F F F
106652024-04-17T17:17:05.115Z 16 320-339 96 96 96 171 171 171 F F F
106662024-04-17T17:17:05.115Z 17 340-359 93 93 93 165 165 165 F F F
106672024-04-17T17:17:05.115Z 18 360-379 98 98 92 173 173 163 F F F <---
106682024-04-17T17:17:05.115Z 19 380-399 94 94 94 167 167 167 T T F <---
106692024-04-17T17:17:05.115Z 20 400-419 97 97 97 172 172 172 F F F
106702024-04-17T17:17:05.115Z 21 420-439 95 95 95 168 168 168 F F F
106712024-04-17T17:17:05.116Z 22 440-459 94 94 94 167 167 167 F F F
106722024-04-17T17:17:05.116Z 23 460-479 94 94 94 167 167 167 F F F
106732024-04-17T17:17:05.116Z 24 480-499 97 97 97 172 172 172 F F F
106742024-04-17T17:17:05.116Z 25 500-519 98 98 97 173 173 172 F F F <---
106752024-04-17T17:17:05.116Z 26 520-539 92 92 92 164 164 164 F F F
106762024-04-17T17:17:05.116Z 27 540-559 96 96 96 171 171 171 F F F
106772024-04-17T17:17:05.116Z 28 560-579 95 95 95 168 168 168 F F F
106782024-04-17T17:17:05.116Z 29 580-599 98 98 94 173 173 167 F F F <---
106792024-04-17T17:17:05.117ZMax gen: 98, Max flush: 174
106802024-04-17T17:17:05.117ZError: Difference in extent metadata found!
106812024-04-17T17:17:05.117ZA Difference in extent metadata is expected here
106822024-04-17T17:17:05.117ZOn loop 49
106832024-04-17T17:17:05.117Z
106842024-04-17T17:17:05.117ZVerifying data now
106852024-04-17T17:17:19.617ZNo change after verify
106862024-04-17T17:17:19.617ZLoop: 49 Downstairs dump after verify (and repair):
106872024-04-17T17:17:19.632ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
106882024-04-17T17:17:19.632Z 0 000-019 93 93 93 165 165 165 F F F
106892024-04-17T17:17:19.637Z 1 020-039 94 94 94 167 167 167 F F F
106902024-04-17T17:17:19.637Z 2 040-059 99 99 99 175 175 175 F F F
106912024-04-17T17:17:19.637Z 3 060-079 98 98 98 174 174 174 F F F
106922024-04-17T17:17:19.637Z 4 080-099 96 96 96 169 169 169 F F F
106932024-04-17T17:17:19.637Z 5 100-119 97 97 97 172 172 172 F F F
106942024-04-17T17:17:19.637Z 6 120-139 98 98 98 173 173 173 F F F
106952024-04-17T17:17:19.637Z 7 140-159 96 96 96 171 171 171 F F F
106962024-04-17T17:17:19.637Z 8 160-179 96 96 96 171 171 171 F F F
106972024-04-17T17:17:19.637Z 9 180-199 97 97 97 172 172 172 F F F
106982024-04-17T17:17:19.637Z 10 200-219 98 98 98 173 173 173 F F F
106992024-04-17T17:17:19.637Z 11 220-239 98 98 98 173 173 173 F F F
107002024-04-17T17:17:19.637Z 12 240-259 99 99 99 175 175 175 F F F
107012024-04-17T17:17:19.637Z 13 260-279 97 97 97 172 172 172 F F F
107022024-04-17T17:17:19.637Z 14 280-299 98 98 98 173 173 173 F F F
107032024-04-17T17:17:19.637Z 15 300-319 96 96 96 171 171 171 F F F
107042024-04-17T17:17:19.637Z 16 320-339 96 96 96 171 171 171 F F F
107052024-04-17T17:17:19.637Z 17 340-359 93 93 93 165 165 165 F F F
107062024-04-17T17:17:19.637Z 18 360-379 98 98 98 173 173 173 F F F
107072024-04-17T17:17:19.637Z 19 380-399 99 99 99 175 175 175 F F F
107082024-04-17T17:17:19.637Z 20 400-419 97 97 97 172 172 172 F F F
107092024-04-17T17:17:19.638Z 21 420-439 95 95 95 168 168 168 F F F
107102024-04-17T17:17:19.638Z 22 440-459 94 94 94 167 167 167 F F F
107112024-04-17T17:17:19.638Z 23 460-479 94 94 94 167 167 167 F F F
107122024-04-17T17:17:19.638Z 24 480-499 97 97 97 172 172 172 F F F
107132024-04-17T17:17:19.638Z 25 500-519 98 98 98 173 173 173 F F F
107142024-04-17T17:17:19.638Z 26 520-539 92 92 92 164 164 164 F F F
107152024-04-17T17:17:19.638Z 27 540-559 96 96 96 171 171 171 F F F
107162024-04-17T17:17:19.638Z 28 560-579 95 95 95 168 168 168 F F F
107172024-04-17T17:17:19.638Z 29 580-599 98 98 98 173 173 173 F F F
107182024-04-17T17:17:19.638ZMax gen: 99, Max flush: 175
107192024-04-17T17:17:19.638Z
107202024-04-17T17:17:19.638ZBegin loop 50
107212024-04-17T17:17:19.638ZDownstairs to restart: 2
107222024-04-17T17:17:19.657Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.265420716Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571}
107232024-04-17T17:17:19.666Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.265699533Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571}
107242024-04-17T17:17:19.666Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.265742496Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571}
107252024-04-17T17:17:19.666Z{"msg":"Crucible baef2e8d-7ba4-42f2-b54c-2c75bc856450 has session id: 5414fa28-1ba9-400a-8a49-14fa79600b03","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.265771558Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107262024-04-17T17:17:19.666Z{"msg":"Upstairs opts: Upstairs UUID: baef2e8d-7ba4-42f2-b54c-2c75bc856450, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.265822911Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107272024-04-17T17:17:19.666Z{"msg":"Crucible stats registered with UUID: baef2e8d-7ba4-42f2-b54c-2c75bc856450","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.265871414Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107282024-04-17T17:17:19.666ZCrucible runtime is spawned
107292024-04-17T17:17:19.666Z{"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.266062545Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571}
107302024-04-17T17:17:19.666Z{"msg":"baef2e8d-7ba4-42f2-b54c-2c75bc856450 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.26613919Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107312024-04-17T17:17:19.666Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.266165742Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"client":"0","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107322024-04-17T17:17:19.667Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.266200994Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"client":"0","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107332024-04-17T17:17:19.667Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.266227305Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"client":"1","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107342024-04-17T17:17:19.667Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.266252107Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"client":"1","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107352024-04-17T17:17:19.667Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.266284119Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"client":"2","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107362024-04-17T17:17:19.667Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.26630803Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"client":"2","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107372024-04-17T17:17:19.667Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.266344412Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107382024-04-17T17:17:19.667Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.266441138Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107392024-04-17T17:17:19.667Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.266493381Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107402024-04-17T17:17:19.667Z{"msg":"ds_connection connected from Ok(127.0.0.1:51488)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.266523753Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107412024-04-17T17:17:19.667Z{"msg":"ds_connection connected from Ok(127.0.0.1:63080)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.266552015Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107422024-04-17T17:17:19.667Z{"msg":"ds_connection connected from Ok(127.0.0.1:44898)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.266596828Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107432024-04-17T17:17:19.667Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.267225536Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"client":"1","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107442024-04-17T17:17:19.667Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.267261758Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"client":"1","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107452024-04-17T17:17:19.667Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.267372935Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"client":"2","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107462024-04-17T17:17:19.667Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.267428178Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"client":"2","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107472024-04-17T17:17:19.667Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.267453549Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"client":"0","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107482024-04-17T17:17:19.667Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.267469681Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"client":"0","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107492024-04-17T17:17:19.667Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.267666342Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"client":"1","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107502024-04-17T17:17:19.667Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.268009763Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"client":"2","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107512024-04-17T17:17:19.667Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.268069277Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"client":"0","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107522024-04-17T17:17:19.667Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.268192114Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"client":"1","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107532024-04-17T17:17:19.667Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.268240547Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"client":"1","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107542024-04-17T17:17:19.667Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.268265829Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107552024-04-17T17:17:19.668Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.268481322Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"client":"0","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107562024-04-17T17:17:19.668Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.268519394Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"client":"0","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107572024-04-17T17:17:19.668Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.268535575Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107582024-04-17T17:17:19.668Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.268556996Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"client":"2","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107592024-04-17T17:17:19.668Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.268572577Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"client":"2","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107602024-04-17T17:17:19.668Z{"msg":"[0]R flush_numbers[0..12]: [165, 167, 175, 174, 169, 172, 173, 171, 171, 172, 173, 173]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.26861376Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"":"downstairs","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107612024-04-17T17:17:19.668Z{"msg":"[0]R generation[0..12]: [93, 94, 99, 98, 96, 97, 98, 96, 96, 97, 98, 98]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.268638451Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"":"downstairs","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107622024-04-17T17:17:19.668Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.268653172Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"":"downstairs","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107632024-04-17T17:17:19.668Z{"msg":"[1]R flush_numbers[0..12]: [165, 167, 175, 174, 169, 172, 173, 171, 171, 172, 173, 173]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.268667013Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"":"downstairs","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107642024-04-17T17:17:19.668Z{"msg":"[1]R generation[0..12]: [93, 94, 99, 98, 96, 97, 98, 96, 96, 97, 98, 98]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.268690104Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"":"downstairs","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107652024-04-17T17:17:19.668Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.268714946Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"":"downstairs","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107662024-04-17T17:17:19.668Z{"msg":"[2]R flush_numbers[0..12]: [165, 167, 175, 174, 169, 172, 173, 171, 171, 172, 173, 173]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.268744698Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"":"downstairs","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107672024-04-17T17:17:19.668Z{"msg":"[2]R generation[0..12]: [93, 94, 99, 98, 96, 97, 98, 96, 96, 97, 98, 98]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
107682024-04-17T17:17:19.668Z,"time":"2024-04-17T17:17:18.268770209Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"":"downstairs","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107692024-04-17T17:17:19.668Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.268804891Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"":"downstairs","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107702024-04-17T17:17:19.668Z{"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.268835833Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"":"downstairs","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107712024-04-17T17:17:19.668Z{"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.268861185Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"":"downstairs","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107722024-04-17T17:17:19.668Z{"msg":"Next flush: 176","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.268890347Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"":"downstairs","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107732024-04-17T17:17:19.668Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.268913518Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"":"downstairs","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107742024-04-17T17:17:19.668Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.2689536Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107752024-04-17T17:17:19.668Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.268973652Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"client":"0","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107762024-04-17T17:17:19.668Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.269003674Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"client":"1","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107772024-04-17T17:17:19.668Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.269039446Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"client":"2","":"downstairs","session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107782024-04-17T17:17:19.669ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
107792024-04-17T17:17:19.669Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.269070127Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107802024-04-17T17:17:19.669Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.269092569Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107812024-04-17T17:17:19.669Z{"msg":"baef2e8d-7ba4-42f2-b54c-2c75bc856450 is now active with session: 5414fa28-1ba9-400a-8a49-14fa79600b03","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.26910907Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107822024-04-17T17:17:19.669Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.269130031Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571,"session_id":"5414fa28-1ba9-400a-8a49-14fa79600b03"}
107832024-04-17T17:17:19.669Z{"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:18.269148132Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571}
107842024-04-17T17:17:19.673ZLoading write count information from file "/tmp/test_repair_verify.data"
107852024-04-17T17:17:19.673ZRun Repair workload
107862024-04-17T17:17:19.673Z01/30 Write block 470 len 40960 data: 8 9 7 7 8 7 8 12 14 13
107872024-04-17T17:17:19.673Z02/30 Flush
107882024-04-17T17:17:19.673Z03/30 Write block 432 len 24576 data: 5 8 8 9 9 9
107892024-04-17T17:17:19.673Z04/30 Read block 438 len 8192
107902024-04-17T17:17:19.673Z05/30 Write block 207 len 40960 data: 14 6 7 7 8 8 8 8 10 8
107912024-04-17T17:17:19.673Z06/30 Flush
107922024-04-17T17:17:19.673Z07/30 Write block 293 len 8192 data: 7 6
107932024-04-17T17:17:19.673Z08/30 Read block 493 len 28672
107942024-04-17T17:17:19.673Z09/30 Read block 351 len 32768
107952024-04-17T17:17:19.677Z10/30 Flush
107962024-04-17T17:17:19.677Z11/30 Write block 188 len 12288 data: 8 8 6
107972024-04-17T17:17:19.677Z12/30 Flush
107982024-04-17T17:17:19.680Z13/30 Write block 296 len 40960 data: 3 4 4 5 5 6 5 6 6 6
107992024-04-17T17:17:19.680Z14/30 Read block 388 len 12288
108002024-04-17T17:17:19.683Z15/30 Flush
108012024-04-17T17:17:19.683Z16/30 Flush
108022024-04-17T17:17:19.687Z17/30 Write block 52 len 36864 data: 6 6 10 8 8 10 9 9 11
108032024-04-17T17:17:19.687Z18/30 Write block 159 len 24576 data: 9 8 9 8 10 7
108042024-04-17T17:17:19.687Z19/30 Read block 418 len 40960
108052024-04-17T17:17:19.687Z20/30 Read block 203 len 12288
108062024-04-17T17:17:19.690Z21/30 Flush
108072024-04-17T17:17:19.690Z22/30 Write block 3 len 12288 data: 7 8 8
108082024-04-17T17:17:19.690Z23/30 Read block 476 len 20480
108092024-04-17T17:17:19.694Z24/30 Flush
108102024-04-17T17:17:19.694Z25/30 Write block 88 len 28672 data: 8 11 12 13 13 12 10
108112024-04-17T17:17:19.697Z26/30 Read block 594 len 8192
108122024-04-17T17:17:19.697Z27/30 Write block 530 len 4096 data: 9
108132024-04-17T17:17:19.697Z28/30 Write block 385 len 40960 data: 7 6 7 6 7 7 8 9 7 8
108142024-04-17T17:17:19.701Z29/30 Write block 423 len 20480 data: 6 7 7 7 7
108152024-04-17T17:17:19.701Z30/30 Read block 18 len 24576
108162024-04-17T17:17:19.701Z----------------------------------------------------------------
108172024-04-17T17:17:19.701Z Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:28
108182024-04-17T17:17:19.701ZStates: Active Active Active
108192024-04-17T17:17:19.701ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
108202024-04-17T17:17:19.701Z 3 Acked 1002 Write 6 Done Done Done false
108212024-04-17T17:17:19.701Z 4 Acked 1003 Read 2 Done Done Done false
108222024-04-17T17:17:19.701Z 5 Acked 1004 Write 10 Done Done Sent false
108232024-04-17T17:17:19.701Z 6 Acked 1005 Flush 0 Done Done Sent false
108242024-04-17T17:17:19.701Z 7 Acked 1006 Write 2 Done Done Sent false
108252024-04-17T17:17:19.701Z 8 Acked 1007 Read 7 Done Done Sent false
108262024-04-17T17:17:19.701Z 9 Acked 1008 Read 8 Done Done Sent false
108272024-04-17T17:17:19.701Z 10 Acked 1009 Flush 0 Done Done Sent false
108282024-04-17T17:17:19.701Z 11 Acked 1010 Write 3 Done Done Sent false
108292024-04-17T17:17:19.701Z 12 Acked 1011 Flush 0 Done Done Sent false
108302024-04-17T17:17:19.701Z 13 Acked 1012 Write 10 Done Done Sent false
108312024-04-17T17:17:19.701Z 14 Acked 1013 Read 3 Done Done Sent false
108322024-04-17T17:17:19.701Z 15 Acked 1014 Flush 0 Done Done Sent false
108332024-04-17T17:17:19.702Z 16 Acked 1015 Flush 0 Done Done Sent false
108342024-04-17T17:17:19.702Z 17 Acked 1016 Write 9 Done Done Sent false
108352024-04-17T17:17:19.702Z 18 Acked 1017 Write 6 Done Done Sent false
108362024-04-17T17:17:19.702Z 19 Acked 1018 Read 10 Done Done Sent false
108372024-04-17T17:17:19.702Z 20 Acked 1019 Read 3 Done Done Sent false
108382024-04-17T17:17:19.702Z 21 Acked 1020 Flush 0 Done Done Sent false
108392024-04-17T17:17:19.702Z 22 Acked 1021 Write 3 Done Done Sent false
108402024-04-17T17:17:19.702Z 23 Acked 1022 Read 5 Done Done Sent false
108412024-04-17T17:17:19.702Z 24 Acked 1023 Flush 0 Done Done Sent false
108422024-04-17T17:17:19.702Z 25 Acked 1024 Write 7 Done Done Sent false
108432024-04-17T17:17:19.702Z 26 Acked 1025 Read 2 Done Done Sent false
108442024-04-17T17:17:19.702Z 27 Acked 1026 Write 1 Done Done Sent false
108452024-04-17T17:17:19.702Z 28 Acked 1027 Write 10 Done Done Sent false
108462024-04-17T17:17:19.702Z 29 Acked 1028 Write 5 Done Done Sent false
108472024-04-17T17:17:19.702Z 30 Acked 1029 Read 6 Done Sent Sent false
108482024-04-17T17:17:19.702Z STATES DS:0 DS:1 DS:2 TOTAL
108492024-04-17T17:17:19.702Z New 0 0 0 0
108502024-04-17T17:17:19.702Z Sent 0 1 26 27
108512024-04-17T17:17:19.702Z Done 28 27 2 57
108522024-04-17T17:17:19.702Z Skipped 0 0 0 0
108532024-04-17T17:17:19.702Z Error 0 0 0 0
108542024-04-17T17:17:19.702ZLast Flush: 1023 1023 1001
108552024-04-17T17:17:19.703ZDownstairs last five completed: 1001 1000
108562024-04-17T17:17:19.703ZUpstairs last five completed: 30 29 28 27 26
108572024-04-17T17:17:19.703Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:17:18.307645676Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1571}
108582024-04-17T17:17:19.703ZWrote out file "/tmp/test_repair_verify.data"
108592024-04-17T17:17:19.706Z
108602024-04-17T17:17:19.706ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
108612024-04-17T17:17:19.724ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
108622024-04-17T17:17:19.724Z 0 000-019 100 100 93 183 183 165 F F F <---
108632024-04-17T17:17:19.724Z 1 020-039 94 94 94 167 167 167 F F F
108642024-04-17T17:17:19.724Z 2 040-059 100 100 99 182 182 175 F F F <---
108652024-04-17T17:17:19.724Z 3 060-079 100 100 98 182 182 174 F F F <---
108662024-04-17T17:17:19.724Z 4 080-099 96 96 96 169 169 169 T T F <---
108672024-04-17T17:17:19.724Z 5 100-119 97 97 97 172 172 172 F F F
108682024-04-17T17:17:19.724Z 6 120-139 98 98 98 173 173 173 F F F
108692024-04-17T17:17:19.728Z 7 140-159 100 100 96 182 182 171 F F F <---
108702024-04-17T17:17:19.728Z 8 160-179 100 100 96 182 182 171 F F F <---
108712024-04-17T17:17:19.728Z 9 180-199 100 100 97 179 179 172 F F F <---
108722024-04-17T17:17:19.728Z 10 200-219 100 100 98 177 177 173 F F F <---
108732024-04-17T17:17:19.728Z 11 220-239 98 98 98 173 173 173 F F F
108742024-04-17T17:17:19.728Z 12 240-259 99 99 99 175 175 175 F F F
108752024-04-17T17:17:19.728Z 13 260-279 97 97 97 172 172 172 F F F
108762024-04-17T17:17:19.728Z 14 280-299 100 100 98 180 180 173 F F F <---
108772024-04-17T17:17:19.728Z 15 300-319 100 100 96 180 180 171 F F F <---
108782024-04-17T17:17:19.728Z 16 320-339 96 96 96 171 171 171 F F F
108792024-04-17T17:17:19.728Z 17 340-359 93 93 93 165 165 165 F F F
108802024-04-17T17:17:19.728Z 18 360-379 98 98 98 173 173 173 F F F
108812024-04-17T17:17:19.728Z 19 380-399 99 99 99 175 175 175 T T F <---
108822024-04-17T17:17:19.728Z 20 400-419 97 97 97 172 172 172 F F F
108832024-04-17T17:17:19.728Z 21 420-439 100 100 95 177 177 168 T T T <---
108842024-04-17T17:17:19.728Z 22 440-459 94 94 94 167 167 167 F F F
108852024-04-17T17:17:19.728Z 23 460-479 100 100 100 176 176 176 F F F
108862024-04-17T17:17:19.729Z 24 480-499 97 97 97 172 172 172 F F F
108872024-04-17T17:17:19.729Z 25 500-519 98 98 98 173 173 173 F F F
108882024-04-17T17:17:19.729Z 26 520-539 92 92 92 164 164 164 T T F <---
108892024-04-17T17:17:19.729Z 27 540-559 96 96 96 171 171 171 F F F
108902024-04-17T17:17:19.729Z 28 560-579 95 95 95 168 168 168 F F F
108912024-04-17T17:17:19.729Z 29 580-599 98 98 98 173 173 173 F F F
108922024-04-17T17:17:19.729ZMax gen: 100, Max flush: 183
108932024-04-17T17:17:19.729ZError: Difference in extent metadata found!
108942024-04-17T17:17:19.729ZA Difference in extent metadata is expected here
108952024-04-17T17:17:19.729ZOn loop 50
108962024-04-17T17:17:19.729Z
108972024-04-17T17:17:19.733ZVerifying data now
108982024-04-17T17:17:24.334ZNo change after verify
108992024-04-17T17:17:24.334ZLoop: 50 Downstairs dump after verify (and repair):
109002024-04-17T17:17:24.351ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
109012024-04-17T17:17:24.351Z 0 000-019 100 100 100 183 183 183 F F F
109022024-04-17T17:17:24.357Z 1 020-039 94 94 94 167 167 167 F F F
109032024-04-17T17:17:24.357Z 2 040-059 100 100 100 182 182 182 F F F
109042024-04-17T17:17:24.357Z 3 060-079 100 100 100 182 182 182 F F F
109052024-04-17T17:17:24.357Z 4 080-099 101 101 101 184 184 184 F F F
109062024-04-17T17:17:24.357Z 5 100-119 97 97 97 172 172 172 F F F
109072024-04-17T17:17:24.357Z 6 120-139 98 98 98 173 173 173 F F F
109082024-04-17T17:17:24.357Z 7 140-159 100 100 100 182 182 182 F F F
109092024-04-17T17:17:24.357Z 8 160-179 100 100 100 182 182 182 F F F
109102024-04-17T17:17:24.357Z 9 180-199 100 100 100 179 179 179 F F F
109112024-04-17T17:17:24.357Z 10 200-219 100 100 100 177 177 177 F F F
109122024-04-17T17:17:24.357Z 11 220-239 98 98 98 173 173 173 F F F
109132024-04-17T17:17:24.357Z 12 240-259 99 99 99 175 175 175 F F F
109142024-04-17T17:17:24.357Z 13 260-279 97 97 97 172 172 172 F F F
109152024-04-17T17:17:24.358Z 14 280-299 100 100 100 180 180 180 F F F
109162024-04-17T17:17:24.358Z 15 300-319 100 100 100 180 180 180 F F F
109172024-04-17T17:17:24.358Z 16 320-339 96 96 96 171 171 171 F F F
109182024-04-17T17:17:24.358Z 17 340-359 93 93 93 165 165 165 F F F
109192024-04-17T17:17:24.358Z 18 360-379 98 98 98 173 173 173 F F F
109202024-04-17T17:17:24.358Z 19 380-399 101 101 101 184 184 184 F F F
109212024-04-17T17:17:24.358Z 20 400-419 97 97 97 172 172 172 F F F
109222024-04-17T17:17:24.358Z 21 420-439 101 101 101 184 184 184 F F F
109232024-04-17T17:17:24.358Z 22 440-459 94 94 94 167 167 167 F F F
109242024-04-17T17:17:24.358Z 23 460-479 100 100 100 176 176 176 F F F
109252024-04-17T17:17:24.358Z 24 480-499 97 97 97 172 172 172 F F F
109262024-04-17T17:17:24.358Z 25 500-519 98 98 98 173 173 173 F F F
109272024-04-17T17:17:24.358Z 26 520-539 101 101 101 184 184 184 F F F
109282024-04-17T17:17:24.358Z 27 540-559 96 96 96 171 171 171 F F F
109292024-04-17T17:17:24.358Z 28 560-579 95 95 95 168 168 168 F F F
109302024-04-17T17:17:24.358Z 29 580-599 98 98 98 173 173 173 F F F
109312024-04-17T17:17:24.358ZMax gen: 101, Max flush: 184
109322024-04-17T17:17:24.358Z
109332024-04-17T17:17:24.358ZBegin loop 51
109342024-04-17T17:17:24.358ZDownstairs to restart: 1
109352024-04-17T17:17:24.371Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.979436001Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579}
109362024-04-17T17:17:24.371Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.979658605Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579}
109372024-04-17T17:17:24.371Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.979676426Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579}
109382024-04-17T17:17:24.371Z{"msg":"Crucible 5dfffb46-dd58-403f-84e3-3e426781e3a7 has session id: d7f097fa-679e-41b8-a1bf-04509680607b","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.979697947Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109392024-04-17T17:17:24.383Z{"msg":"Upstairs opts: Upstairs UUID: 5dfffb46-dd58-403f-84e3-3e426781e3a7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.979799753Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109402024-04-17T17:17:24.383Z{"msg":"Crucible stats registered with UUID: 5dfffb46-dd58-403f-84e3-3e426781e3a7","v":0,"name":"crucible","level":30Crucible runtime is spawned
109412024-04-17T17:17:24.383Z,"time":"2024-04-17T17:17:22.979849056Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109422024-04-17T17:17:24.383Z{"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.979988955Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579}
109432024-04-17T17:17:24.383Z{"msg":"5dfffb46-dd58-403f-84e3-3e426781e3a7 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.980040708Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109442024-04-17T17:17:24.383Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.980063809Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"client":"0","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109452024-04-17T17:17:24.383Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.98007892Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"client":"0","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109462024-04-17T17:17:24.383Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.980104052Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"client":"1","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109472024-04-17T17:17:24.383Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.980129553Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"client":"1","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109482024-04-17T17:17:24.383Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.980166576Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"client":"2","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109492024-04-17T17:17:24.383Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.980220339Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"client":"2","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109502024-04-17T17:17:24.383Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.980279152Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"":"io task","client":"0","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109512024-04-17T17:17:24.383Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.980313874Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"":"io task","client":"2","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109522024-04-17T17:17:24.383Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.980338886Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"":"io task","client":"1","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109532024-04-17T17:17:24.383Z{"msg":"ds_connection connected from Ok(127.0.0.1:49068)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.980447802Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"":"io task","client":"0","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109542024-04-17T17:17:24.383Z{"msg":"ds_connection connected from Ok(127.0.0.1:47349)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.980497636Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"":"io task","client":"1","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109552024-04-17T17:17:24.384Z{"msg":"ds_connection connected from Ok(127.0.0.1:54426)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.980530727Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"":"io task","client":"2","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109562024-04-17T17:17:24.384Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.981199178Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"client":"2","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109572024-04-17T17:17:24.384Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.98123786Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"client":"2","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109582024-04-17T17:17:24.384Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.981263872Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"client":"0","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109592024-04-17T17:17:24.384Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.981289024Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"client":"0","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109602024-04-17T17:17:24.384Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.981744561Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"client":"1","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109612024-04-17T17:17:24.384Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.981781323Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"client":"1","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109622024-04-17T17:17:24.384Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.981910111Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"client":"2","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109632024-04-17T17:17:24.384Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.981937553Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"client":"0","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109642024-04-17T17:17:24.384Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.982265983Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"client":"1","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109652024-04-17T17:17:24.384Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.982462765Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"client":"0","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109662024-04-17T17:17:24.384Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.982509627Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"client":"0","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109672024-04-17T17:17:24.384Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.982564481Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109682024-04-17T17:17:24.384Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.982597753Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"client":"2","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109692024-04-17T17:17:24.384Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.982621944Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"client":"2","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109702024-04-17T17:17:24.384Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.982655286Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109712024-04-17T17:17:24.384Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.982755372Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"client":"1","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109722024-04-17T17:17:24.384Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.982816196Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"client":"1","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109732024-04-17T17:17:24.384Z{"msg":"[0]R flush_numbers[0..12]: [183, 167, 182, 182, 184, 172, 173, 182, 182, 179, 177, 173]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.982850558Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"":"downstairs","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109742024-04-17T17:17:24.384Z{"msg":"[0]R generation[0..12]: [100, 94, 100, 100, 101, 97, 98, 100, 100, 100, 100, 98]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.982867009Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"":"downstairs","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109752024-04-17T17:17:24.384Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.98288871Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"":"downstairs","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109762024-04-17T17:17:24.384Z{"msg":"[1]R flush_numbers[0..12]: [183, 167, 182, 182, 184, 172, 173, 182, 182, 179, 177, 173]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.982908392Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"":"downstairs","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109772024-04-17T17:17:24.385ZWait for a query_work_queue command to finish before sending IO
109782024-04-17T17:17:24.385Z{"msg":"[1]R generation[0..12]: [100, 94, 100, 100, 101, 97, 98, 100, 100, 100, 100, 98]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.982947474Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"":"downstairs","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109792024-04-17T17:17:24.385Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.982981846Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"":"downstairs","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109802024-04-17T17:17:24.385Z{"msg":"[2]R flush_numbers[0..12]: [183, 167, 182, 182, 184, 172, 173, 182, 182, 179, 177, 173]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.983011038Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"":"downstairs","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109812024-04-17T17:17:24.385Z{"msg":"[2]R generation[0..12]: [100, 94, 100, 100, 101, 97, 98, 100, 100, 100, 100, 98]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.98304708Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"":"downstairs","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109822024-04-17T17:17:24.385Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.983079102Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"":"downstairs","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109832024-04-17T17:17:24.385Z{"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.983113914Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"":"downstairs","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109842024-04-17T17:17:24.385Z{"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.983142546Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"":"downstairs","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109852024-04-17T17:17:24.385Z{"msg":"Next flush: 185","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.983161487Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"":"downstairs","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109862024-04-17T17:17:24.385Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.983195619Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"":"downstairs","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109872024-04-17T17:17:24.385Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.983227351Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109882024-04-17T17:17:24.385Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.983241312Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"client":"0","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109892024-04-17T17:17:24.385Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.983268564Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"client":"1","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109902024-04-17T17:17:24.385Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.983303766Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"client":"2","":"downstairs","session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109912024-04-17T17:17:24.385ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
109922024-04-17T17:17:24.385Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.983340208Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109932024-04-17T17:17:24.385Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.98337261Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109942024-04-17T17:17:24.385Z{"msg":"5dfffb46-dd58-403f-84e3-3e426781e3a7 is now active with session: d7f097fa-679e-41b8-a1bf-04509680607b","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.983406792Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109952024-04-17T17:17:24.385Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.983438274Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579,"session_id":"d7f097fa-679e-41b8-a1bf-04509680607b"}
109962024-04-17T17:17:24.385Z{"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:22.983469116Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579}
109972024-04-17T17:17:24.385ZLoading write count information from file "/tmp/test_repair_verify.data"
109982024-04-17T17:17:24.385ZRun Repair workload
109992024-04-17T17:17:24.389Z01/30 Write block 46 len 32768 data: 8 6 5 6 5 6 7 7
110002024-04-17T17:17:24.389Z02/30 Read block 310 len 28672
110012024-04-17T17:17:24.389Z03/30 Write block 524 len 12288 data: 5 6 7
110022024-04-17T17:17:24.390Z04/30 Read block 298 len 32768
110032024-04-17T17:17:24.390Z05/30 Write block 54 len 32768 data: 11 9 9 11 10 10 12 10
110042024-04-17T17:17:24.390Z06/30 Write block 475 len 20480 data: 8 9 13 15 14
110052024-04-17T17:17:24.390Z07/30 Write block 572 len 24576 data: 7 8 10 11 10 9
110062024-04-17T17:17:24.390Z08/30 Write block 74 len 40960 data: 5 6 7 7 5 4 5 5 7 10
110072024-04-17T17:17:24.390Z09/30 Read block 393 len 36864
110082024-04-17T17:17:24.390Z10/30 Read block 460 len 16384
110092024-04-17T17:17:24.390Z11/30 Read block 157 len 28672
110102024-04-17T17:17:24.390Z12/30 Write block 5 len 12288 data: 9 7 6
110112024-04-17T17:17:24.390Z13/30 Read block 139 len 36864
110122024-04-17T17:17:24.390Z14/30 Read block 72 len 24576
110132024-04-17T17:17:24.394Z15/30 Read block 236 len 4096
110142024-04-17T17:17:24.394Z16/30 Read block 465 len 24576
110152024-04-17T17:17:24.394Z17/30 Read block 544 len 12288
110162024-04-17T17:17:24.394Z18/30 Write block 304 len 24576 data: 7 7 7 6 7 6
110172024-04-17T17:17:24.394Z19/30 Flush
110182024-04-17T17:17:24.397Z20/30 Write block 503 len 32768 data: 9 10 8 7 8 8 7 6
110192024-04-17T17:17:24.398Z21/30 Write block 526 len 16384 data: 8 8 8 9
110202024-04-17T17:17:24.402Z22/30 Write block 385 len 36864 data: 8 7 8 7 8 8 9 10 8
110212024-04-17T17:17:24.402Z23/30 Read block 509 len 24576
110222024-04-17T17:17:24.402Z24/30 Write block 261 len 36864 data: 10 9 9 8 6 6 6 6 9
110232024-04-17T17:17:24.402Z25/30 Write block 305 len 4096 data: 8
110242024-04-17T17:17:24.406Z26/30 Write block 546 len 40960 data: 7 8 11 12 10 10 11 9 9 9
110252024-04-17T17:17:24.406Z27/30 Read block 17 len 16384
110262024-04-17T17:17:24.406Z28/30 Read block 141 len 28672
110272024-04-17T17:17:24.406Z29/30 Write block 564 len 8192 data: 10 7
110282024-04-17T17:17:24.411Z30/30 Write block 539 len 32768 data: 8 8 9 10 11 10 8 8
110292024-04-17T17:17:24.411Z----------------------------------------------------------------
110302024-04-17T17:17:24.411Z Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:30
110312024-04-17T17:17:24.411ZStates: Active Active Active
110322024-04-17T17:17:24.411ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
110332024-04-17T17:17:24.411Z 1 Acked 1000 Write 8 Done Done Done false
110342024-04-17T17:17:24.411Z 2 Acked 1001 Read 7 Done Done Done false
110352024-04-17T17:17:24.411Z 3 Acked 1002 Write 3 Done Done Done false
110362024-04-17T17:17:24.411Z 4 Acked 1003 Read 8 Done Done Done false
110372024-04-17T17:17:24.411Z 5 Acked 1004 Write 8 Done Done Done false
110382024-04-17T17:17:24.411Z 6 Acked 1005 Write 5 Done Done Done false
110392024-04-17T17:17:24.411Z 7 Acked 1006 Write 6 Done Done Done false
110402024-04-17T17:17:24.411Z 8 Acked 1007 Write 10 Done Done Done false
110412024-04-17T17:17:24.411Z 9 Acked 1008 Read 9 Done Done Done false
110422024-04-17T17:17:24.411Z 10 Acked 1009 Read 4 Done Done Done false
110432024-04-17T17:17:24.411Z 11 Acked 1010 Read 7 Done Done Done false
110442024-04-17T17:17:24.411Z 12 Acked 1011 Write 3 Done Sent Done false
110452024-04-17T17:17:24.411Z 13 Acked 1012 Read 9 Done Sent Done false
110462024-04-17T17:17:24.412Z 14 Acked 1013 Read 6 Done Sent Done false
110472024-04-17T17:17:24.412Z 15 Acked 1014 Read 1 Done Sent Done false
110482024-04-17T17:17:24.412Z 16 Acked 1015 Read 6 Done Sent Done false
110492024-04-17T17:17:24.412Z 17 Acked 1016 Read 3 Done Sent Done false
110502024-04-17T17:17:24.412Z 18 Acked 1017 Write 6 Done Sent Done false
110512024-04-17T17:17:24.412Z 19 Acked 1018 Flush 0 Done Sent Done false
110522024-04-17T17:17:24.412Z 20 Acked 1019 Write 8 Done Sent Done false
110532024-04-17T17:17:24.412Z 21 Acked 1020 Write 4 Done Sent Done false
110542024-04-17T17:17:24.412Z 22 Acked 1021 Write 9 Done Sent Done false
110552024-04-17T17:17:24.412Z 23 Acked 1022 Read 6 Done Sent Done false
110562024-04-17T17:17:24.412Z 24 Acked 1023 Write 9 Done Sent Done false
110572024-04-17T17:17:24.412Z 25 Acked 1024 Write 1 Done Sent Done false
110582024-04-17T17:17:24.412Z 26 Acked 1025 Write 10 Done Sent Done false
110592024-04-17T17:17:24.412Z 27 Acked 1026 Read 4 Done Sent Done false
110602024-04-17T17:17:24.412Z 28 Acked 1027 Read 7 Done Sent Done false
110612024-04-17T17:17:24.412Z 29 Acked 1028 Write 2 Done Sent Done false
110622024-04-17T17:17:24.412Z 30 Acked 1029 Write 8 Sent Sent Sent false
110632024-04-17T17:17:24.412Z STATES DS:0 DS:1 DS:2 TOTAL
110642024-04-17T17:17:24.412Z New 0 0 0 0
110652024-04-17T17:17:24.412Z Sent 1 19 1 21
110662024-04-17T17:17:24.412Z Done 29 11 29 69
110672024-04-17T17:17:24.412Z Skipped 0 0 0 0
110682024-04-17T17:17:24.412Z Error 0 0 0 0
110692024-04-17T17:17:24.413ZLast Flush: 1018 0 1018
110702024-04-17T17:17:24.413ZDownstairs last five completed:
110712024-04-17T17:17:24.413ZUpstairs last five completed: 30 29 28 27 26
110722024-04-17T17:17:24.413Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:17:23.015908532Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1579}
110732024-04-17T17:17:24.413ZWrote out file "/tmp/test_repair_verify.data"
110742024-04-17T17:17:24.417Z
110752024-04-17T17:17:24.417ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
110762024-04-17T17:17:24.433ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
110772024-04-17T17:17:24.433Z 0 000-019 102 100 102 185 183 185 F F F <---
110782024-04-17T17:17:24.433Z 1 020-039 94 94 94 167 167 167 F F F
110792024-04-17T17:17:24.433Z 2 040-059 102 100 102 185 182 185 F T F <---
110802024-04-17T17:17:24.433Z 3 060-079 102 100 102 185 182 185 F T F <---
110812024-04-17T17:17:24.433Z 4 080-099 102 101 102 185 184 185 F T F <---
110822024-04-17T17:17:24.433Z 5 100-119 97 97 97 172 172 172 F F F
110832024-04-17T17:17:24.433Z 6 120-139 98 98 98 173 173 173 F F F
110842024-04-17T17:17:24.433Z 7 140-159 100 100 100 182 182 182 F F F
110852024-04-17T17:17:24.433Z 8 160-179 100 100 100 182 182 182 F F F
110862024-04-17T17:17:24.434Z 9 180-199 100 100 100 179 179 179 F F F
110872024-04-17T17:17:24.434Z 10 200-219 100 100 100 177 177 177 F F F
110882024-04-17T17:17:24.437Z 11 220-239 98 98 98 173 173 173 F F F
110892024-04-17T17:17:24.437Z 12 240-259 99 99 99 175 175 175 F F F
110902024-04-17T17:17:24.437Z 13 260-279 97 97 97 172 172 172 T F T <---
110912024-04-17T17:17:24.438Z 14 280-299 100 100 100 180 180 180 F F F
110922024-04-17T17:17:24.438Z 15 300-319 102 100 102 185 180 185 T F T <---
110932024-04-17T17:17:24.438Z 16 320-339 96 96 96 171 171 171 F F F
110942024-04-17T17:17:24.438Z 17 340-359 93 93 93 165 165 165 F F F
110952024-04-17T17:17:24.438Z 18 360-379 98 98 98 173 173 173 F F F
110962024-04-17T17:17:24.438Z 19 380-399 101 101 101 184 184 184 T F T <---
110972024-04-17T17:17:24.438Z 20 400-419 97 97 97 172 172 172 F F F
110982024-04-17T17:17:24.438Z 21 420-439 101 101 101 184 184 184 F F F
110992024-04-17T17:17:24.438Z 22 440-459 94 94 94 167 167 167 F F F
111002024-04-17T17:17:24.438Z 23 460-479 102 100 102 185 176 185 F T F <---
111012024-04-17T17:17:24.438Z 24 480-499 97 97 97 172 172 172 F F F
111022024-04-17T17:17:24.438Z 25 500-519 98 98 98 173 173 173 T F T <---
111032024-04-17T17:17:24.438Z 26 520-539 102 101 102 185 184 185 T T T <---
111042024-04-17T17:17:24.438Z 27 540-559 96 96 96 171 171 171 T F T <---
111052024-04-17T17:17:24.438Z 28 560-579 102 95 102 185 168 185 T T T <---
111062024-04-17T17:17:24.438Z 29 580-599 98 98 98 173 173 173 F F F
111072024-04-17T17:17:24.438ZMax gen: 102, Max flush: 185
111082024-04-17T17:17:24.438ZError: Difference in extent metadata found!
111092024-04-17T17:17:24.438ZA Difference in extent metadata is expected here
111102024-04-17T17:17:24.438ZOn loop 51
111112024-04-17T17:17:24.438Z
111122024-04-17T17:17:24.442ZVerifying data now
111132024-04-17T17:17:35.003ZNo change after verify
111142024-04-17T17:17:35.003ZLoop: 51 Downstairs dump after verify (and repair):
111152024-04-17T17:17:35.018ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
111162024-04-17T17:17:35.018Z 0 000-019 102 102 102 185 185 185 F F F
111172024-04-17T17:17:35.022Z 1 020-039 94 94 94 167 167 167 F F F
111182024-04-17T17:17:35.022Z 2 040-059 102 102 102 185 185 185 F F F
111192024-04-17T17:17:35.022Z 3 060-079 102 102 102 185 185 185 F F F
111202024-04-17T17:17:35.022Z 4 080-099 102 102 102 185 185 185 F F F
111212024-04-17T17:17:35.022Z 5 100-119 97 97 97 172 172 172 F F F
111222024-04-17T17:17:35.022Z 6 120-139 98 98 98 173 173 173 F F F
111232024-04-17T17:17:35.022Z 7 140-159 100 100 100 182 182 182 F F F
111242024-04-17T17:17:35.022Z 8 160-179 100 100 100 182 182 182 F F F
111252024-04-17T17:17:35.022Z 9 180-199 100 100 100 179 179 179 F F F
111262024-04-17T17:17:35.022Z 10 200-219 100 100 100 177 177 177 F F F
111272024-04-17T17:17:35.022Z 11 220-239 98 98 98 173 173 173 F F F
111282024-04-17T17:17:35.022Z 12 240-259 99 99 99 175 175 175 F F F
111292024-04-17T17:17:35.022Z 13 260-279 103 103 103 186 186 186 F F F
111302024-04-17T17:17:35.022Z 14 280-299 100 100 100 180 180 180 F F F
111312024-04-17T17:17:35.022Z 15 300-319 103 103 103 186 186 186 F F F
111322024-04-17T17:17:35.023Z 16 320-339 96 96 96 171 171 171 F F F
111332024-04-17T17:17:35.023Z 17 340-359 93 93 93 165 165 165 F F F
111342024-04-17T17:17:35.023Z 18 360-379 98 98 98 173 173 173 F F F
111352024-04-17T17:17:35.023Z 19 380-399 103 103 103 186 186 186 F F F
111362024-04-17T17:17:35.023Z 20 400-419 97 97 97 172 172 172 F F F
111372024-04-17T17:17:35.023Z 21 420-439 101 101 101 184 184 184 F F F
111382024-04-17T17:17:35.023Z 22 440-459 94 94 94 167 167 167 F F F
111392024-04-17T17:17:35.023Z 23 460-479 102 102 102 185 185 185 F F F
111402024-04-17T17:17:35.023Z 24 480-499 97 97 97 172 172 172 F F F
111412024-04-17T17:17:35.023Z 25 500-519 103 103 103 186 186 186 F F F
111422024-04-17T17:17:35.023Z 26 520-539 103 103 103 186 186 186 F F F
111432024-04-17T17:17:35.023Z 27 540-559 103 103 103 186 186 186 F F F
111442024-04-17T17:17:35.023Z 28 560-579 103 103 103 186 186 186 F F F
111452024-04-17T17:17:35.023Z 29 580-599 98 98 98 173 173 173 F F F
111462024-04-17T17:17:35.023ZMax gen: 103, Max flush: 186
111472024-04-17T17:17:35.023Z
111482024-04-17T17:17:35.023ZBegin loop 52
111492024-04-17T17:17:35.023ZDownstairs to restart: 1
111502024-04-17T17:17:35.040Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.648381735Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587}
111512024-04-17T17:17:35.047Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.648704195Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587}
111522024-04-17T17:17:35.047Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30Crucible runtime is spawned
111532024-04-17T17:17:35.047Z,"time":"2024-04-17T17:17:33.648740027Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587}
111542024-04-17T17:17:35.047Z{"msg":"Crucible 6f18417d-91d9-4627-9dcf-7a82d096b66f has session id: 12977389-f3fb-4fd1-a526-6394fd8768d8","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.648772449Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111552024-04-17T17:17:35.047Z{"msg":"Upstairs opts: Upstairs UUID: 6f18417d-91d9-4627-9dcf-7a82d096b66f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.648825073Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111562024-04-17T17:17:35.047Z{"msg":"Crucible stats registered with UUID: 6f18417d-91d9-4627-9dcf-7a82d096b66f","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.648853784Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111572024-04-17T17:17:35.047Z{"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.648870515Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587}
111582024-04-17T17:17:35.047Z{"msg":"6f18417d-91d9-4627-9dcf-7a82d096b66f active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.64895437Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111592024-04-17T17:17:35.047Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.648977882Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"client":"0","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111602024-04-17T17:17:35.048Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.649019164Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"client":"0","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111612024-04-17T17:17:35.048Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.649057447Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"client":"1","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111622024-04-17T17:17:35.048Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.649079798Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"client":"1","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111632024-04-17T17:17:35.048Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.64910922Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"client":"2","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111642024-04-17T17:17:35.048Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.649143592Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"client":"2","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111652024-04-17T17:17:35.048Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.649170623Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"":"io task","client":"2","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111662024-04-17T17:17:35.048Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.649212576Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"":"io task","client":"0","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111672024-04-17T17:17:35.048Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.649247978Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"":"io task","client":"1","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111682024-04-17T17:17:35.048Z{"msg":"ds_connection connected from Ok(127.0.0.1:41784)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.64928463Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"":"io task","client":"0","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111692024-04-17T17:17:35.048Z{"msg":"ds_connection connected from Ok(127.0.0.1:47792)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.649490963Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"":"io task","client":"1","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111702024-04-17T17:17:35.048Z{"msg":"ds_connection connected from Ok(127.0.0.1:64751)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.649550256Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"":"io task","client":"2","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111712024-04-17T17:17:35.048Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.649959311Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"client":"2","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111722024-04-17T17:17:35.048Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.650001374Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"client":"2","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111732024-04-17T17:17:35.048Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.650053887Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"client":"0","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111742024-04-17T17:17:35.048Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.650084229Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"client":"0","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111752024-04-17T17:17:35.048Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.650124271Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"client":"1","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111762024-04-17T17:17:35.048Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.650146753Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"client":"1","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111772024-04-17T17:17:35.048Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.650625502Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"client":"2","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111782024-04-17T17:17:35.048Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.650709687Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"client":"0","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111792024-04-17T17:17:35.048Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.650753269Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"client":"1","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111802024-04-17T17:17:35.048Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.651246109Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"client":"2","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111812024-04-17T17:17:35.048Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.651294832Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"client":"2","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111822024-04-17T17:17:35.048Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.651357816Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111832024-04-17T17:17:35.048Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.651378217Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"client":"0","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111842024-04-17T17:17:35.049Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.651408679Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"client":"0","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111852024-04-17T17:17:35.049Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.65142954Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111862024-04-17T17:17:35.049Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.651456592Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"client":"1","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111872024-04-17T17:17:35.049ZWait for a query_work_queue command to finish before sending IO
111882024-04-17T17:17:35.049Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.651481323Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"client":"1","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111892024-04-17T17:17:35.049Z{"msg":"[0]R flush_numbers[0..12]: [185, 167, 185, 185, 185, 172, 173, 182, 182, 179, 177, 173]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.651510185Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"":"downstairs","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111902024-04-17T17:17:35.049Z{"msg":"[0]R generation[0..12]: [102, 94, 102, 102, 102, 97, 98, 100, 100, 100, 100, 98]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.651539917Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"":"downstairs","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111912024-04-17T17:17:35.049Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.651564179Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"":"downstairs","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111922024-04-17T17:17:35.049Z{"msg":"[1]R flush_numbers[0..12]: [185, 167, 185, 185, 185, 172, 173, 182, 182, 179, 177, 173]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.65159143Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"":"downstairs","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111932024-04-17T17:17:35.049Z{"msg":"[1]R generation[0..12]: [102, 94, 102, 102, 102, 97, 98, 100, 100, 100, 100, 98]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.651607011Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"":"downstairs","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111942024-04-17T17:17:35.049Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.651638173Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"":"downstairs","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111952024-04-17T17:17:35.049Z{"msg":"[2]R flush_numbers[0..12]: [185, 167, 185, 185, 185, 172, 173, 182, 182, 179, 177, 173]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.651667565Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"":"downstairs","":"downstairs","session_id":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
111962024-04-17T17:17:35.049Z12977389-f3fb-4fd1-a526-6394fd8768d8"}
111972024-04-17T17:17:35.049Z{"msg":"[2]R generation[0..12]: [102, 94, 102, 102, 102, 97, 98, 100, 100, 100, 100, 98]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.651700707Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"":"downstairs","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111982024-04-17T17:17:35.049Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.651719918Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"":"downstairs","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
111992024-04-17T17:17:35.049Z{"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.651748019Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"":"downstairs","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
112002024-04-17T17:17:35.049Z{"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.651766251Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"":"downstairs","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
112012024-04-17T17:17:35.049Z{"msg":"Next flush: 187","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.651794112Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"":"downstairs","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
112022024-04-17T17:17:35.049Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.651813183Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"":"downstairs","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
112032024-04-17T17:17:35.049Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.651841065Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
112042024-04-17T17:17:35.049Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.651857646Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"client":"0","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
112052024-04-17T17:17:35.049Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.651885638Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"client":"1","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
112062024-04-17T17:17:35.049Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.651906509Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"client":"2","":"downstairs","session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
112072024-04-17T17:17:35.050Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.651934071Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
112082024-04-17T17:17:35.050Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.651950982Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
112092024-04-17T17:17:35.050Z{"msg":"6f18417d-91d9-4627-9dcf-7a82d096b66f is now active with session: 12977389-f3fb-4fd1-a526-6394fd8768d8","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.651978273Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
112102024-04-17T17:17:35.050Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.651995125Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587,"session_id":"12977389-f3fb-4fd1-a526-6394fd8768d8"}
112112024-04-17T17:17:35.050Z{"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:33.652025827Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587}
112122024-04-17T17:17:35.050ZLoading write count information from file "/tmp/test_repair_verify.data"
112132024-04-17T17:17:35.050ZRun Repair workload
112142024-04-17T17:17:35.055Z01/30 Write block 49 len 12288 data: 7 6 7
112152024-04-17T17:17:35.055Z02/30 Read block 34 len 28672
112162024-04-17T17:17:35.055Z03/30 Flush
112172024-04-17T17:17:35.055Z04/30 Write block 34 len 20480 data: 18 15 12 10 9
112182024-04-17T17:17:35.055Z05/30 Read block 148 len 16384
112192024-04-17T17:17:35.055Z06/30 Write block 459 len 40960 data: 8 9 7 7 9 9 10 9 8 8
112202024-04-17T17:17:35.055Z07/30 Write block 278 len 32768 data: 7 6 6 4 4 4 5 4
112212024-04-17T17:17:35.055Z08/30 Read block 177 len 32768
112222024-04-17T17:17:35.055Z09/30 Read block 436 len 12288
112232024-04-17T17:17:35.055Z10/30 Write block 149 len 4096 data: 7
112242024-04-17T17:17:35.055Z11/30 Read block 419 len 24576
112252024-04-17T17:17:35.059Z12/30 Read block 137 len 36864
112262024-04-17T17:17:35.059Z13/30 Write block 293 len 12288 data: 8 7 5
112272024-04-17T17:17:35.059Z14/30 Read block 527 len 36864
112282024-04-17T17:17:35.059Z15/30 Read block 489 len 24576
112292024-04-17T17:17:35.059Z16/30 Write block 80 len 24576 data: 6 6 8 11 7 8
112302024-04-17T17:17:35.059Z17/30 Read block 301 len 20480
112312024-04-17T17:17:35.063Z18/30 Read block 250 len 12288
112322024-04-17T17:17:35.063Z19/30 Read block 311 len 40960
112332024-04-17T17:17:35.064Z20/30 Read block 463 len 28672
112342024-04-17T17:17:35.064Z21/30 Write block 383 len 20480 data: 7 7 9 8 9
112352024-04-17T17:17:35.064Z22/30 Flush
112362024-04-17T17:17:35.067Z23/30 Write block 27 len 12288 data: 6 6 10
112372024-04-17T17:17:35.070Z24/30 Flush
112382024-04-17T17:17:35.070Z25/30 Write block 246 len 12288 data: 11 11 10
112392024-04-17T17:17:35.070Z26/30 Read block 35 len 4096
112402024-04-17T17:17:35.074Z27/30 Read block 370 len 24576
112412024-04-17T17:17:35.074Z28/30 Write block 186 len 32768 data: 3 7 9 9 7 6 7 7
112422024-04-17T17:17:35.074Z29/30 Read block 515 len 4096
112432024-04-17T17:17:35.074Z30/30 Read block 585 len 4096
112442024-04-17T17:17:35.078Z----------------------------------------------------------------
112452024-04-17T17:17:35.078Z Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:27
112462024-04-17T17:17:35.078ZStates: Active Active Active
112472024-04-17T17:17:35.078ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
112482024-04-17T17:17:35.078Z 4 Acked 1003 Write 5 Done Done Done false
112492024-04-17T17:17:35.079Z 5 Acked 1004 Read 4 Done Sent Done false
112502024-04-17T17:17:35.079Z 6 Acked 1005 Write 10 Done Sent Done false
112512024-04-17T17:17:35.079Z 7 Acked 1006 Write 8 Done Sent Done false
112522024-04-17T17:17:35.079Z 8 Acked 1007 Read 8 Done Sent Done false
112532024-04-17T17:17:35.079Z 9 Acked 1008 Read 3 Done Sent Done false
112542024-04-17T17:17:35.079Z 10 Acked 1009 Write 1 Done Sent Done false
112552024-04-17T17:17:35.079Z 11 Acked 1010 Read 6 Done Sent Done false
112562024-04-17T17:17:35.079Z 12 Acked 1011 Read 9 Done Sent Done false
112572024-04-17T17:17:35.079Z 13 Acked 1012 Write 3 Done Sent Done false
112582024-04-17T17:17:35.079Z 14 Acked 1013 Read 9 Done Sent Done false
112592024-04-17T17:17:35.079Z 15 Acked 1014 Read 6 Done Sent Done false
112602024-04-17T17:17:35.079Z 16 Acked 1015 Write 6 Done Sent Done false
112612024-04-17T17:17:35.079Z 17 Acked 1016 Read 5 Done Sent Done false
112622024-04-17T17:17:35.079Z 18 Acked 1017 Read 3 Done Sent Done false
112632024-04-17T17:17:35.079Z 19 Acked 1018 Read 10 Done Sent Done false
112642024-04-17T17:17:35.079Z 20 Acked 1019 Read 7 Done Sent Done false
112652024-04-17T17:17:35.079Z 21 Acked 1020 Write 5 Done Sent Done false
112662024-04-17T17:17:35.079Z 22 Acked 1021 Flush 0 Done Sent Done false
112672024-04-17T17:17:35.079Z 23 Acked 1022 Write 3 Done Sent Done false
112682024-04-17T17:17:35.079Z 24 Acked 1023 Flush 0 Done Sent Done false
112692024-04-17T17:17:35.079Z 25 Acked 1024 Write 3 Done Sent Done false
112702024-04-17T17:17:35.079Z 26 Acked 1025 Read 1 Done Sent Done false
112712024-04-17T17:17:35.079Z 27 Acked 1026 Read 6 Done Sent Done false
112722024-04-17T17:17:35.080Z 28 Acked 1027 Write 8 Done Sent Done false
112732024-04-17T17:17:35.080Z 29 Acked 1028 Read 1 Done Sent Done false
112742024-04-17T17:17:35.080Z 30 Acked 1029 Read 1 Done Sent Done false
112752024-04-17T17:17:35.080Z STATES DS:0 DS:1 DS:2 TOTAL
112762024-04-17T17:17:35.080Z New 0 0 0 0
112772024-04-17T17:17:35.080Z Sent 0 26 0 26
112782024-04-17T17:17:35.080Z Done 27 1 27 55
112792024-04-17T17:17:35.080Z Skipped 0 0 0 0
112802024-04-17T17:17:35.080Z Error 0 0 0 0
112812024-04-17T17:17:35.080ZLast Flush: 1023 1002 1023
112822024-04-17T17:17:35.080ZDownstairs last five completed: 1002 1001 1000
112832024-04-17T17:17:35.080ZUpstairs last five completed: 30 29 28 27 26
112842024-04-17T17:17:35.080Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:17:33.682888027Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1587}
112852024-04-17T17:17:35.080ZWrote out file "/tmp/test_repair_verify.data"
112862024-04-17T17:17:35.084Z
112872024-04-17T17:17:35.084ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
112882024-04-17T17:17:35.100ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
112892024-04-17T17:17:35.100Z 0 000-019 102 102 102 185 185 185 F F F
112902024-04-17T17:17:35.104Z 1 020-039 104 94 104 189 167 189 F T F <---
112912024-04-17T17:17:35.104Z 2 040-059 104 104 104 187 187 187 F F F
112922024-04-17T17:17:35.104Z 3 060-079 102 102 102 185 185 185 F F F
112932024-04-17T17:17:35.104Z 4 080-099 104 102 104 188 185 188 F F F <---
112942024-04-17T17:17:35.105Z 5 100-119 97 97 97 172 172 172 F F F
112952024-04-17T17:17:35.105Z 6 120-139 98 98 98 173 173 173 F F F
112962024-04-17T17:17:35.105Z 7 140-159 104 100 104 188 182 188 F F F <---
112972024-04-17T17:17:35.105Z 8 160-179 100 100 100 182 182 182 F F F
112982024-04-17T17:17:35.105Z 9 180-199 100 100 100 179 179 179 T F T <---
112992024-04-17T17:17:35.105Z 10 200-219 100 100 100 177 177 177 F F F
113002024-04-17T17:17:35.105Z 11 220-239 98 98 98 173 173 173 F F F
113012024-04-17T17:17:35.105Z 12 240-259 99 99 99 175 175 175 T F T <---
113022024-04-17T17:17:35.105Z 13 260-279 104 103 104 188 186 188 F F F <---
113032024-04-17T17:17:35.105Z 14 280-299 104 100 104 188 180 188 F F F <---
113042024-04-17T17:17:35.105Z 15 300-319 103 103 103 186 186 186 F F F
113052024-04-17T17:17:35.105Z 16 320-339 96 96 96 171 171 171 F F F
113062024-04-17T17:17:35.105Z 17 340-359 93 93 93 165 165 165 F F F
113072024-04-17T17:17:35.105Z 18 360-379 98 98 98 173 173 173 F F F
113082024-04-17T17:17:35.105Z 19 380-399 104 103 104 188 186 188 F F F <---
113092024-04-17T17:17:35.105Z 20 400-419 97 97 97 172 172 172 F F F
113102024-04-17T17:17:35.105Z 21 420-439 101 101 101 184 184 184 F F F
113112024-04-17T17:17:35.105Z 22 440-459 104 94 104 188 167 188 F F F <---
113122024-04-17T17:17:35.105Z 23 460-479 104 102 104 188 185 188 F F F <---
113132024-04-17T17:17:35.105Z 24 480-499 97 97 97 172 172 172 F F F
113142024-04-17T17:17:35.105Z 25 500-519 103 103 103 186 186 186 F F F
113152024-04-17T17:17:35.105Z 26 520-539 103 103 103 186 186 186 F F F
113162024-04-17T17:17:35.106Z 27 540-559 103 103 103 186 186 186 F F F
113172024-04-17T17:17:35.106Z 28 560-579 103 103 103 186 186 186 F F F
113182024-04-17T17:17:35.106Z 29 580-599 98 98 98 173 173 173 F F F
113192024-04-17T17:17:35.106ZMax gen: 104, Max flush: 189
113202024-04-17T17:17:35.106ZError: Difference in extent metadata found!
113212024-04-17T17:17:35.106ZA Difference in extent metadata is expected here
113222024-04-17T17:17:35.106ZOn loop 52
113232024-04-17T17:17:35.106Z
113242024-04-17T17:17:35.106ZVerifying data now
113252024-04-17T17:17:39.580ZNo change after verify
113262024-04-17T17:17:39.580ZLoop: 52 Downstairs dump after verify (and repair):
113272024-04-17T17:17:39.587ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
113282024-04-17T17:17:39.587Z 0 000-019 102 102 102 185 185 185 F F F
113292024-04-17T17:17:39.597Z 1 020-039 104 104 104 189 189 189 F F F
113302024-04-17T17:17:39.597Z 2 040-059 104 104 104 187 187 187 F F F
113312024-04-17T17:17:39.597Z 3 060-079 102 102 102 185 185 185 F F F
113322024-04-17T17:17:39.597Z 4 080-099 104 104 104 188 188 188 F F F
113332024-04-17T17:17:39.597Z 5 100-119 97 97 97 172 172 172 F F F
113342024-04-17T17:17:39.597Z 6 120-139 98 98 98 173 173 173 F F F
113352024-04-17T17:17:39.597Z 7 140-159 104 104 104 188 188 188 F F F
113362024-04-17T17:17:39.597Z 8 160-179 100 100 100 182 182 182 F F F
113372024-04-17T17:17:39.597Z 9 180-199 105 105 105 190 190 190 F F F
113382024-04-17T17:17:39.597Z 10 200-219 100 100 100 177 177 177 F F F
113392024-04-17T17:17:39.597Z 11 220-239 98 98 98 173 173 173 F F F
113402024-04-17T17:17:39.597Z 12 240-259 105 105 105 190 190 190 F F F
113412024-04-17T17:17:39.597Z 13 260-279 104 104 104 188 188 188 F F F
113422024-04-17T17:17:39.597Z 14 280-299 104 104 104 188 188 188 F F F
113432024-04-17T17:17:39.597Z 15 300-319 103 103 103 186 186 186 F F F
113442024-04-17T17:17:39.597Z 16 320-339 96 96 96 171 171 171 F F F
113452024-04-17T17:17:39.597Z 17 340-359 93 93 93 165 165 165 F F F
113462024-04-17T17:17:39.598Z 18 360-379 98 98 98 173 173 173 F F F
113472024-04-17T17:17:39.598Z 19 380-399 104 104 104 188 188 188 F F F
113482024-04-17T17:17:39.598Z 20 400-419 97 97 97 172 172 172 F F F
113492024-04-17T17:17:39.598Z 21 420-439 101 101 101 184 184 184 F F F
113502024-04-17T17:17:39.598Z 22 440-459 104 104 104 188 188 188 F F F
113512024-04-17T17:17:39.598Z 23 460-479 104 104 104 188 188 188 F F F
113522024-04-17T17:17:39.598Z 24 480-499 97 97 97 172 172 172 F F F
113532024-04-17T17:17:39.598Z 25 500-519 103 103 103 186 186 186 F F F
113542024-04-17T17:17:39.598Z 26 520-539 103 103 103 186 186 186 F F F
113552024-04-17T17:17:39.598Z 27 540-559 103 103 103 186 186 186 F F F
113562024-04-17T17:17:39.598Z 28 560-579 103 103 103 186 186 186 F F F
113572024-04-17T17:17:39.598Z 29 580-599 98 98 98 173 173 173 F F F
113582024-04-17T17:17:39.598ZMax gen: 105, Max flush: 190
113592024-04-17T17:17:39.598Z
113602024-04-17T17:17:39.598ZBegin loop 53
113612024-04-17T17:17:39.598ZDownstairs to restart: 1
113622024-04-17T17:17:39.610Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.217154967Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595}
113632024-04-17T17:17:39.610Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.217481637Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595}
113642024-04-17T17:17:39.610Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30Crucible runtime is spawned
113652024-04-17T17:17:39.610Z,"time":"2024-04-17T17:17:38.217525769Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595}
113662024-04-17T17:17:39.611Z{"msg":"Crucible 74ce8cd0-be46-4940-912e-bbc9123ea8ed has session id: 4ac024ee-dce6-4420-857a-16343eacd64e","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.217565642Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113672024-04-17T17:17:39.628Z{"msg":"Upstairs opts: Upstairs UUID: 74ce8cd0-be46-4940-912e-bbc9123ea8ed, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.217612945Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113682024-04-17T17:17:39.628Z{"msg":"Crucible stats registered with UUID: 74ce8cd0-be46-4940-912e-bbc9123ea8ed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.217653937Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113692024-04-17T17:17:39.628Z{"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.217678818Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595}
113702024-04-17T17:17:39.628Z{"msg":"74ce8cd0-be46-4940-912e-bbc9123ea8ed active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.217775715Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113712024-04-17T17:17:39.628Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.217803956Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"client":"0","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113722024-04-17T17:17:39.628Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.217829398Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"client":"0","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113732024-04-17T17:17:39.628Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.21787228Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"client":"1","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113742024-04-17T17:17:39.628Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.217898632Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"client":"1","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113752024-04-17T17:17:39.628Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.217935904Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"client":"2","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113762024-04-17T17:17:39.628Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.217971936Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"client":"2","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113772024-04-17T17:17:39.628Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.218066692Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113782024-04-17T17:17:39.629Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.218298086Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113792024-04-17T17:17:39.629Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.218330368Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113802024-04-17T17:17:39.629Z{"msg":"ds_connection connected from Ok(127.0.0.1:39878)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.21835897Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113812024-04-17T17:17:39.629Z{"msg":"ds_connection connected from Ok(127.0.0.1:56361)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.218384421Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113822024-04-17T17:17:39.629Z{"msg":"ds_connection connected from Ok(127.0.0.1:62811)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.218428874Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113832024-04-17T17:17:39.629Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.219076083Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"client":"2","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113842024-04-17T17:17:39.629Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.219117276Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"client":"2","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113852024-04-17T17:17:39.629Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.219172889Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"client":"0","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113862024-04-17T17:17:39.629Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.219218782Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"client":"0","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113872024-04-17T17:17:39.629Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.219247914Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"client":"1","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113882024-04-17T17:17:39.629Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.219289956Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"client":"1","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113892024-04-17T17:17:39.629Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.219622666Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"client":"2","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113902024-04-17T17:17:39.629Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.21969084Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"client":"1","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113912024-04-17T17:17:39.629Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.219830219Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"client":"0","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113922024-04-17T17:17:39.629Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.220130957Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"client":"2","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113932024-04-17T17:17:39.629Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.220170509Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"client":"2","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113942024-04-17T17:17:39.629Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.220195971Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113952024-04-17T17:17:39.629Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.220236353Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"client":"1","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113962024-04-17T17:17:39.629Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.220270456Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"client":"1","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113972024-04-17T17:17:39.629Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.220294367Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113982024-04-17T17:17:39.629Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.220319909Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"client":"0","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
113992024-04-17T17:17:39.629Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.22034318Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"client":"0","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
114002024-04-17T17:17:39.629Z{"msg":"[0]R flush_numbers[0..12]: [185, 189, 187, 185, 188, 172, 173, 188, 182, 190, 177, 173]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.220371572Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"":"downstairs","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
114012024-04-17T17:17:39.630Z{"msg":"[0]R generation[0..12]: [102, 104, 104, 102, 104, 97, 98, 104, 100, 105, 100, 98]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.220404614Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"":"downstairs","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
114022024-04-17T17:17:39.630Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.220437876Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"":"downstairs","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
114032024-04-17T17:17:39.630Z{"msg":"[1]R flush_numbers[0..12]: [185, 189, 187, 185, 188, 172, 173, 188, 182, 190, 177, 173]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.220463107Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"":"downstairs","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
114042024-04-17T17:17:39.630Z{"msg":"[1]R generation[0..12]: [102, 104, 104, 102, 104, 97, 98, 104, 100, 105, 100, 98]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.220492609Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"":"downstairs","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
114052024-04-17T17:17:39.630Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
114062024-04-17T17:17:39.630Z,"time":"2024-04-17T17:17:38.220518861Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"":"downstairs","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
114072024-04-17T17:17:39.630Z{"msg":"[2]R flush_numbers[0..12]: [185, 189, 187, 185, 188, 172, 173, 188, 182, 190, 177, 173]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.220553683Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"":"downstairs","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
114082024-04-17T17:17:39.630Z{"msg":"[2]R generation[0..12]: [102, 104, 104, 102, 104, 97, 98, 104, 100, 105, 100, 98]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.220586085Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"":"downstairs","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
114092024-04-17T17:17:39.630Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.220611746Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"":"downstairs","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
114102024-04-17T17:17:39.630Z{"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.220640028Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"":"downstairs","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
114112024-04-17T17:17:39.630Z{"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.22067239Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"":"downstairs","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
114122024-04-17T17:17:39.630Z{"msg":"Next flush: 191","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.220702202Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"":"downstairs","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
114132024-04-17T17:17:39.630Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.220724753Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"":"downstairs","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
114142024-04-17T17:17:39.630Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.220751715Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
114152024-04-17T17:17:39.630Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.220773126Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"client":"0","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
114162024-04-17T17:17:39.630Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.220799688Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"client":"1","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
114172024-04-17T17:17:39.630ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
114182024-04-17T17:17:39.630Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.220825209Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"client":"2","":"downstairs","session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
114192024-04-17T17:17:39.630Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.220852491Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
114202024-04-17T17:17:39.630Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.220874562Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
114212024-04-17T17:17:39.630Z{"msg":"74ce8cd0-be46-4940-912e-bbc9123ea8ed is now active with session: 4ac024ee-dce6-4420-857a-16343eacd64e","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.220902064Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
114222024-04-17T17:17:39.630Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.220923585Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595,"session_id":"4ac024ee-dce6-4420-857a-16343eacd64e"}
114232024-04-17T17:17:39.631Z{"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:38.220949017Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595}
114242024-04-17T17:17:39.631ZLoading write count information from file "/tmp/test_repair_verify.data"
114252024-04-17T17:17:39.631ZRun Repair workload
114262024-04-17T17:17:39.631Z01/30 Write block 19 len 24576 data: 6 7 10 11 10 10
114272024-04-17T17:17:39.631Z02/30 Write block 387 len 32768 data: 10 8 9 9 10 11 9 9
114282024-04-17T17:17:39.631Z03/30 Read block 125 len 32768
114292024-04-17T17:17:39.641Z04/30 Read block 292 len 24576
114302024-04-17T17:17:39.641Z05/30 Read block 73 len 24576
114312024-04-17T17:17:39.641Z06/30 Read block 103 len 40960
114322024-04-17T17:17:39.641Z07/30 Write block 58 len 8192 data: 11 11
114332024-04-17T17:17:39.641Z08/30 Write block 234 len 36864 data: 11 12 10 8 6 8 8 11 9
114342024-04-17T17:17:39.641Z09/30 Read block 440 len 8192
114352024-04-17T17:17:39.641Z10/30 Read block 1 len 24576
114362024-04-17T17:17:39.641Z11/30 Write block 472 len 12288 data: 8 8 9
114372024-04-17T17:17:39.641Z12/30 Flush
114382024-04-17T17:17:39.641Z13/30 Write block 78 len 32768 data: 6 5 7 7 9 12 8 9
114392024-04-17T17:17:39.641Z14/30 Write block 444 len 4096 data: 9
114402024-04-17T17:17:39.642Z15/30 Read block 264 len 24576
114412024-04-17T17:17:39.642Z16/30 Read block 212 len 40960
114422024-04-17T17:17:39.642Z17/30 Write block 226 len 24576 data: 13 14 13 13 12 9
114432024-04-17T17:17:39.642Z18/30 Read block 65 len 24576
114442024-04-17T17:17:39.642Z19/30 Read block 256 len 40960
114452024-04-17T17:17:39.642Z20/30 Read block 206 len 16384
114462024-04-17T17:17:39.642Z21/30 Write block 562 len 8192 data: 10 9
114472024-04-17T17:17:39.642Z22/30 Read block 519 len 28672
114482024-04-17T17:17:39.642Z23/30 Write block 497 len 24576 data: 7 10 9 10 9 9
114492024-04-17T17:17:39.642Z24/30 Read block 499 len 32768
114502024-04-17T17:17:39.642Z25/30 Flush
114512024-04-17T17:17:39.642Z26/30 Write block 35 len 4096 data: 16
114522024-04-17T17:17:39.657Z27/30 Write block 248 len 28672 data: 11 8 8 8 9 9 8
114532024-04-17T17:17:39.657Z28/30 Read block 248 len 20480
114542024-04-17T17:17:39.657Z29/30 Read block 90 len 32768
114552024-04-17T17:17:39.657Z30/30 Read block 395 len 40960
114562024-04-17T17:17:39.657Z----------------------------------------------------------------
114572024-04-17T17:17:39.657Z Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:30
114582024-04-17T17:17:39.657ZStates: Active Active Active
114592024-04-17T17:17:39.657ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
114602024-04-17T17:17:39.657Z 1 Acked 1000 Write 6 Done Done Done false
114612024-04-17T17:17:39.657Z 2 Acked 1001 Write 8 Done Done Done false
114622024-04-17T17:17:39.657Z 3 Acked 1002 Read 8 Done Done Done false
114632024-04-17T17:17:39.658Z 4 Acked 1003 Read 6 Done Done Done false
114642024-04-17T17:17:39.658Z 5 Acked 1004 Read 6 Done Sent Done false
114652024-04-17T17:17:39.658Z 6 Acked 1005 Read 10 Done Sent Done false
114662024-04-17T17:17:39.658Z 7 Acked 1006 Write 2 Done Sent Done false
114672024-04-17T17:17:39.658Z 8 Acked 1007 Write 9 Done Sent Done false
114682024-04-17T17:17:39.658Z 9 Acked 1008 Read 2 Done Sent Done false
114692024-04-17T17:17:39.658Z 10 Acked 1009 Read 6 Done Sent Done false
114702024-04-17T17:17:39.658Z 11 Acked 1010 Write 3 Done Sent Done false
114712024-04-17T17:17:39.658Z 12 Acked 1011 Flush 0 Done Sent Done false
114722024-04-17T17:17:39.658Z 13 Acked 1012 Write 8 Done Sent Done false
114732024-04-17T17:17:39.659Z 14 Acked 1013 Write 1 Done Sent Done false
114742024-04-17T17:17:39.659Z 15 Acked 1014 Read 6 Done Sent Done false
114752024-04-17T17:17:39.659Z 16 Acked 1015 Read 10 Done Sent Done false
114762024-04-17T17:17:39.659Z 17 Acked 1016 Write 6 Done Sent Done false
114772024-04-17T17:17:39.659Z 18 Acked 1017 Read 6 Done Sent Done false
114782024-04-17T17:17:39.659Z 19 Acked 1018 Read 10 Done Sent Done false
114792024-04-17T17:17:39.659Z 20 Acked 1019 Read 4 Done Sent Done false
114802024-04-17T17:17:39.659Z 21 Acked 1020 Write 2 Done Sent Done false
114812024-04-17T17:17:39.659Z 22 Acked 1021 Read 7 Done Sent Done false
114822024-04-17T17:17:39.659Z 23 Acked 1022 Write 6 Done Sent Done false
114832024-04-17T17:17:39.659Z 24 Acked 1023 Read 8 Done Sent Done false
114842024-04-17T17:17:39.660Z 25 Acked 1024 Flush 0 Done Sent Done false
114852024-04-17T17:17:39.660Z 26 Acked 1025 Write 1 Done Sent Done false
114862024-04-17T17:17:39.660Z 27 Acked 1026 Write 7 Done Sent Done false
114872024-04-17T17:17:39.660Z 28 Acked 1027 Read 5 Done Sent Done false
114882024-04-17T17:17:39.660Z 29 Acked 1028 Read 8 Done Sent Done false
114892024-04-17T17:17:39.660Z 30 Acked 1029 Read 10 Sent Sent Done false
114902024-04-17T17:17:39.660Z STATES DS:0 DS:1 DS:2 TOTAL
114912024-04-17T17:17:39.660Z New 0 0 0 0
114922024-04-17T17:17:39.660Z Sent 1 26 0 27
114932024-04-17T17:17:39.660Z Done 29 4 30 63
114942024-04-17T17:17:39.660Z Skipped 0 0 0 0
114952024-04-17T17:17:39.660Z Error 0 0 0 0
114962024-04-17T17:17:39.660ZLast Flush: 1024 0 1024
114972024-04-17T17:17:39.660ZDownstairs last five completed:
114982024-04-17T17:17:39.661ZUpstairs last five completed: 30 29 28 27 26
114992024-04-17T17:17:39.661Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:17:38.251375151Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1595}
115002024-04-17T17:17:39.661ZWrote out file "/tmp/test_repair_verify.data"
115012024-04-17T17:17:39.661Z
115022024-04-17T17:17:39.671ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
115032024-04-17T17:17:39.671ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
115042024-04-17T17:17:39.671Z 0 000-019 106 102 106 191 185 191 F T F <---
115052024-04-17T17:17:39.671Z 1 020-039 106 104 106 191 189 191 T T T <---
115062024-04-17T17:17:39.671Z 2 040-059 106 104 106 191 187 191 F F F <---
115072024-04-17T17:17:39.671Z 3 060-079 106 102 106 192 185 192 F F F <---
115082024-04-17T17:17:39.671Z 4 080-099 106 104 106 192 188 192 F F F <---
115092024-04-17T17:17:39.671Z 5 100-119 97 97 97 172 172 172 F F F
115102024-04-17T17:17:39.671Z 6 120-139 98 98 98 173 173 173 F F F
115112024-04-17T17:17:39.671Z 7 140-159 104 104 104 188 188 188 F F F
115122024-04-17T17:17:39.671Z 8 160-179 100 100 100 182 182 182 F F F
115132024-04-17T17:17:39.671Z 9 180-199 105 105 105 190 190 190 F F F
115142024-04-17T17:17:39.672Z 10 200-219 100 100 100 177 177 177 F F F
115152024-04-17T17:17:39.672Z 11 220-239 106 98 106 192 173 192 F F F <---
115162024-04-17T17:17:39.672Z 12 240-259 106 105 106 191 190 191 T F T <---
115172024-04-17T17:17:39.672Z 13 260-279 104 104 104 188 188 188 F F F
115182024-04-17T17:17:39.672Z 14 280-299 104 104 104 188 188 188 F F F
115192024-04-17T17:17:39.672Z 15 300-319 103 103 103 186 186 186 F F F
115202024-04-17T17:17:39.672Z 16 320-339 96 96 96 171 171 171 F F F
115212024-04-17T17:17:39.672Z 17 340-359 93 93 93 165 165 165 F F F
115222024-04-17T17:17:39.672Z 18 360-379 98 98 98 173 173 173 F F F
115232024-04-17T17:17:39.672Z 19 380-399 106 104 106 191 188 191 F T F <---
115242024-04-17T17:17:39.672Z 20 400-419 97 97 97 172 172 172 F F F
115252024-04-17T17:17:39.672Z 21 420-439 101 101 101 184 184 184 F F F
115262024-04-17T17:17:39.672Z 22 440-459 106 104 106 192 188 192 F F F <---
115272024-04-17T17:17:39.672Z 23 460-479 106 104 106 191 188 191 F F F <---
115282024-04-17T17:17:39.672Z 24 480-499 106 97 106 192 172 192 F F F <---
115292024-04-17T17:17:39.672Z 25 500-519 106 103 106 192 186 192 F F F <---
115302024-04-17T17:17:39.672Z 26 520-539 103 103 103 186 186 186 F F F
115312024-04-17T17:17:39.672Z 27 540-559 103 103 103 186 186 186 F F F
115322024-04-17T17:17:39.672Z 28 560-579 106 103 106 192 186 192 F F F <---
115332024-04-17T17:17:39.672Z 29 580-599 98 98 98 173 173 173 F F F
115342024-04-17T17:17:39.672ZMax gen: 106, Max flush: 192
115352024-04-17T17:17:39.672ZError: Difference in extent metadata found!
115362024-04-17T17:17:39.692ZA Difference in extent metadata is expected here
115372024-04-17T17:17:39.692ZOn loop 53
115382024-04-17T17:17:39.692Z
115392024-04-17T17:17:39.693ZVerifying data now
115402024-04-17T17:17:54.267ZNo change after verify
115412024-04-17T17:17:54.267ZLoop: 53 Downstairs dump after verify (and repair):
115422024-04-17T17:17:54.282ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
115432024-04-17T17:17:54.282Z 0 000-019 106 106 106 191 191 191 F F F
115442024-04-17T17:17:54.285Z 1 020-039 107 107 107 193 193 193 F F F
115452024-04-17T17:17:54.285Z 2 040-059 106 106 106 191 191 191 F F F
115462024-04-17T17:17:54.285Z 3 060-079 106 106 106 192 192 192 F F F
115472024-04-17T17:17:54.285Z 4 080-099 106 106 106 192 192 192 F F F
115482024-04-17T17:17:54.285Z 5 100-119 97 97 97 172 172 172 F F F
115492024-04-17T17:17:54.286Z 6 120-139 98 98 98 173 173 173 F F F
115502024-04-17T17:17:54.286Z 7 140-159 104 104 104 188 188 188 F F F
115512024-04-17T17:17:54.286Z 8 160-179 100 100 100 182 182 182 F F F
115522024-04-17T17:17:54.286Z 9 180-199 105 105 105 190 190 190 F F F
115532024-04-17T17:17:54.286Z 10 200-219 100 100 100 177 177 177 F F F
115542024-04-17T17:17:54.286Z 11 220-239 106 106 106 192 192 192 F F F
115552024-04-17T17:17:54.286Z 12 240-259 107 107 107 193 193 193 F F F
115562024-04-17T17:17:54.286Z 13 260-279 104 104 104 188 188 188 F F F
115572024-04-17T17:17:54.286Z 14 280-299 104 104 104 188 188 188 F F F
115582024-04-17T17:17:54.286Z 15 300-319 103 103 103 186 186 186 F F F
115592024-04-17T17:17:54.286Z 16 320-339 96 96 96 171 171 171 F F F
115602024-04-17T17:17:54.286Z 17 340-359 93 93 93 165 165 165 F F F
115612024-04-17T17:17:54.286Z 18 360-379 98 98 98 173 173 173 F F F
115622024-04-17T17:17:54.286Z 19 380-399 106 106 106 191 191 191 F F F
115632024-04-17T17:17:54.286Z 20 400-419 97 97 97 172 172 172 F F F
115642024-04-17T17:17:54.286Z 21 420-439 101 101 101 184 184 184 F F F
115652024-04-17T17:17:54.286Z 22 440-459 106 106 106 192 192 192 F F F
115662024-04-17T17:17:54.286Z 23 460-479 106 106 106 191 191 191 F F F
115672024-04-17T17:17:54.286Z 24 480-499 106 106 106 192 192 192 F F F
115682024-04-17T17:17:54.286Z 25 500-519 106 106 106 192 192 192 F F F
115692024-04-17T17:17:54.286Z 26 520-539 103 103 103 186 186 186 F F F
115702024-04-17T17:17:54.286Z 27 540-559 103 103 103 186 186 186 F F F
115712024-04-17T17:17:54.287Z 28 560-579 106 106 106 192 192 192 F F F
115722024-04-17T17:17:54.287Z 29 580-599 98 98 98 173 173 173 F F F
115732024-04-17T17:17:54.287ZMax gen: 107, Max flush: 193
115742024-04-17T17:17:54.291Z
115752024-04-17T17:17:54.291ZBegin loop 54
115762024-04-17T17:17:54.291ZDownstairs to restart: 2
115772024-04-17T17:17:54.305Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.913956806Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603}
115782024-04-17T17:17:54.305Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.914238383Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603}
115792024-04-17T17:17:54.312Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.914278346Z","Crucible runtime is spawned
115802024-04-17T17:17:54.312Zhostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603}
115812024-04-17T17:17:54.312Z{"msg":"Crucible 53b9e769-eb14-44b6-85d0-449c91e4126f has session id: f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.914335689Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
115822024-04-17T17:17:54.312Z{"msg":"Upstairs opts: Upstairs UUID: 53b9e769-eb14-44b6-85d0-449c91e4126f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.914407554Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
115832024-04-17T17:17:54.313Z{"msg":"Crucible stats registered with UUID: 53b9e769-eb14-44b6-85d0-449c91e4126f","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.914451787Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
115842024-04-17T17:17:54.313Z{"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.91451511Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603}
115852024-04-17T17:17:54.313Z{"msg":"53b9e769-eb14-44b6-85d0-449c91e4126f active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.914544832Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
115862024-04-17T17:17:54.313Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.914591615Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"client":"0","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
115872024-04-17T17:17:54.313Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.914624747Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"client":"0","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
115882024-04-17T17:17:54.313Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.91467087Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"client":"1","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
115892024-04-17T17:17:54.313Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.914716163Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"client":"1","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
115902024-04-17T17:17:54.313Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.914747504Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"client":"2","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
115912024-04-17T17:17:54.313Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.914775636Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"client":"2","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
115922024-04-17T17:17:54.313Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.914824929Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
115932024-04-17T17:17:54.313Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.914858601Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
115942024-04-17T17:17:54.313Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.914891663Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
115952024-04-17T17:17:54.313Z{"msg":"ds_connection connected from Ok(127.0.0.1:58323)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.914931076Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
115962024-04-17T17:17:54.313Z{"msg":"ds_connection connected from Ok(127.0.0.1:56594)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.914955827Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
115972024-04-17T17:17:54.313Z{"msg":"ds_connection connected from Ok(127.0.0.1:56635)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.914979599Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
115982024-04-17T17:17:54.313Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.915452067Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"client":"1","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
115992024-04-17T17:17:54.313Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.915519801Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"client":"1","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116002024-04-17T17:17:54.313Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.915569854Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"client":"0","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116012024-04-17T17:17:54.313Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.915598776Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"client":"0","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116022024-04-17T17:17:54.313Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.916036903Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"client":"1","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116032024-04-17T17:17:54.313Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.916084615Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"client":"2","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116042024-04-17T17:17:54.314Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.916116577Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"client":"2","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116052024-04-17T17:17:54.314Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.916173571Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"client":"0","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116062024-04-17T17:17:54.314Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.916588786Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"client":"1","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116072024-04-17T17:17:54.314Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.916624898Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"client":"1","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116082024-04-17T17:17:54.314Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.916640109Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116092024-04-17T17:17:54.314Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.91665288Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"client":"0","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116102024-04-17T17:17:54.314Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.916674521Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"client":"0","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116112024-04-17T17:17:54.314Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.916687982Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116122024-04-17T17:17:54.314Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.916701543Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"client":"2","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116132024-04-17T17:17:54.314Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.91714406Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"client":"2","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116142024-04-17T17:17:54.314Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.917182842Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"client":"2","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116152024-04-17T17:17:54.314Z{"msg":"[0]R flush_numbers[0..12]: [191, 193, 191, 192, 192, 172, 173, 188, 182, 190, 177, 192]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.917199633Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"":"downstairs","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116162024-04-17T17:17:54.314Z{"msg":"[0]R generation[0..12]: [106, 107, 106, 106, 106, 97, 98, 104, 100, 105, 100, 106]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.917222874Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"":"downstairs","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116172024-04-17T17:17:54.314Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.917238265Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"":"downstairs","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116182024-04-17T17:17:54.314Z{"msg":"[1]R flush_numbers[0..12]: [191, 193, 191, 192, 192, 172, 173, 188, 182, 190, 177, 192]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.917269827Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"":"downstairs","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116192024-04-17T17:17:54.314Z{Wait for a query_work_queue command to finish before sending IO
116202024-04-17T17:17:54.314Z"msg":"[1]R generation[0..12]: [106, 107, 106, 106, 106, 97, 98, 104, 100, 105, 100, 106]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.917298609Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"":"downstairs","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116212024-04-17T17:17:54.314Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.917336381Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"":"downstairs","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116222024-04-17T17:17:54.314Z{"msg":"[2]R flush_numbers[0..12]: [191, 193, 191, 192, 192, 172, 173, 188, 182, 190, 177, 192]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.917358343Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"":"downstairs","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116232024-04-17T17:17:54.314Z{"msg":"[2]R generation[0..12]: [106, 107, 106, 106, 106, 97, 98, 104, 100, 105, 100, 106]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.917391825Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"":"downstairs","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116242024-04-17T17:17:54.314Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.917429287Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"":"downstairs","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116252024-04-17T17:17:54.314Z{"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.917456329Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"":"downstairs","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116262024-04-17T17:17:54.314Z{"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.91748206Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"":"downstairs","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116272024-04-17T17:17:54.314Z{"msg":"Next flush: 194","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.917508892Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"":"downstairs","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116282024-04-17T17:17:54.315Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.917522873Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"":"downstairs","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116292024-04-17T17:17:54.315Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.917543594Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116302024-04-17T17:17:54.315Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.917556155Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"client":"0","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116312024-04-17T17:17:54.315Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.917582936Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"client":"1","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116322024-04-17T17:17:54.315Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.917613428Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"client":"2","":"downstairs","session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116332024-04-17T17:17:54.315Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.91764793Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116342024-04-17T17:17:54.315ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
116352024-04-17T17:17:54.315Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.917676402Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116362024-04-17T17:17:54.315Z{"msg":"53b9e769-eb14-44b6-85d0-449c91e4126f is now active with session: f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.917706844Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116372024-04-17T17:17:54.315Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.917721045Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603,"session_id":"f58fd49c-cb9c-4bf9-9ca1-bd452f4428d3"}
116382024-04-17T17:17:54.315Z{"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:17:52.917740886Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603}
116392024-04-17T17:17:54.319ZLoading write count information from file "/tmp/test_repair_verify.data"
116402024-04-17T17:17:54.319ZRun Repair workload
116412024-04-17T17:17:54.319Z01/30 Write block 469 len 20480 data: 8 9 10 9 9
116422024-04-17T17:17:54.319Z02/30 Flush
116432024-04-17T17:17:54.319Z03/30 Write block 181 len 32768 data: 5 5 5 3 2 4 8 10
116442024-04-17T17:17:54.319Z04/30 Read block 497 len 16384
116452024-04-17T17:17:54.319Z05/30 Write block 251 len 28672 data: 9 10 10 9 6 10 10
116462024-04-17T17:17:54.319Z06/30 Read block 582 len 12288
116472024-04-17T17:17:54.319Z07/30 Read block 17 len 20480
116482024-04-17T17:17:54.319Z08/30 Read block 160 len 40960
116492024-04-17T17:17:54.322Z09/30 Write block 34 len 4096 data: 19
116502024-04-17T17:17:54.322Z10/30 Write block 590 len 4096 data: 7
116512024-04-17T17:17:54.322Z11/30 Write block 549 len 4096 data: 13
116522024-04-17T17:17:54.322Z12/30 Flush
116532024-04-17T17:17:54.326Z13/30 Write block 486 len 20480 data: 14 11 11 9 9
116542024-04-17T17:17:54.326Z14/30 Read block 209 len 40960
116552024-04-17T17:17:54.326Z15/30 Read block 337 len 32768
116562024-04-17T17:17:54.329Z16/30 Write block 312 len 16384 data: 10 11 11 10
116572024-04-17T17:17:54.329Z17/30 Flush
116582024-04-17T17:17:54.330Z18/30 Write block 375 len 36864 data: 7 5 7 7 8 7 6 7 8
116592024-04-17T17:17:54.333Z19/30 Read block 314 len 12288
116602024-04-17T17:17:54.333Z20/30 Write block 522 len 8192 data: 8 8
116612024-04-17T17:17:54.333Z21/30 Flush
116622024-04-17T17:17:54.338Z22/30 Write block 198 len 4096 data: 8
116632024-04-17T17:17:54.338Z23/30 Write block 417 len 24576 data: 11 10 11 10 6 7
116642024-04-17T17:17:54.338Z24/30 Read block 100 len 36864
116652024-04-17T17:17:54.338Z25/30 Read block 517 len 4096
116662024-04-17T17:17:54.345Z26/30 Write block 451 len 8192 data: 6 7
116672024-04-17T17:17:54.345Z27/30 Write block 24 len 8192 data: 11 10
116682024-04-17T17:17:54.345Z28/30 Read block 199 len 36864
116692024-04-17T17:17:54.345Z29/30 Read block 271 len 32768
116702024-04-17T17:17:54.345Z30/30 Write block 196 len 20480 data: 7 7 9 7 9
116712024-04-17T17:17:54.345Z----------------------------------------------------------------
116722024-04-17T17:17:54.345Z Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:28
116732024-04-17T17:17:54.345ZStates: Active Active Active
116742024-04-17T17:17:54.345ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
116752024-04-17T17:17:54.345Z 3 Acked 1002 Write 8 Done Done Done false
116762024-04-17T17:17:54.345Z 4 Acked 1003 Read 4 Done Done Done false
116772024-04-17T17:17:54.345Z 5 Acked 1004 Write 7 Done Done Done false
116782024-04-17T17:17:54.345Z 6 Acked 1005 Read 3 Done Done Done false
116792024-04-17T17:17:54.345Z 7 Acked 1006 Read 5 Done Done Sent false
116802024-04-17T17:17:54.345Z 8 Acked 1007 Read 10 Done Done Sent false
116812024-04-17T17:17:54.345Z 9 Acked 1008 Write 1 Done Done Sent false
116822024-04-17T17:17:54.346Z 10 Acked 1009 Write 1 Done Done Sent false
116832024-04-17T17:17:54.346Z 11 Acked 1010 Write 1 Done Done Sent false
116842024-04-17T17:17:54.346Z 12 Acked 1011 Flush 0 Done Done Sent false
116852024-04-17T17:17:54.346Z 13 Acked 1012 Write 5 Done Done Sent false
116862024-04-17T17:17:54.346Z 14 Acked 1013 Read 10 Done Done Sent false
116872024-04-17T17:17:54.346Z 15 Acked 1014 Read 8 Done Done Sent false
116882024-04-17T17:17:54.346Z 16 Acked 1015 Write 4 Done Done Sent false
116892024-04-17T17:17:54.346Z 17 Acked 1016 Flush 0 Done Done Sent false
116902024-04-17T17:17:54.346Z 18 Acked 1017 Write 9 Done Done Sent false
116912024-04-17T17:17:54.346Z 19 Acked 1018 Read 3 Done Done Sent false
116922024-04-17T17:17:54.346Z 20 Acked 1019 Write 2 Done Done Sent false
116932024-04-17T17:17:54.346Z 21 Acked 1020 Flush 0 Done Done Sent false
116942024-04-17T17:17:54.346Z 22 Acked 1021 Write 1 Done Done Sent false
116952024-04-17T17:17:54.346Z 23 Acked 1022 Write 6 Done Done Sent false
116962024-04-17T17:17:54.346Z 24 Acked 1023 Read 9 Done Done Sent false
116972024-04-17T17:17:54.346Z 25 Acked 1024 Read 1 Done Done Sent false
116982024-04-17T17:17:54.346Z 26 Acked 1025 Write 2 Done Done Sent false
116992024-04-17T17:17:54.346Z 27 Acked 1026 Write 2 Done Done Sent false
117002024-04-17T17:17:54.346Z 28 Acked 1027 Read 9 Done Done Sent false
117012024-04-17T17:17:54.346Z 29 Acked 1028 Read 8 Done Done Sent false
117022024-04-17T17:17:54.346Z 30 Acked 1029 Write 5 Sent Sent Sent false
117032024-04-17T17:17:54.346Z STATES DS:0 DS:1 DS:2 TOTAL
117042024-04-17T17:17:54.346Z New 0 0 0 0
117052024-04-17T17:17:54.346Z Sent 1 1 24 26
117062024-04-17T17:17:54.347Z Done 27 27 4 58
117072024-04-17T17:17:54.347Z Skipped 0 0 0 0
117082024-04-17T17:17:54.347Z Error 0 0 0 0
117092024-04-17T17:17:54.347ZLast Flush: 1020 1020 1001
117102024-04-17T17:17:54.347ZDownstairs last five completed: 1001 1000
117112024-04-17T17:17:54.347ZUpstairs last five completed: 30 29 28 27 26
117122024-04-17T17:17:54.347Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:17:52.949588046Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1603}
117132024-04-17T17:17:54.347ZWrote out file "/tmp/test_repair_verify.data"
117142024-04-17T17:17:54.351Z
117152024-04-17T17:17:54.351ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
117162024-04-17T17:17:54.365ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
117172024-04-17T17:17:54.365Z 0 000-019 106 106 106 191 191 191 F F F
117182024-04-17T17:17:54.365Z 1 020-039 108 108 107 195 195 193 T T F <---
117192024-04-17T17:17:54.365Z 2 040-059 106 106 106 191 191 191 F F F
117202024-04-17T17:17:54.365Z 3 060-079 106 106 106 192 192 192 F F F
117212024-04-17T17:17:54.365Z 4 080-099 106 106 106 192 192 192 F F F
117222024-04-17T17:17:54.365Z 5 100-119 97 97 97 172 172 172 F F F
117232024-04-17T17:17:54.366Z 6 120-139 98 98 98 173 173 173 F F F
117242024-04-17T17:17:54.366Z 7 140-159 104 104 104 188 188 188 F F F
117252024-04-17T17:17:54.366Z 8 160-179 100 100 100 182 182 182 F F F
117262024-04-17T17:17:54.369Z 9 180-199 108 108 105 195 195 190 T T T <---
117272024-04-17T17:17:54.369Z 10 200-219 100 100 100 177 177 177 T T F <---
117282024-04-17T17:17:54.369Z 11 220-239 106 106 106 192 192 192 F F F
117292024-04-17T17:17:54.369Z 12 240-259 108 108 107 195 195 193 F F T <---
117302024-04-17T17:17:54.369Z 13 260-279 104 104 104 188 188 188 F F F
117312024-04-17T17:17:54.369Z 14 280-299 104 104 104 188 188 188 F F F
117322024-04-17T17:17:54.369Z 15 300-319 108 108 103 196 196 186 F F F <---
117332024-04-17T17:17:54.370Z 16 320-339 96 96 96 171 171 171 F F F
117342024-04-17T17:17:54.370Z 17 340-359 93 93 93 165 165 165 F F F
117352024-04-17T17:17:54.370Z 18 360-379 108 108 98 197 197 173 F F F <---
117362024-04-17T17:17:54.370Z 19 380-399 108 108 106 197 197 191 F F F <---
117372024-04-17T17:17:54.370Z 20 400-419 97 97 97 172 172 172 T T F <---
117382024-04-17T17:17:54.370Z 21 420-439 101 101 101 184 184 184 T T F <---
117392024-04-17T17:17:54.370Z 22 440-459 106 106 106 192 192 192 T T F <---
117402024-04-17T17:17:54.370Z 23 460-479 108 108 108 194 194 194 F F F
117412024-04-17T17:17:54.370Z 24 480-499 108 108 106 196 196 192 F F F <---
117422024-04-17T17:17:54.370Z 25 500-519 106 106 106 192 192 192 F F F
117432024-04-17T17:17:54.370Z 26 520-539 108 108 103 197 197 186 F F F <---
117442024-04-17T17:17:54.370Z 27 540-559 108 108 103 195 195 186 F F F <---
117452024-04-17T17:17:54.370Z 28 560-579 106 106 106 192 192 192 F F F
117462024-04-17T17:17:54.370Z 29 580-599 108 108 98 195 195 173 F F F <---
117472024-04-17T17:17:54.370ZMax gen: 108, Max flush: 197
117482024-04-17T17:17:54.370ZError: Difference in extent metadata found!
117492024-04-17T17:17:54.370ZA Difference in extent metadata is expected here
117502024-04-17T17:17:54.370ZOn loop 54
117512024-04-17T17:17:54.370Z
117522024-04-17T17:17:54.373ZVerifying data now
117532024-04-17T17:18:09.033ZNo change after verify
117542024-04-17T17:18:09.033ZLoop: 54 Downstairs dump after verify (and repair):
117552024-04-17T17:18:09.048ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
117562024-04-17T17:18:09.048Z 0 000-019 106 106 106 191 191 191 F F F
117572024-04-17T17:18:09.052Z 1 020-039 109 109 109 198 198 198 F F F
117582024-04-17T17:18:09.052Z 2 040-059 106 106 106 191 191 191 F F F
117592024-04-17T17:18:09.052Z 3 060-079 106 106 106 192 192 192 F F F
117602024-04-17T17:18:09.052Z 4 080-099 106 106 106 192 192 192 F F F
117612024-04-17T17:18:09.052Z 5 100-119 97 97 97 172 172 172 F F F
117622024-04-17T17:18:09.052Z 6 120-139 98 98 98 173 173 173 F F F
117632024-04-17T17:18:09.052Z 7 140-159 104 104 104 188 188 188 F F F
117642024-04-17T17:18:09.052Z 8 160-179 100 100 100 182 182 182 F F F
117652024-04-17T17:18:09.052Z 9 180-199 109 109 109 198 198 198 F F F
117662024-04-17T17:18:09.053Z 10 200-219 109 109 109 198 198 198 F F F
117672024-04-17T17:18:09.053Z 11 220-239 106 106 106 192 192 192 F F F
117682024-04-17T17:18:09.053Z 12 240-259 108 108 108 195 195 195 F F F
117692024-04-17T17:18:09.053Z 13 260-279 104 104 104 188 188 188 F F F
117702024-04-17T17:18:09.053Z 14 280-299 104 104 104 188 188 188 F F F
117712024-04-17T17:18:09.053Z 15 300-319 108 108 108 196 196 196 F F F
117722024-04-17T17:18:09.053Z 16 320-339 96 96 96 171 171 171 F F F
117732024-04-17T17:18:09.053Z 17 340-359 93 93 93 165 165 165 F F F
117742024-04-17T17:18:09.053Z 18 360-379 108 108 108 197 197 197 F F F
117752024-04-17T17:18:09.053Z 19 380-399 108 108 108 197 197 197 F F F
117762024-04-17T17:18:09.053Z 20 400-419 109 109 109 198 198 198 F F F
117772024-04-17T17:18:09.053Z 21 420-439 109 109 109 198 198 198 F F F
117782024-04-17T17:18:09.053Z 22 440-459 109 109 109 198 198 198 F F F
117792024-04-17T17:18:09.053Z 23 460-479 108 108 108 194 194 194 F F F
117802024-04-17T17:18:09.053Z 24 480-499 108 108 108 196 196 196 F F F
117812024-04-17T17:18:09.054Z 25 500-519 106 106 106 192 192 192 F F F
117822024-04-17T17:18:09.054Z 26 520-539 108 108 108 197 197 197 F F F
117832024-04-17T17:18:09.054Z 27 540-559 108 108 108 195 195 195 F F F
117842024-04-17T17:18:09.054Z 28 560-579 106 106 106 192 192 192 F F F
117852024-04-17T17:18:09.054Z 29 580-599 108 108 108 195 195 195 F F F
117862024-04-17T17:18:09.054ZMax gen: 109, Max flush: 198
117872024-04-17T17:18:09.054Z
117882024-04-17T17:18:09.054ZBegin loop 55
117892024-04-17T17:18:09.054ZDownstairs to restart: 2
117902024-04-17T17:18:09.070Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.679134911Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611}
117912024-04-17T17:18:09.071Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.679373925Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611}
117922024-04-17T17:18:09.080Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.679407887Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611}
117932024-04-17T17:18:09.080Z{"msg":"Crucible 0fd197dc-f808-4902-9e32-ddf02e8c7a57 has session id: acdbdac3-842d-4fb3-a984-0cc585b779a3","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.679441219Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
117942024-04-17T17:18:09.080Z{"msg":"Crucible runtime is spawned
117952024-04-17T17:18:09.080ZUpstairs opts: Upstairs UUID: 0fd197dc-f808-4902-9e32-ddf02e8c7a57, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.679511184Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
117962024-04-17T17:18:09.080Z{"msg":"Crucible stats registered with UUID: 0fd197dc-f808-4902-9e32-ddf02e8c7a57","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.679539035Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
117972024-04-17T17:18:09.080Z{"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.679558906Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611}
117982024-04-17T17:18:09.080Z{"msg":"0fd197dc-f808-4902-9e32-ddf02e8c7a57 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.679664203Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
117992024-04-17T17:18:09.080Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.679693955Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"client":"0","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118002024-04-17T17:18:09.080Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.679719416Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"client":"0","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118012024-04-17T17:18:09.080Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.679763249Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"client":"1","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118022024-04-17T17:18:09.080Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.67979123Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"client":"1","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118032024-04-17T17:18:09.080Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.679827813Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"client":"2","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118042024-04-17T17:18:09.080Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.679876306Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"client":"2","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118052024-04-17T17:18:09.080Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.679919888Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"":"io task","client":"0","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118062024-04-17T17:18:09.080Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.679994623Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"":"io task","client":"2","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118072024-04-17T17:18:09.080Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.680027565Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118082024-04-17T17:18:09.081Z{"msg":"ds_connection connected from Ok(127.0.0.1:59786)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.680053016Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"":"io task","client":"0","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118092024-04-17T17:18:09.081Z{"msg":"ds_connection connected from Ok(127.0.0.1:35273)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.68012083Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118102024-04-17T17:18:09.081Z{"msg":"ds_connection connected from Ok(127.0.0.1:43045)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.680180974Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"":"io task","client":"2","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118112024-04-17T17:18:09.081Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.680737898Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"client":"0","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118122024-04-17T17:18:09.081Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.68077384Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"client":"0","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118132024-04-17T17:18:09.081Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.680790171Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"client":"1","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118142024-04-17T17:18:09.081Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.680845824Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"client":"1","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118152024-04-17T17:18:09.081Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.680883767Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"client":"2","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118162024-04-17T17:18:09.081Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.680900748Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"client":"2","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118172024-04-17T17:18:09.081Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.681352275Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"client":"0","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118182024-04-17T17:18:09.081Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.681512185Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"client":"1","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118192024-04-17T17:18:09.081Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.681554397Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"client":"2","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118202024-04-17T17:18:09.081Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.681817153Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"client":"0","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118212024-04-17T17:18:09.081Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.681866956Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"client":"0","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118222024-04-17T17:18:09.081Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.681896598Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118232024-04-17T17:18:09.081Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.681944671Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"client":"1","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118242024-04-17T17:18:09.081Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.681976003Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"client":"1","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118252024-04-17T17:18:09.081Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.682003135Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118262024-04-17T17:18:09.081Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.682052567Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"client":"2","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118272024-04-17T17:18:09.081Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.682103161Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"client":"2","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118282024-04-17T17:18:09.082Z{"msg":"[0]R flush_numbers[0..12]: [191, 198, 191, 192, 192, 172, 173, 188, 182, 198, 198, 192]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.682143873Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"":"downstairs","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118292024-04-17T17:18:09.082Z{"msg":"[0]R generation[0..12]: [106, 109, 106, 106, 106, 97, 98, 104, 100, 109, 109, 106]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.682166194Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"":"downstairs","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118302024-04-17T17:18:09.082Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.682199106Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"":"downstairs","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118312024-04-17T17:18:09.082Z{"msg":"[1]R flush_numbers[0..12]: [191, 198, 191, 192, 192, 172, 173, 188, 182, 198, 198, 192]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.682231648Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
118322024-04-17T17:18:09.082Zacdbdac3-842d-4fb3-a984-0cc585b779a3"}
118332024-04-17T17:18:09.082Z{"msg":"[1]R generation[0..12]: [106, 109, 106, 106, 106, 97, 98, 104, 100, 109, 109, 106]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.682265691Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"":"downstairs","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118342024-04-17T17:18:09.082Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.682298633Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"":"downstairs","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118352024-04-17T17:18:09.082Z{"msg":"[2]R flush_numbers[0..12]: [191, 198, 191, 192, 192, 172, 173, 188, 182, 198, 198, 192]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.682331884Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"":"downstairs","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118362024-04-17T17:18:09.082Z{"msg":"[2]R generation[0..12]: [106, 109, 106, 106, 106, 97, 98, 104, 100, 109, 109, 106]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.682366877Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"":"downstairs","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118372024-04-17T17:18:09.082Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.682401409Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"":"downstairs","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118382024-04-17T17:18:09.082Z{"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.68242308Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"":"downstairs","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118392024-04-17T17:18:09.082Z{"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.682450832Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"":"downstairs","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118402024-04-17T17:18:09.082Z{"msg":"Next flush: 199","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.682464952Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"":"downstairs","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118412024-04-17T17:18:09.082Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.682494984Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"":"downstairs","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118422024-04-17T17:18:09.082Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.682511295Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118432024-04-17T17:18:09.082ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
118442024-04-17T17:18:09.082Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.682540667Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"client":"0","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118452024-04-17T17:18:09.082Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.682557368Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"client":"1","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118462024-04-17T17:18:09.082Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.68258665Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"client":"2","":"downstairs","session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118472024-04-17T17:18:09.082Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.682604291Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118482024-04-17T17:18:09.082Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.682631293Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118492024-04-17T17:18:09.082Z{"msg":"0fd197dc-f808-4902-9e32-ddf02e8c7a57 is now active with session: acdbdac3-842d-4fb3-a984-0cc585b779a3","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.682649494Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118502024-04-17T17:18:09.082Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.682675805Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611,"session_id":"acdbdac3-842d-4fb3-a984-0cc585b779a3"}
118512024-04-17T17:18:09.083Z{"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:07.682692346Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611}
118522024-04-17T17:18:09.083ZLoading write count information from file "/tmp/test_repair_verify.data"
118532024-04-17T17:18:09.083ZRun Repair workload
118542024-04-17T17:18:09.083Z01/30 Write block 528 len 16384 data: 9 10 10 9
118552024-04-17T17:18:09.089Z02/30 Read block 160 len 16384
118562024-04-17T17:18:09.089Z03/30 Read block 414 len 28672
118572024-04-17T17:18:09.089Z04/30 Read block 420 len 40960
118582024-04-17T17:18:09.089Z05/30 Read block 175 len 8192
118592024-04-17T17:18:09.089Z06/30 Write block 316 len 28672 data: 9 10 10 9 7 6 4
118602024-04-17T17:18:09.089Z07/30 Read block 73 len 36864
118612024-04-17T17:18:09.089Z08/30 Write block 422 len 24576 data: 8 7 8 8 8 8
118622024-04-17T17:18:09.089Z09/30 Write block 344 len 16384 data: 11 12 11 10
118632024-04-17T17:18:09.089Z10/30 Flush
118642024-04-17T17:18:09.089Z11/30 Write block 53 len 40960 data: 8 12 10 10 12 12 12 13 11 10
118652024-04-17T17:18:09.089Z12/30 Read block 47 len 4096
118662024-04-17T17:18:09.093Z13/30 Read block 342 len 8192
118672024-04-17T17:18:09.093Z14/30 Write block 159 len 40960 data: 10 9 10 9 11 8 6 8 9 8
118682024-04-17T17:18:09.094Z15/30 Read block 288 len 24576
118692024-04-17T17:18:09.094Z16/30 Write block 564 len 28672 data: 11 8 8 8 8 7 5
118702024-04-17T17:18:09.094Z17/30 Write block 487 len 32768 data: 12 12 10 10 6 7 6 6
118712024-04-17T17:18:09.098Z18/30 Write block 207 len 40960 data: 15 7 8 8 9 9 9 9 11 9
118722024-04-17T17:18:09.098Z19/30 Read block 212 len 16384
118732024-04-17T17:18:09.098Z20/30 Write block 368 len 36864 data: 9 9 8 10 7 8 7 8 6
118742024-04-17T17:18:09.098Z21/30 Read block 278 len 8192
118752024-04-17T17:18:09.102Z22/30 Write block 551 len 8192 data: 11 12
118762024-04-17T17:18:09.102Z23/30 Flush
118772024-04-17T17:18:09.105Z24/30 Write block 12 len 8192 data: 7 8
118782024-04-17T17:18:09.105Z25/30 Write block 17 len 28672 data: 7 6 7 8 11 12 11
118792024-04-17T17:18:09.109Z26/30 Write block 209 len 40960 data: 9 9 10 10 10 10 12 10 7 7
118802024-04-17T17:18:09.109Z27/30 Read block 19 len 36864
118812024-04-17T17:18:09.109Z28/30 Write block 431 len 12288 data: 6 6 9
118822024-04-17T17:18:09.109Z29/30 Read block 162 len 32768
118832024-04-17T17:18:09.113Z30/30 Read block 1 len 32768
118842024-04-17T17:18:09.113Z----------------------------------------------------------------
118852024-04-17T17:18:09.113Z Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30
118862024-04-17T17:18:09.113ZStates: Active Active Active
118872024-04-17T17:18:09.113ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
118882024-04-17T17:18:09.113Z 1 Acked 1000 Write 4 Done Done Sent false
118892024-04-17T17:18:09.113Z 2 Acked 1001 Read 4 Done Done Sent false
118902024-04-17T17:18:09.113Z 3 Acked 1002 Read 7 Done Done Sent false
118912024-04-17T17:18:09.114Z 4 Acked 1003 Read 10 Done Done Sent false
118922024-04-17T17:18:09.114Z 5 Acked 1004 Read 2 Done Done Sent false
118932024-04-17T17:18:09.114Z 6 Acked 1005 Write 7 Done Done Sent false
118942024-04-17T17:18:09.114Z 7 Acked 1006 Read 9 Done Done Sent false
118952024-04-17T17:18:09.114Z 8 Acked 1007 Write 6 Done Done Sent false
118962024-04-17T17:18:09.114Z 9 Acked 1008 Write 4 Done Done Sent false
118972024-04-17T17:18:09.114Z 10 Acked 1009 Flush 0 Done Done Sent false
118982024-04-17T17:18:09.114Z 11 Acked 1010 Write 10 Done Done Sent false
118992024-04-17T17:18:09.114Z 12 Acked 1011 Read 1 Done Done Sent false
119002024-04-17T17:18:09.114Z 13 Acked 1012 Read 2 Done Done Sent false
119012024-04-17T17:18:09.114Z 14 Acked 1013 Write 10 Done Done Sent false
119022024-04-17T17:18:09.114Z 15 Acked 1014 Read 6 Done Done Sent false
119032024-04-17T17:18:09.114Z 16 Acked 1015 Write 7 Done Done Sent false
119042024-04-17T17:18:09.114Z 17 Acked 1016 Write 8 Done Done Sent false
119052024-04-17T17:18:09.114Z 18 Acked 1017 Write 10 Done Done Sent false
119062024-04-17T17:18:09.114Z 19 Acked 1018 Read 4 Done Done Sent false
119072024-04-17T17:18:09.114Z 20 Acked 1019 Write 9 Done Done Sent false
119082024-04-17T17:18:09.114Z 21 Acked 1020 Read 2 Done Done Sent false
119092024-04-17T17:18:09.114Z 22 Acked 1021 Write 2 Done Done Sent false
119102024-04-17T17:18:09.114Z 23 Acked 1022 Flush 0 Done Done Sent false
119112024-04-17T17:18:09.114Z 24 Acked 1023 Write 2 Done Done Sent false
119122024-04-17T17:18:09.115Z 25 Acked 1024 Write 7 Done Done Sent false
119132024-04-17T17:18:09.115Z 26 Acked 1025 Write 10 Done Done Sent false
119142024-04-17T17:18:09.115Z 27 Acked 1026 Read 9 Done Done Sent false
119152024-04-17T17:18:09.115Z 28 Acked 1027 Write 3 Done Done Sent false
119162024-04-17T17:18:09.115Z 29 Acked 1028 Read 8 Done Done Sent false
119172024-04-17T17:18:09.115Z 30 Acked 1029 Read 8 Done Done Sent false
119182024-04-17T17:18:09.115Z STATES DS:0 DS:1 DS:2 TOTAL
119192024-04-17T17:18:09.115Z New 0 0 0 0
119202024-04-17T17:18:09.115Z Sent 0 0 30 30
119212024-04-17T17:18:09.115Z Done 30 30 0 60
119222024-04-17T17:18:09.115Z Skipped 0 0 0 0
119232024-04-17T17:18:09.115Z Error 0 0 0 0
119242024-04-17T17:18:09.115ZLast Flush: 1022 1022 0
119252024-04-17T17:18:09.115ZDownstairs last five completed:
119262024-04-17T17:18:09.115ZUpstairs last five completed: 30 29 28 27 26
119272024-04-17T17:18:09.115Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:18:07.719426203Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1611}
119282024-04-17T17:18:09.115ZWrote out file "/tmp/test_repair_verify.data"
119292024-04-17T17:18:09.119Z
119302024-04-17T17:18:09.119ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
119312024-04-17T17:18:09.134ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
119322024-04-17T17:18:09.134Z 0 000-019 106 106 106 191 191 191 T T F <---
119332024-04-17T17:18:09.134Z 1 020-039 109 109 109 198 198 198 T T F <---
119342024-04-17T17:18:09.134Z 2 040-059 110 110 106 200 200 191 F F F <---
119352024-04-17T17:18:09.134Z 3 060-079 110 110 106 200 200 192 F F F <---
119362024-04-17T17:18:09.135Z 4 080-099 106 106 106 192 192 192 F F F
119372024-04-17T17:18:09.135Z 5 100-119 97 97 97 172 172 172 F F F
119382024-04-17T17:18:09.135Z 6 120-139 98 98 98 173 173 173 F F F
119392024-04-17T17:18:09.135Z 7 140-159 110 110 104 200 200 188 F F F <---
119402024-04-17T17:18:09.135Z 8 160-179 110 110 100 200 200 182 F F F <---
119412024-04-17T17:18:09.135Z 9 180-199 109 109 109 198 198 198 F F F
119422024-04-17T17:18:09.135Z 10 200-219 110 110 109 200 200 198 T T F <---
119432024-04-17T17:18:09.135Z 11 220-239 106 106 106 192 192 192 F F F
119442024-04-17T17:18:09.139Z 12 240-259 108 108 108 195 195 195 F F F
119452024-04-17T17:18:09.139Z 13 260-279 104 104 104 188 188 188 F F F
119462024-04-17T17:18:09.139Z 14 280-299 104 104 104 188 188 188 F F F
119472024-04-17T17:18:09.139Z 15 300-319 110 110 108 199 199 196 F F F <---
119482024-04-17T17:18:09.139Z 16 320-339 110 110 96 199 199 171 F F F <---
119492024-04-17T17:18:09.139Z 17 340-359 110 110 93 199 199 165 F F F <---
119502024-04-17T17:18:09.139Z 18 360-379 110 110 108 200 200 197 F F F <---
119512024-04-17T17:18:09.139Z 19 380-399 108 108 108 197 197 197 F F F
119522024-04-17T17:18:09.139Z 20 400-419 109 109 109 198 198 198 F F F
119532024-04-17T17:18:09.139Z 21 420-439 110 110 109 199 199 198 T T F <---
119542024-04-17T17:18:09.139Z 22 440-459 109 109 109 198 198 198 F F F
119552024-04-17T17:18:09.139Z 23 460-479 108 108 108 194 194 194 F F F
119562024-04-17T17:18:09.139Z 24 480-499 110 110 108 200 200 196 F F F <---
119572024-04-17T17:18:09.140Z 25 500-519 106 106 106 192 192 192 F F F
119582024-04-17T17:18:09.140Z 26 520-539 110 110 108 199 199 197 F F F <---
119592024-04-17T17:18:09.140Z 27 540-559 110 110 108 200 200 195 F F F <---
119602024-04-17T17:18:09.140Z 28 560-579 110 110 106 200 200 192 F F F <---
119612024-04-17T17:18:09.140Z 29 580-599 108 108 108 195 195 195 F F F
119622024-04-17T17:18:09.140ZMax gen: 110, Max flush: 200
119632024-04-17T17:18:09.140ZError: Difference in extent metadata found!
119642024-04-17T17:18:09.140ZA Difference in extent metadata is expected here
119652024-04-17T17:18:09.140ZOn loop 55
119662024-04-17T17:18:09.140Z
119672024-04-17T17:18:09.145ZVerifying data now
119682024-04-17T17:18:13.865ZNo change after verify
119692024-04-17T17:18:13.865ZLoop: 55 Downstairs dump after verify (and repair):
119702024-04-17T17:18:13.880ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
119712024-04-17T17:18:13.880Z 0 000-019 111 111 111 201 201 201 F F F
119722024-04-17T17:18:13.884Z 1 020-039 111 111 111 201 201 201 F F F
119732024-04-17T17:18:13.884Z 2 040-059 110 110 110 200 200 200 F F F
119742024-04-17T17:18:13.884Z 3 060-079 110 110 110 200 200 200 F F F
119752024-04-17T17:18:13.884Z 4 080-099 106 106 106 192 192 192 F F F
119762024-04-17T17:18:13.884Z 5 100-119 97 97 97 172 172 172 F F F
119772024-04-17T17:18:13.884Z 6 120-139 98 98 98 173 173 173 F F F
119782024-04-17T17:18:13.884Z 7 140-159 110 110 110 200 200 200 F F F
119792024-04-17T17:18:13.884Z 8 160-179 110 110 110 200 200 200 F F F
119802024-04-17T17:18:13.884Z 9 180-199 109 109 109 198 198 198 F F F
119812024-04-17T17:18:13.885Z 10 200-219 111 111 111 201 201 201 F F F
119822024-04-17T17:18:13.885Z 11 220-239 106 106 106 192 192 192 F F F
119832024-04-17T17:18:13.885Z 12 240-259 108 108 108 195 195 195 F F F
119842024-04-17T17:18:13.885Z 13 260-279 104 104 104 188 188 188 F F F
119852024-04-17T17:18:13.885Z 14 280-299 104 104 104 188 188 188 F F F
119862024-04-17T17:18:13.885Z 15 300-319 110 110 110 199 199 199 F F F
119872024-04-17T17:18:13.885Z 16 320-339 110 110 110 199 199 199 F F F
119882024-04-17T17:18:13.885Z 17 340-359 110 110 110 199 199 199 F F F
119892024-04-17T17:18:13.885Z 18 360-379 110 110 110 200 200 200 F F F
119902024-04-17T17:18:13.885Z 19 380-399 108 108 108 197 197 197 F F F
119912024-04-17T17:18:13.885Z 20 400-419 109 109 109 198 198 198 F F F
119922024-04-17T17:18:13.885Z 21 420-439 111 111 111 201 201 201 F F F
119932024-04-17T17:18:13.885Z 22 440-459 109 109 109 198 198 198 F F F
119942024-04-17T17:18:13.885Z 23 460-479 108 108 108 194 194 194 F F F
119952024-04-17T17:18:13.885Z 24 480-499 110 110 110 200 200 200 F F F
119962024-04-17T17:18:13.885Z 25 500-519 106 106 106 192 192 192 F F F
119972024-04-17T17:18:13.885Z 26 520-539 110 110 110 199 199 199 F F F
119982024-04-17T17:18:13.885Z 27 540-559 110 110 110 200 200 200 F F F
119992024-04-17T17:18:13.885Z 28 560-579 110 110 110 200 200 200 F F F
120002024-04-17T17:18:13.885Z 29 580-599 108 108 108 195 195 195 F F F
120012024-04-17T17:18:13.885ZMax gen: 111, Max flush: 201
120022024-04-17T17:18:13.885Z
120032024-04-17T17:18:13.885ZBegin loop 56
120042024-04-17T17:18:13.885ZDownstairs to restart: 1
120052024-04-17T17:18:13.903Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.51152863Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619}
120062024-04-17T17:18:13.910Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.511794936Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619}
120072024-04-17T17:18:13.910Z{"msg":"Crucible runtime is spawned
120082024-04-17T17:18:13.910ZUpstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.511845999Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619}
120092024-04-17T17:18:13.910Z{"msg":"Crucible b83799fd-3cb3-40c1-b96d-b3b25728bcd3 has session id: 9057a055-3be1-44f1-bb3c-552537351dd1","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.511891792Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120102024-04-17T17:18:13.910Z{"msg":"Upstairs opts: Upstairs UUID: b83799fd-3cb3-40c1-b96d-b3b25728bcd3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.511948565Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120112024-04-17T17:18:13.910Z{"msg":"Crucible stats registered with UUID: b83799fd-3cb3-40c1-b96d-b3b25728bcd3","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.511970497Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120122024-04-17T17:18:13.910Z{"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.512014739Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619}
120132024-04-17T17:18:13.910Z{"msg":"b83799fd-3cb3-40c1-b96d-b3b25728bcd3 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.51202998Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120142024-04-17T17:18:13.910Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.512064972Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"client":"0","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120152024-04-17T17:18:13.910Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.512081153Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"client":"0","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120162024-04-17T17:18:13.910Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.512133566Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"client":"1","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120172024-04-17T17:18:13.910Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.512155568Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"client":"1","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120182024-04-17T17:18:13.911Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.51218611Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"client":"2","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120192024-04-17T17:18:13.911Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.51220226Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"client":"2","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120202024-04-17T17:18:13.911Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.512233802Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120212024-04-17T17:18:13.911Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.512263914Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"":"io task","client":"0","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120222024-04-17T17:18:13.911Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.512342509Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"":"io task","client":"2","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120232024-04-17T17:18:13.911Z{"msg":"ds_connection connected from Ok(127.0.0.1:39004)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.512427314Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"":"io task","client":"2","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120242024-04-17T17:18:13.911Z{"msg":"ds_connection connected from Ok(127.0.0.1:59135)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.512483077Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"":"io task","client":"0","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120252024-04-17T17:18:13.911Z{"msg":"ds_connection connected from Ok(127.0.0.1:60479)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.512508269Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120262024-04-17T17:18:13.911Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.51318198Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"client":"2","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120272024-04-17T17:18:13.911Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.513228753Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"client":"2","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120282024-04-17T17:18:13.911Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.513263295Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"client":"0","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120292024-04-17T17:18:13.911Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.513303487Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"client":"0","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120302024-04-17T17:18:13.911Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.513633737Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"client":"1","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120312024-04-17T17:18:13.911Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.51368659Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"client":"1","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120322024-04-17T17:18:13.911Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.513829929Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"client":"2","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120332024-04-17T17:18:13.911Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.513869041Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"client":"0","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120342024-04-17T17:18:13.911Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.514190461Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"client":"1","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120352024-04-17T17:18:13.911Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.514455307Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"client":"2","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120362024-04-17T17:18:13.911Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.5144987Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"client":"2","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120372024-04-17T17:18:13.911Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.514523851Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120382024-04-17T17:18:13.911Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.514554993Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"client":"0","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120392024-04-17T17:18:13.911Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.514572604Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"client":"0","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120402024-04-17T17:18:13.911Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.514604306Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120412024-04-17T17:18:13.912Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.514626678Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"client":"1","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120422024-04-17T17:18:13.912Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.514659189Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"client":"1","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120432024-04-17T17:18:13.912Z{"msg":"[0]R flush_numbers[0..12]: [201, 201, 200, 200, 192, 172, 173, 200, 200, 198, 201, 192]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.514682971Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"":"downstairs","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120442024-04-17T17:18:13.912Z{"msg":"[0]R generation[0..12]: [111, 111, 110, 110, 106, 97, 98, 110, 110, 109, 111, 106]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.514717263Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"":"downstairs","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120452024-04-17T17:18:13.912Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.514741585Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"":"downstairs","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120462024-04-17T17:18:13.912Z{"msg":"[1]R flush_numbers[0..12]: [201, 201, 200, 200, 192, 172, 173, 200, 200, 198, 201, 192]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.514755925Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"":"downstairs","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120472024-04-17T17:18:13.912Z{"msg":"[1]R generation[0..12]: [111, 111, 110, 110, 106, 97, 98, 110, 110, 109, 111, 106]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.514775987Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"":"downstairs","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120482024-04-17T17:18:13.912Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.514790777Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"":"downstairs"Wait for a query_work_queue command to finish before sending IO
120492024-04-17T17:18:13.912Z,"":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120502024-04-17T17:18:13.912Z{"msg":"[2]R flush_numbers[0..12]: [201, 201, 200, 200, 192, 172, 173, 200, 200, 198, 201, 192]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.514815679Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"":"downstairs","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120512024-04-17T17:18:13.912Z{"msg":"[2]R generation[0..12]: [111, 111, 110, 110, 106, 97, 98, 110, 110, 109, 111, 106]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.51482906Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"":"downstairs","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120522024-04-17T17:18:13.912Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.514843711Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"":"downstairs","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120532024-04-17T17:18:13.912Z{"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.514856881Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"":"downstairs","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120542024-04-17T17:18:13.912Z{"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.514870352Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"":"downstairs","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120552024-04-17T17:18:13.912Z{"msg":"Next flush: 202","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.514883103Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"":"downstairs","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120562024-04-17T17:18:13.912Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.514904534Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"":"downstairs","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120572024-04-17T17:18:13.912Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.514918075Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120582024-04-17T17:18:13.912Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.514930386Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"client":"0","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120592024-04-17T17:18:13.912Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.514950267Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"client":"1","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120602024-04-17T17:18:13.912Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.514963508Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"client":"2","":"downstairs","session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120612024-04-17T17:18:13.912Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.514976319Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120622024-04-17T17:18:13.912Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.51499544Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120632024-04-17T17:18:13.912Z{"msg":"b83799fd-3cb3-40c1-b96d-b3b25728bcd3 is now active with session: 9057a055-3be1-44f1-bb3c-552537351dd1","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.515009021Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120642024-04-17T17:18:13.913Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.515021081Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619,"session_id":"9057a055-3be1-44f1-bb3c-552537351dd1"}
120652024-04-17T17:18:13.913Z{"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:12.515032922Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619}
120662024-04-17T17:18:13.913ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
120672024-04-17T17:18:13.913ZLoading write count information from file "/tmp/test_repair_verify.data"
120682024-04-17T17:18:13.913ZRun Repair workload
120692024-04-17T17:18:13.925Z01/30 Write block 487 len 36864 data: 13 13 11 11 7 8 7 7 8
120702024-04-17T17:18:13.925Z02/30 Read block 146 len 28672
120712024-04-17T17:18:13.925Z03/30 Write block 74 len 28672 data: 6 7 8 8 7 6 8
120722024-04-17T17:18:13.925Z04/30 Read block 374 len 40960
120732024-04-17T17:18:13.925Z05/30 Write block 85 len 16384 data: 10 6 10 9
120742024-04-17T17:18:13.925Z06/30 Write block 456 len 20480 data: 6 6 8 9 10
120752024-04-17T17:18:13.925Z07/30 Read block 383 len 24576
120762024-04-17T17:18:13.925Z08/30 Flush
120772024-04-17T17:18:13.925Z09/30 Write block 482 len 8192 data: 14 16
120782024-04-17T17:18:13.926Z10/30 Read block 219 len 28672
120792024-04-17T17:18:13.926Z11/30 Write block 279 len 40960 data: 7 7 5 5 5 6 5 4 5 6
120802024-04-17T17:18:13.926Z12/30 Write block 319 len 12288 data: 10 8 7
120812024-04-17T17:18:13.926Z13/30 Write block 474 len 36864 data: 10 9 10 14 16 15 15 14 15
120822024-04-17T17:18:13.926Z14/30 Write block 567 len 32768 data: 9 9 8 6 6 8 9 11
120832024-04-17T17:18:13.926Z15/30 Flush
120842024-04-17T17:18:13.930Z16/30 Write block 479 len 20480 data: 16 16 15 16 17
120852024-04-17T17:18:13.930Z17/30 Flush
120862024-04-17T17:18:13.934Z18/30 Write block 14 len 40960 data: 10 7 5 8 7 8 9 12 13 12
120872024-04-17T17:18:13.934Z19/30 Read block 476 len 4096
120882024-04-17T17:18:13.934Z20/30 Write block 469 len 36864 data: 9 10 11 10 10 11 10 11 15
120892024-04-17T17:18:13.934Z21/30 Flush
120902024-04-17T17:18:13.937Z22/30 Write block 28 len 12288 data: 7 11 9
120912024-04-17T17:18:13.938Z23/30 Read block 77 len 24576
120922024-04-17T17:18:13.941Z24/30 Write block 576 len 36864 data: 11 10 9 11 9 10 10 10 11
120932024-04-17T17:18:13.941Z25/30 Write block 314 len 24576 data: 12 11 10 11 11 11
120942024-04-17T17:18:13.941Z26/30 Read block 434 len 32768
120952024-04-17T17:18:13.941Z27/30 Read block 208 len 40960
120962024-04-17T17:18:13.946Z28/30 Read block 187 len 16384
120972024-04-17T17:18:13.946Z29/30 Write block 555 len 40960 data: 10 9 6 7 8 10 10 11 10 12
120982024-04-17T17:18:13.946Z30/30 Read block 22 len 8192
120992024-04-17T17:18:13.946Z----------------------------------------------------------------
121002024-04-17T17:18:13.946Z Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30
121012024-04-17T17:18:13.946ZStates: Active Active Active
121022024-04-17T17:18:13.946ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
121032024-04-17T17:18:13.946Z 1 Acked 1000 Write 9 Done Done Done false
121042024-04-17T17:18:13.946Z 2 Acked 1001 Read 7 Done Sent Done false
121052024-04-17T17:18:13.946Z 3 Acked 1002 Write 7 Done Sent Done false
121062024-04-17T17:18:13.946Z 4 Acked 1003 Read 10 Done Sent Done false
121072024-04-17T17:18:13.946Z 5 Acked 1004 Write 4 Done Sent Done false
121082024-04-17T17:18:13.946Z 6 Acked 1005 Write 5 Done Sent Done false
121092024-04-17T17:18:13.946Z 7 Acked 1006 Read 6 Done Sent Done false
121102024-04-17T17:18:13.946Z 8 Acked 1007 Flush 0 Done Sent Done false
121112024-04-17T17:18:13.946Z 9 Acked 1008 Write 2 Done Sent Done false
121122024-04-17T17:18:13.946Z 10 Acked 1009 Read 7 Done Sent Done false
121132024-04-17T17:18:13.946Z 11 Acked 1010 Write 10 Done Sent Done false
121142024-04-17T17:18:13.946Z 12 Acked 1011 Write 3 Done Sent Done false
121152024-04-17T17:18:13.946Z 13 Acked 1012 Write 9 Done Sent Done false
121162024-04-17T17:18:13.946Z 14 Acked 1013 Write 8 Done Sent Done false
121172024-04-17T17:18:13.947Z 15 Acked 1014 Flush 0 Done Sent Done false
121182024-04-17T17:18:13.947Z 16 Acked 1015 Write 5 Done Sent Done false
121192024-04-17T17:18:13.947Z 17 Acked 1016 Flush 0 Done Sent Done false
121202024-04-17T17:18:13.947Z 18 Acked 1017 Write 10 Done Sent Done false
121212024-04-17T17:18:13.947Z 19 Acked 1018 Read 1 Done Sent Done false
121222024-04-17T17:18:13.947Z 20 Acked 1019 Write 9 Done Sent Done false
121232024-04-17T17:18:13.947Z 21 Acked 1020 Flush 0 Done Sent Done false
121242024-04-17T17:18:13.947Z 22 Acked 1021 Write 3 Done Sent Done false
121252024-04-17T17:18:13.947Z 23 Acked 1022 Read 6 Done Sent Done false
121262024-04-17T17:18:13.947Z 24 Acked 1023 Write 9 Done Sent Done false
121272024-04-17T17:18:13.947Z 25 Acked 1024 Write 6 Done Sent Done false
121282024-04-17T17:18:13.947Z 26 Acked 1025 Read 8 Done Sent Done false
121292024-04-17T17:18:13.947Z 27 Acked 1026 Read 10 Done Sent Done false
121302024-04-17T17:18:13.947Z 28 Acked 1027 Read 4 Done Sent Done false
121312024-04-17T17:18:13.947Z 29 Acked 1028 Write 10 Done Sent Done false
121322024-04-17T17:18:13.947Z 30 Acked 1029 Read 2 Sent Sent Done false
121332024-04-17T17:18:13.947Z STATES DS:0 DS:1 DS:2 TOTAL
121342024-04-17T17:18:13.947Z New 0 0 0 0
121352024-04-17T17:18:13.947Z Sent 1 29 0 30
121362024-04-17T17:18:13.947Z Done 29 1 30 60
121372024-04-17T17:18:13.947Z Skipped 0 0 0 0
121382024-04-17T17:18:13.947Z Error 0 0 0 0
121392024-04-17T17:18:13.947ZLast Flush: 1020 0 1020
121402024-04-17T17:18:13.947ZDownstairs last five completed:
121412024-04-17T17:18:13.948ZUpstairs last five completed: 30 29 28 27 26
121422024-04-17T17:18:13.948Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:18:12.553389247Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1619}
121432024-04-17T17:18:13.948ZWrote out file "/tmp/test_repair_verify.data"
121442024-04-17T17:18:13.951Z
121452024-04-17T17:18:13.956ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
121462024-04-17T17:18:13.968ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
121472024-04-17T17:18:13.968Z 0 000-019 112 111 112 205 201 205 F F F <---
121482024-04-17T17:18:13.972Z 1 020-039 112 111 112 205 201 205 T F T <---
121492024-04-17T17:18:13.972Z 2 040-059 110 110 110 200 200 200 F F F
121502024-04-17T17:18:13.973Z 3 060-079 112 110 112 202 200 202 F F F <---
121512024-04-17T17:18:13.973Z 4 080-099 112 106 112 202 192 202 F F F <---
121522024-04-17T17:18:13.973Z 5 100-119 97 97 97 172 172 172 F F F
121532024-04-17T17:18:13.973Z 6 120-139 98 98 98 173 173 173 F F F
121542024-04-17T17:18:13.973Z 7 140-159 110 110 110 200 200 200 F F F
121552024-04-17T17:18:13.973Z 8 160-179 110 110 110 200 200 200 F F F
121562024-04-17T17:18:13.973Z 9 180-199 109 109 109 198 198 198 F F F
121572024-04-17T17:18:13.973Z 10 200-219 111 111 111 201 201 201 F F F
121582024-04-17T17:18:13.973Z 11 220-239 106 106 106 192 192 192 F F F
121592024-04-17T17:18:13.973Z 12 240-259 108 108 108 195 195 195 F F F
121602024-04-17T17:18:13.973Z 13 260-279 112 104 112 203 188 203 F F F <---
121612024-04-17T17:18:13.973Z 14 280-299 112 104 112 203 188 203 F F F <---
121622024-04-17T17:18:13.973Z 15 300-319 112 110 112 203 199 203 T F T <---
121632024-04-17T17:18:13.973Z 16 320-339 112 110 112 203 199 203 F F F <---
121642024-04-17T17:18:13.973Z 17 340-359 110 110 110 199 199 199 F F F
121652024-04-17T17:18:13.973Z 18 360-379 110 110 110 200 200 200 F F F
121662024-04-17T17:18:13.973Z 19 380-399 108 108 108 197 197 197 F F F
121672024-04-17T17:18:13.973Z 20 400-419 109 109 109 198 198 198 F F F
121682024-04-17T17:18:13.973Z 21 420-439 111 111 111 201 201 201 F F F
121692024-04-17T17:18:13.973Z 22 440-459 112 109 112 202 198 202 F F F <---
121702024-04-17T17:18:13.973Z 23 460-479 112 108 112 205 194 205 F F F <---
121712024-04-17T17:18:13.973Z 24 480-499 112 110 112 204 200 204 F T F <---
121722024-04-17T17:18:13.974Z 25 500-519 106 106 106 192 192 192 F F F
121732024-04-17T17:18:13.974Z 26 520-539 110 110 110 199 199 199 F F F
121742024-04-17T17:18:13.974Z 27 540-559 110 110 110 200 200 200 T F T <---
121752024-04-17T17:18:13.974Z 28 560-579 112 110 112 203 200 203 T F T <---
121762024-04-17T17:18:13.974Z 29 580-599 108 108 108 195 195 195 T F T <---
121772024-04-17T17:18:13.974ZMax gen: 112, Max flush: 205
121782024-04-17T17:18:13.974ZError: Difference in extent metadata found!
121792024-04-17T17:18:13.974ZA Difference in extent metadata is expected here
121802024-04-17T17:18:13.974ZOn loop 56
121812024-04-17T17:18:13.974Z
121822024-04-17T17:18:13.977ZVerifying data now
121832024-04-17T17:18:18.605ZNo change after verify
121842024-04-17T17:18:18.605ZLoop: 56 Downstairs dump after verify (and repair):
121852024-04-17T17:18:18.621ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
121862024-04-17T17:18:18.622Z 0 000-019 112 112 112 205 205 205 F F F
121872024-04-17T17:18:18.625Z 1 020-039 113 113 113 206 206 206 F F F
121882024-04-17T17:18:18.626Z 2 040-059 110 110 110 200 200 200 F F F
121892024-04-17T17:18:18.626Z 3 060-079 112 112 112 202 202 202 F F F
121902024-04-17T17:18:18.626Z 4 080-099 112 112 112 202 202 202 F F F
121912024-04-17T17:18:18.626Z 5 100-119 97 97 97 172 172 172 F F F
121922024-04-17T17:18:18.626Z 6 120-139 98 98 98 173 173 173 F F F
121932024-04-17T17:18:18.626Z 7 140-159 110 110 110 200 200 200 F F F
121942024-04-17T17:18:18.626Z 8 160-179 110 110 110 200 200 200 F F F
121952024-04-17T17:18:18.626Z 9 180-199 109 109 109 198 198 198 F F F
121962024-04-17T17:18:18.626Z 10 200-219 111 111 111 201 201 201 F F F
121972024-04-17T17:18:18.626Z 11 220-239 106 106 106 192 192 192 F F F
121982024-04-17T17:18:18.626Z 12 240-259 108 108 108 195 195 195 F F F
121992024-04-17T17:18:18.626Z 13 260-279 112 112 112 203 203 203 F F F
122002024-04-17T17:18:18.626Z 14 280-299 112 112 112 203 203 203 F F F
122012024-04-17T17:18:18.626Z 15 300-319 113 113 113 206 206 206 F F F
122022024-04-17T17:18:18.626Z 16 320-339 112 112 112 203 203 203 F F F
122032024-04-17T17:18:18.626Z 17 340-359 110 110 110 199 199 199 F F F
122042024-04-17T17:18:18.626Z 18 360-379 110 110 110 200 200 200 F F F
122052024-04-17T17:18:18.626Z 19 380-399 108 108 108 197 197 197 F F F
122062024-04-17T17:18:18.626Z 20 400-419 109 109 109 198 198 198 F F F
122072024-04-17T17:18:18.626Z 21 420-439 111 111 111 201 201 201 F F F
122082024-04-17T17:18:18.626Z 22 440-459 112 112 112 202 202 202 F F F
122092024-04-17T17:18:18.626Z 23 460-479 112 112 112 205 205 205 F F F
122102024-04-17T17:18:18.626Z 24 480-499 112 112 112 204 204 204 F F F
122112024-04-17T17:18:18.627Z 25 500-519 106 106 106 192 192 192 F F F
122122024-04-17T17:18:18.627Z 26 520-539 110 110 110 199 199 199 F F F
122132024-04-17T17:18:18.627Z 27 540-559 113 113 113 206 206 206 F F F
122142024-04-17T17:18:18.627Z 28 560-579 113 113 113 206 206 206 F F F
122152024-04-17T17:18:18.627Z 29 580-599 113 113 113 206 206 206 F F F
122162024-04-17T17:18:18.627ZMax gen: 113, Max flush: 206
122172024-04-17T17:18:18.627Z
122182024-04-17T17:18:18.627ZBegin loop 57
122192024-04-17T17:18:18.627ZDownstairs to restart: 2
122202024-04-17T17:18:18.644Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.252664587Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627}
122212024-04-17T17:18:18.644Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.252902512Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627}
122222024-04-17T17:18:18.644Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.252919693Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627}
122232024-04-17T17:18:18.651Z{"msg":"Crucible 7c8f2d68-85fe-46dc-8cd2-26f69a7179f9 has session id: c4b6ad68-f565-4e91-a4d5-a641f48ad58b","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.252944614Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122242024-04-17T17:18:18.651Z{"msg":"Upstairs opts: Upstairs UUID: 7c8f2d68-85fe-46dc-8cd2-26f69a7179f9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.253002078Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122252024-04-17T17:18:18.651ZCrucible runtime is spawned
122262024-04-17T17:18:18.651Z{"msg":"Crucible stats registered with UUID: 7c8f2d68-85fe-46dc-8cd2-26f69a7179f9","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.253051421Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122272024-04-17T17:18:18.651Z{"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.253091733Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627}
122282024-04-17T17:18:18.651Z{"msg":"7c8f2d68-85fe-46dc-8cd2-26f69a7179f9 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.253237152Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122292024-04-17T17:18:18.651Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.253265924Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"client":"0","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122302024-04-17T17:18:18.651Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.253282395Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"client":"0","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122312024-04-17T17:18:18.651Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.253295966Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"client":"1","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122322024-04-17T17:18:18.651Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.253309936Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"client":"1","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122332024-04-17T17:18:18.651Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.253326107Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"client":"2","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122342024-04-17T17:18:18.652Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.25336523Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"client":"2","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122352024-04-17T17:18:18.652Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.253389561Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"":"io task","client":"0","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122362024-04-17T17:18:18.652Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.253431924Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"":"io task","client":"2","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122372024-04-17T17:18:18.652Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.253459265Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"":"io task","client":"1","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122382024-04-17T17:18:18.652Z{"msg":"ds_connection connected from Ok(127.0.0.1:36836)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.253506818Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"":"io task","client":"1","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122392024-04-17T17:18:18.652Z{"msg":"ds_connection connected from Ok(127.0.0.1:52927)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.253551891Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"":"io task","client":"0","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122402024-04-17T17:18:18.652Z{"msg":"ds_connection connected from Ok(127.0.0.1:54159)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.253653307Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"":"io task","client":"2","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122412024-04-17T17:18:18.652Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.254296106Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"client":"0","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122422024-04-17T17:18:18.652Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.254336449Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"client":"0","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122432024-04-17T17:18:18.652Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.25435239Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"client":"1","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122442024-04-17T17:18:18.652Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.25436763Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"client":"1","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122452024-04-17T17:18:18.652Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.254583693Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"client":"2","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122462024-04-17T17:18:18.652Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.254617726Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"client":"2","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122472024-04-17T17:18:18.652Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.254995838Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"client":"0","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122482024-04-17T17:18:18.652Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.255032881Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"client":"1","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122492024-04-17T17:18:18.652Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.255253754Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"client":"2","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122502024-04-17T17:18:18.652Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.255547732Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"client":"1","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122512024-04-17T17:18:18.652Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.255603065Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"client":"1","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122522024-04-17T17:18:18.652Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.255629137Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122532024-04-17T17:18:18.652Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.255663039Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"client":"0","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122542024-04-17T17:18:18.652Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.255697291Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"client":"0","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122552024-04-17T17:18:18.652Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.255733673Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122562024-04-17T17:18:18.652Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.255772665Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"client":"2","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122572024-04-17T17:18:18.653Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.255809778Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"client":"2","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122582024-04-17T17:18:18.653Z{"msg":"[0]R flush_numbers[0..12]: [205, 206, 200, 202, 202, 172, 173, 200, 200, 198, 201, 192]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.255828859Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"":"downstairs","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122592024-04-17T17:18:18.653Z{"msg":"[0]R generation[0..12]: [112, 113, 110, 112, 112, 97, 98, 110, 110, 109, 111, 106]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.25585258Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"":"downstairs","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122602024-04-17T17:18:18.653Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.255867871Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"":"downstairs","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122612024-04-17T17:18:18.653Z{"msg":"[1]R flush_numbers[0..12]: [205, 206, 200, 202, 202, 172, 173, 200, 200, 198, 201, 192]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.255890153Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"":"downstairs","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122622024-04-17T17:18:18.653Z{"msg":"[1]R generation[0..12]: [112, 113, 110, 112, 112, 97, 98, 110, 110, 109, 111, 106]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.255904664Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"":"downstairs","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122632024-04-17T17:18:18.653Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.255920534Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"":"downstairs","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122642024-04-17T17:18:18.653ZWait for a query_work_queue command to finish before sending IO
122652024-04-17T17:18:18.653Z{"msg":"[2]R flush_numbers[0..12]: [205, 206, 200, 202, 202, 172, 173, 200, 200, 198, 201, 192]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.255948976Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"":"downstairs","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122662024-04-17T17:18:18.653Z{"msg":"[2]R generation[0..12]: [112, 113, 110, 112, 112, 97, 98, 110, 110, 109, 111, 106]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.255977028Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"":"downstairs","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122672024-04-17T17:18:18.653Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.25601193Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"":"downstairs","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122682024-04-17T17:18:18.653Z{"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.256036441Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"":"downstairs","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122692024-04-17T17:18:18.653Z{"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.256059483Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"":"downstairs","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122702024-04-17T17:18:18.653Z{"msg":"Next flush: 207","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.256090205Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"":"downstairs","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122712024-04-17T17:18:18.653Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.256113416Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"":"downstairs","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122722024-04-17T17:18:18.653Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.256140878Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122732024-04-17T17:18:18.653Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.256162129Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"client":"0","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122742024-04-17T17:18:18.653Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.256189851Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"client":"1","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122752024-04-17T17:18:18.653Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.256213172Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"client":"2","":"downstairs","session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122762024-04-17T17:18:18.653Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.256240244Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122772024-04-17T17:18:18.653Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
122782024-04-17T17:18:18.653Z2024-04-17T17:18:17.256262085Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122792024-04-17T17:18:18.654Z{"msg":"7c8f2d68-85fe-46dc-8cd2-26f69a7179f9 is now active with session: c4b6ad68-f565-4e91-a4d5-a641f48ad58b","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.256297467Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122802024-04-17T17:18:18.654Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.256319319Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627,"session_id":"c4b6ad68-f565-4e91-a4d5-a641f48ad58b"}
122812024-04-17T17:18:18.654Z{"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:17.25634051Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627}
122822024-04-17T17:18:18.658ZLoading write count information from file "/tmp/test_repair_verify.data"
122832024-04-17T17:18:18.658ZRun Repair workload
122842024-04-17T17:18:18.658Z01/30 Write block 199 len 40960 data: 8 10 12 14 15 16 15 14 16 8
122852024-04-17T17:18:18.658Z02/30 Write block 80 len 32768 data: 9 8 10 13 9 11 7 11
122862024-04-17T17:18:18.658Z03/30 Write block 303 len 32768 data: 7 8 9 8 7 8 7 8
122872024-04-17T17:18:18.658Z04/30 Write block 542 len 4096 data: 11
122882024-04-17T17:18:18.658Z05/30 Write block 442 len 8192 data: 9 9
122892024-04-17T17:18:18.658Z06/30 Read block 255 len 20480
122902024-04-17T17:18:18.658Z07/30 Write block 539 len 24576 data: 9 9 10 12 12 11
122912024-04-17T17:18:18.658Z08/30 Read block 591 len 28672
122922024-04-17T17:18:18.658Z09/30 Write block 583 len 8192 data: 11 12
122932024-04-17T17:18:18.658Z10/30 Read block 557 len 24576
122942024-04-17T17:18:18.661Z11/30 Read block 361 len 36864
122952024-04-17T17:18:18.661Z12/30 Read block 165 len 16384
122962024-04-17T17:18:18.661Z13/30 Write block 553 len 28672 data: 10 10 11 10 7 8 9
122972024-04-17T17:18:18.661Z14/30 Read block 46 len 36864
122982024-04-17T17:18:18.665Z15/30 Read block 97 len 28672
122992024-04-17T17:18:18.665Z16/30 Read block 147 len 36864
123002024-04-17T17:18:18.665Z17/30 Write block 512 len 8192 data: 8 4
123012024-04-17T17:18:18.665Z18/30 Write block 507 len 20480 data: 9 9 8 7 6
123022024-04-17T17:18:18.669Z19/30 Write block 31 len 28672 data: 11 16 17 20 17 13 11
123032024-04-17T17:18:18.669Z20/30 Write block 12 len 8192 data: 8 9
123042024-04-17T17:18:18.669Z21/30 Read block 0 len 32768
123052024-04-17T17:18:18.669Z22/30 Write block 92 len 20480 data: 14 13 11 9 8
123062024-04-17T17:18:18.675Z23/30 Write block 298 len 36864 data: 5 6 6 7 6 8 9 10 9
123072024-04-17T17:18:18.675Z24/30 Write block 58 len 4096 data: 13
123082024-04-17T17:18:18.675Z25/30 Read block 90 len 32768
123092024-04-17T17:18:18.675Z26/30 Read block 7 len 16384
123102024-04-17T17:18:18.675Z27/30 Write block 483 len 16384 data: 18 16 13 15
123112024-04-17T17:18:18.679Z28/30 Read block 582 len 16384
123122024-04-17T17:18:18.679Z29/30 Read block 257 len 24576
123132024-04-17T17:18:18.680Z30/30 Read block 323 len 36864
123142024-04-17T17:18:18.680Z----------------------------------------------------------------
123152024-04-17T17:18:18.680Z Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30
123162024-04-17T17:18:18.680ZStates: Active Active Active
123172024-04-17T17:18:18.680ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
123182024-04-17T17:18:18.680Z 1 Acked 1000 Write 10 Done Done Done false
123192024-04-17T17:18:18.680Z 2 Acked 1001 Write 8 Done Done Sent false
123202024-04-17T17:18:18.680Z 3 Acked 1002 Write 8 Done Done Sent false
123212024-04-17T17:18:18.680Z 4 Acked 1003 Write 1 Done Done Sent false
123222024-04-17T17:18:18.680Z 5 Acked 1004 Write 2 Done Done Sent false
123232024-04-17T17:18:18.680Z 6 Acked 1005 Read 5 Done Done Sent false
123242024-04-17T17:18:18.680Z 7 Acked 1006 Write 6 Done Done Sent false
123252024-04-17T17:18:18.680Z 8 Acked 1007 Read 7 Done Done Sent false
123262024-04-17T17:18:18.680Z 9 Acked 1008 Write 2 Done Done Sent false
123272024-04-17T17:18:18.680Z 10 Acked 1009 Read 6 Done Done Sent false
123282024-04-17T17:18:18.680Z 11 Acked 1010 Read 9 Done Done Sent false
123292024-04-17T17:18:18.680Z 12 Acked 1011 Read 4 Done Done Sent false
123302024-04-17T17:18:18.680Z 13 Acked 1012 Write 7 Done Done Sent false
123312024-04-17T17:18:18.680Z 14 Acked 1013 Read 9 Done Done Sent false
123322024-04-17T17:18:18.680Z 15 Acked 1014 Read 7 Done Done Sent false
123332024-04-17T17:18:18.680Z 16 Acked 1015 Read 9 Done Done Sent false
123342024-04-17T17:18:18.680Z 17 Acked 1016 Write 2 Done Done Sent false
123352024-04-17T17:18:18.680Z 18 Acked 1017 Write 5 Done Done Sent false
123362024-04-17T17:18:18.681Z 19 Acked 1018 Write 7 Done Done Sent false
123372024-04-17T17:18:18.681Z 20 Acked 1019 Write 2 Done Done Sent false
123382024-04-17T17:18:18.681Z 21 Acked 1020 Read 8 Done Done Sent false
123392024-04-17T17:18:18.681Z 22 Acked 1021 Write 5 Done Done Sent false
123402024-04-17T17:18:18.681Z 23 Acked 1022 Write 9 Done Done Sent false
123412024-04-17T17:18:18.681Z 24 Acked 1023 Write 1 Done Done Sent false
123422024-04-17T17:18:18.681Z 25 Acked 1024 Read 8 Done Done Sent false
123432024-04-17T17:18:18.681Z 26 Acked 1025 Read 4 Done Done Sent false
123442024-04-17T17:18:18.681Z 27 Acked 1026 Write 4 Done Done Sent false
123452024-04-17T17:18:18.681Z 28 Acked 1027 Read 4 Done Done Sent false
123462024-04-17T17:18:18.681Z 29 Acked 1028 Read 6 Done Done Sent false
123472024-04-17T17:18:18.681Z 30 Acked 1029 Read 9 Done Done Sent false
123482024-04-17T17:18:18.681Z STATES DS:0 DS:1 DS:2 TOTAL
123492024-04-17T17:18:18.681Z New 0 0 0 0
123502024-04-17T17:18:18.681Z Sent 0 0 29 29
123512024-04-17T17:18:18.681Z Done 30 30 1 61
123522024-04-17T17:18:18.681Z Skipped 0 0 0 0
123532024-04-17T17:18:18.681Z Error 0 0 0 0
123542024-04-17T17:18:18.681ZLast Flush: 0 0 0
123552024-04-17T17:18:18.681ZDownstairs last five completed:
123562024-04-17T17:18:18.681ZUpstairs last five completed: 30 29 28 27 26
123572024-04-17T17:18:18.681Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:18:17.284527058Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1627}
123582024-04-17T17:18:18.681ZWrote out file "/tmp/test_repair_verify.data"
123592024-04-17T17:18:18.685Z
123602024-04-17T17:18:18.685ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
123612024-04-17T17:18:18.705ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
123622024-04-17T17:18:18.705Z 0 000-019 112 112 112 205 205 205 T T F <---
123632024-04-17T17:18:18.705Z 1 020-039 113 113 113 206 206 206 T T F <---
123642024-04-17T17:18:18.705Z 2 040-059 110 110 110 200 200 200 T T F <---
123652024-04-17T17:18:18.705Z 3 060-079 112 112 112 202 202 202 F F F
123662024-04-17T17:18:18.706Z 4 080-099 112 112 112 202 202 202 T T F <---
123672024-04-17T17:18:18.706Z 5 100-119 97 97 97 172 172 172 F F F
123682024-04-17T17:18:18.709Z 6 120-139 98 98 98 173 173 173 F F F
123692024-04-17T17:18:18.709Z 7 140-159 110 110 110 200 200 200 F F F
123702024-04-17T17:18:18.709Z 8 160-179 110 110 110 200 200 200 F F F
123712024-04-17T17:18:18.709Z 9 180-199 109 109 109 198 198 198 T T T
123722024-04-17T17:18:18.709Z 10 200-219 111 111 111 201 201 201 T T T
123732024-04-17T17:18:18.709Z 11 220-239 106 106 106 192 192 192 F F F
123742024-04-17T17:18:18.709Z 12 240-259 108 108 108 195 195 195 F F F
123752024-04-17T17:18:18.709Z 13 260-279 112 112 112 203 203 203 F F F
123762024-04-17T17:18:18.709Z 14 280-299 112 112 112 203 203 203 T T F <---
123772024-04-17T17:18:18.710Z 15 300-319 113 113 113 206 206 206 T T F <---
123782024-04-17T17:18:18.710Z 16 320-339 112 112 112 203 203 203 F F F
123792024-04-17T17:18:18.710Z 17 340-359 110 110 110 199 199 199 F F F
123802024-04-17T17:18:18.710Z 18 360-379 110 110 110 200 200 200 F F F
123812024-04-17T17:18:18.710Z 19 380-399 108 108 108 197 197 197 F F F
123822024-04-17T17:18:18.710Z 20 400-419 109 109 109 198 198 198 F F F
123832024-04-17T17:18:18.710Z 21 420-439 111 111 111 201 201 201 F F F
123842024-04-17T17:18:18.710Z 22 440-459 112 112 112 202 202 202 T T F <---
123852024-04-17T17:18:18.710Z 23 460-479 112 112 112 205 205 205 F F F
123862024-04-17T17:18:18.710Z 24 480-499 112 112 112 204 204 204 T T F <---
123872024-04-17T17:18:18.710Z 25 500-519 106 106 106 192 192 192 T T F <---
123882024-04-17T17:18:18.710Z 26 520-539 110 110 110 199 199 199 T T F <---
123892024-04-17T17:18:18.710Z 27 540-559 113 113 113 206 206 206 T T F <---
123902024-04-17T17:18:18.710Z 28 560-579 113 113 113 206 206 206 F F F
123912024-04-17T17:18:18.710Z 29 580-599 113 113 113 206 206 206 T T F <---
123922024-04-17T17:18:18.710ZMax gen: 113, Max flush: 206
123932024-04-17T17:18:18.710ZError: Difference in extent metadata found!
123942024-04-17T17:18:18.710ZA Difference in extent metadata is expected here
123952024-04-17T17:18:18.710ZOn loop 57
123962024-04-17T17:18:18.710Z
123972024-04-17T17:18:18.714ZVerifying data now
123982024-04-17T17:18:23.378ZNo change after verify
123992024-04-17T17:18:23.378ZLoop: 57 Downstairs dump after verify (and repair):
124002024-04-17T17:18:23.392ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
124012024-04-17T17:18:23.392Z 0 000-019 114 114 114 207 207 207 F F F
124022024-04-17T17:18:23.392Z 1 020-039 114 114 114 207 207 207 F F F
124032024-04-17T17:18:23.392Z 2 040-059 114 114 114 207 207 207 F F F
124042024-04-17T17:18:23.392Z 3 060-079 112 112 112 202 202 202 F F F
124052024-04-17T17:18:23.392Z 4 080-099 114 114 114 207 207 207 F F F
124062024-04-17T17:18:23.392Z 5 100-119 97 97 97 172 172 172 F F F
124072024-04-17T17:18:23.396Z 6 120-139 98 98 98 173 173 173 F F F
124082024-04-17T17:18:23.396Z 7 140-159 110 110 110 200 200 200 F F F
124092024-04-17T17:18:23.396Z 8 160-179 110 110 110 200 200 200 F F F
124102024-04-17T17:18:23.396Z 9 180-199 114 114 114 207 207 207 F F F
124112024-04-17T17:18:23.396Z 10 200-219 114 114 114 207 207 207 F F F
124122024-04-17T17:18:23.396Z 11 220-239 106 106 106 192 192 192 F F F
124132024-04-17T17:18:23.396Z 12 240-259 108 108 108 195 195 195 F F F
124142024-04-17T17:18:23.396Z 13 260-279 112 112 112 203 203 203 F F F
124152024-04-17T17:18:23.397Z 14 280-299 114 114 114 207 207 207 F F F
124162024-04-17T17:18:23.397Z 15 300-319 114 114 114 207 207 207 F F F
124172024-04-17T17:18:23.397Z 16 320-339 112 112 112 203 203 203 F F F
124182024-04-17T17:18:23.397Z 17 340-359 110 110 110 199 199 199 F F F
124192024-04-17T17:18:23.397Z 18 360-379 110 110 110 200 200 200 F F F
124202024-04-17T17:18:23.397Z 19 380-399 108 108 108 197 197 197 F F F
124212024-04-17T17:18:23.397Z 20 400-419 109 109 109 198 198 198 F F F
124222024-04-17T17:18:23.397Z 21 420-439 111 111 111 201 201 201 F F F
124232024-04-17T17:18:23.397Z 22 440-459 114 114 114 207 207 207 F F F
124242024-04-17T17:18:23.397Z 23 460-479 112 112 112 205 205 205 F F F
124252024-04-17T17:18:23.397Z 24 480-499 114 114 114 207 207 207 F F F
124262024-04-17T17:18:23.397Z 25 500-519 114 114 114 207 207 207 F F F
124272024-04-17T17:18:23.397Z 26 520-539 114 114 114 207 207 207 F F F
124282024-04-17T17:18:23.397Z 27 540-559 114 114 114 207 207 207 F F F
124292024-04-17T17:18:23.397Z 28 560-579 113 113 113 206 206 206 F F F
124302024-04-17T17:18:23.397Z 29 580-599 114 114 114 207 207 207 F F F
124312024-04-17T17:18:23.397ZMax gen: 114, Max flush: 207
124322024-04-17T17:18:23.397Z
124332024-04-17T17:18:23.397ZBegin loop 58
124342024-04-17T17:18:23.397ZDownstairs to restart: 1
124352024-04-17T17:18:23.415Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.023767561Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635}
124362024-04-17T17:18:23.423Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.024032666Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635}
124372024-04-17T17:18:23.423Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.024073319Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635}
124382024-04-17T17:18:23.423ZCrucible runtime is spawned
124392024-04-17T17:18:23.423Z{"msg":"Crucible e9db9c84-fdb2-4bba-ba13-42955dfd4c62 has session id: aaa9514c-a9b1-4aa0-af3b-f7ba7862c063","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.024169805Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124402024-04-17T17:18:23.423Z{"msg":"Upstairs opts: Upstairs UUID: e9db9c84-fdb2-4bba-ba13-42955dfd4c62, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.024234309Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124412024-04-17T17:18:23.423Z{"msg":"Crucible stats registered with UUID: e9db9c84-fdb2-4bba-ba13-42955dfd4c62","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.024275291Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124422024-04-17T17:18:23.423Z{"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.024304203Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635}
124432024-04-17T17:18:23.423Z{"msg":"e9db9c84-fdb2-4bba-ba13-42955dfd4c62 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.024349646Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124442024-04-17T17:18:23.423Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.024377557Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"client":"0","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124452024-04-17T17:18:23.423Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.02441632Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"client":"0","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124462024-04-17T17:18:23.423Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.024446722Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"client":"1","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124472024-04-17T17:18:23.423Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.024517896Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"client":"1","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124482024-04-17T17:18:23.423Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.024680116Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"client":"2","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124492024-04-17T17:18:23.423Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.024740099Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"client":"2","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124502024-04-17T17:18:23.423Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.024785472Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"":"io task","client":"2","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124512024-04-17T17:18:23.423Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.024851466Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"":"io task","client":"0","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124522024-04-17T17:18:23.423Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.024872877Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"":"io task","client":"1","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124532024-04-17T17:18:23.424Z{"msg":"ds_connection connected from Ok(127.0.0.1:39944)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.024900219Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"":"io task","client":"0","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124542024-04-17T17:18:23.424Z{"msg":"ds_connection connected from Ok(127.0.0.1:52943)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.024936631Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"":"io task","client":"2","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124552024-04-17T17:18:23.424Z{"msg":"ds_connection connected from Ok(127.0.0.1:65121)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.024956333Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"":"io task","client":"1","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124562024-04-17T17:18:23.424Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.025333275Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"client":"2","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124572024-04-17T17:18:23.424Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.025370658Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"client":"2","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124582024-04-17T17:18:23.424Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.025386268Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"client":"0","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124592024-04-17T17:18:23.424Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.025400689Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"client":"0","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124602024-04-17T17:18:23.424Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.02541468Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"client":"1","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124612024-04-17T17:18:23.424Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.025436542Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"client":"1","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124622024-04-17T17:18:23.424Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.025852537Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"client":"2","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124632024-04-17T17:18:23.424Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.025921571Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"client":"0","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124642024-04-17T17:18:23.424Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.026032348Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"client":"1","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124652024-04-17T17:18:23.424Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.026480515Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"client":"2","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124662024-04-17T17:18:23.424Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.026558159Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"client":"2","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124672024-04-17T17:18:23.424Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.026606113Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124682024-04-17T17:18:23.424Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.026631214Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"client":"1","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124692024-04-17T17:18:23.424Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.026660496Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"client":"1","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124702024-04-17T17:18:23.424Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.026679717Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124712024-04-17T17:18:23.424Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.026701598Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"client":"0","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124722024-04-17T17:18:23.424Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.026718319Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"client":"0","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124732024-04-17T17:18:23.424Z{"msg":"[0]R flush_numbers[0..12]: [207, 207, 207, 202, 207, 172, 173, 200, 200, 207, 207, 192]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.026742041Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"":"downstairs","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124742024-04-17T17:18:23.424Z{"msg":"[0]R generation[0..12]: [114, 114, 114, 112, 114, 97, 98, 110, 110, 114, 114, 106]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.026760872Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"":"downstairs","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124752024-04-17T17:18:23.424Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.026785043Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"":"downstairs","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124762024-04-17T17:18:23.424Z{"msg":"[1]R flush_numbers[0..12]: [207, 207, 207, 202, 207, 172, 173, 200, 200, 207, 207, 192]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.026801274Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"":"downstairs","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124772024-04-17T17:18:23.425ZWait for a query_work_queue command to finish before sending IO
124782024-04-17T17:18:23.425Z{"msg":"[1]R generation[0..12]: [114, 114, 114, 112, 114, 97, 98, 110, 110, 114, 114, 106]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.026828186Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"":"downstairs","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124792024-04-17T17:18:23.425Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.026844577Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"":"downstairs","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124802024-04-17T17:18:23.425Z{"msg":"[2]R flush_numbers[0..12]: [207, 207, 207, 202, 207, 172, 173, 200, 200, 207, 207, 192]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.026867098Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"":"downstairs","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124812024-04-17T17:18:23.425Z{"msg":"[2]R generation[0..12]: [114, 114, 114, 112, 114, 97, 98, 110, 110, 114, 114, 106]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.026883639Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"":"downstairs","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124822024-04-17T17:18:23.425Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.026902611Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"":"downstairs","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124832024-04-17T17:18:23.425Z{"msg":"Max found gen is 115","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.026923962Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"":"downstairs","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124842024-04-17T17:18:23.425Z{"msg":"Generation requested: 116 >= found:115","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.026938753Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"":"downstairs","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124852024-04-17T17:18:23.425Z{"msg":"Next flush: 208","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.026957534Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"":"downstairs","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124862024-04-17T17:18:23.425Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.026973345Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"":"downstairs","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124872024-04-17T17:18:23.425Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.026999366Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124882024-04-17T17:18:23.425Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.027018787Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"client":"0","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124892024-04-17T17:18:23.425Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.027040459Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"client":"1","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124902024-04-17T17:18:23.425Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.02705451Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"client":"2","":"downstairs","session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124912024-04-17T17:18:23.425Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.027083401Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124922024-04-17T17:18:23.425Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.027113323Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124932024-04-17T17:18:23.425Z{"msg":"e9db9c84-fdb2-4bba-ba13-42955dfd4c62 is now active with session: aaa9514c-a9b1-4aa0-af3b-f7ba7862c063","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.027127804Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124942024-04-17T17:18:23.425Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.027141885Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635,"session_id":"aaa9514c-a9b1-4aa0-af3b-f7ba7862c063"}
124952024-04-17T17:18:23.425Z{"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:22.027164646Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635}
124962024-04-17T17:18:23.425ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
124972024-04-17T17:18:23.425ZLoading write count information from file "/tmp/test_repair_verify.data"
124982024-04-17T17:18:23.425ZRun Repair workload
124992024-04-17T17:18:23.430Z01/30 Write block 164 len 32768 data: 9 7 9 10 9 8 7 8
125002024-04-17T17:18:23.430Z02/30 Read block 29 len 36864
125012024-04-17T17:18:23.430Z03/30 Write block 442 len 16384 data: 10 10 10 7
125022024-04-17T17:18:23.430Z04/30 Write block 432 len 12288 data: 7 10 9
125032024-04-17T17:18:23.430Z05/30 Write block 524 len 12288 data: 6 7 9
125042024-04-17T17:18:23.430Z06/30 Read block 49 len 8192
125052024-04-17T17:18:23.430Z07/30 Write block 184 len 28672 data: 4 3 5 9 11 10 8
125062024-04-17T17:18:23.430Z08/30 Read block 225 len 12288
125072024-04-17T17:18:23.430Z09/30 Write block 282 len 16384 data: 6 6 7 6
125082024-04-17T17:18:23.430Z10/30 Read block 428 len 40960
125092024-04-17T17:18:23.430Z11/30 Write block 311 len 12288 data: 7 11 12
125102024-04-17T17:18:23.430Z12/30 Write block 318 len 16384 data: 12 12 9 8
125112024-04-17T17:18:23.433Z13/30 Write block 368 len 24576 data: 10 10 9 11 8 9
125122024-04-17T17:18:23.433Z14/30 Read block 88 len 8192
125132024-04-17T17:18:23.434Z15/30 Flush
125142024-04-17T17:18:23.437Z16/30 Flush
125152024-04-17T17:18:23.437Z17/30 Write block 521 len 4096 data: 7
125162024-04-17T17:18:23.437Z18/30 Read block 104 len 20480
125172024-04-17T17:18:23.437Z19/30 Read block 183 len 32768
125182024-04-17T17:18:23.440Z20/30 Write block 187 len 28672 data: 10 12 11 9 7 8 8
125192024-04-17T17:18:23.441Z21/30 Read block 97 len 28672
125202024-04-17T17:18:23.441Z22/30 Write block 85 len 20480 data: 12 8 12 10 12
125212024-04-17T17:18:23.441Z23/30 Read block 595 len 8192
125222024-04-17T17:18:23.444Z24/30 Write block 340 len 16384 data: 10 10 12 12
125232024-04-17T17:18:23.444Z25/30 Write block 71 len 40960 data: 7 7 5 7 8 9 9 8 7 10
125242024-04-17T17:18:23.444Z26/30 Read block 553 len 24576
125252024-04-17T17:18:23.448Z27/30 Write block 527 len 12288 data: 9 10 11
125262024-04-17T17:18:23.448Z28/30 Read block 152 len 28672
125272024-04-17T17:18:23.448Z29/30 Read block 462 len 12288
125282024-04-17T17:18:23.452Z30/30 Write block 390 len 40960 data: 10 11 12 10 10 6 6 7 8 8
125292024-04-17T17:18:23.452Z----------------------------------------------------------------
125302024-04-17T17:18:23.452Z Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:30
125312024-04-17T17:18:23.452ZStates: Active Active Active
125322024-04-17T17:18:23.452ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
125332024-04-17T17:18:23.452Z 1 Acked 1000 Write 8 Done Sent Done false
125342024-04-17T17:18:23.452Z 2 Acked 1001 Read 9 Done Sent Done false
125352024-04-17T17:18:23.452Z 3 Acked 1002 Write 4 Done Sent Done false
125362024-04-17T17:18:23.452Z 4 Acked 1003 Write 3 Done Sent Done false
125372024-04-17T17:18:23.452Z 5 Acked 1004 Write 3 Done Sent Done false
125382024-04-17T17:18:23.452Z 6 Acked 1005 Read 2 Done Sent Done false
125392024-04-17T17:18:23.452Z 7 Acked 1006 Write 7 Done Sent Done false
125402024-04-17T17:18:23.452Z 8 Acked 1007 Read 3 Done Sent Done false
125412024-04-17T17:18:23.452Z 9 Acked 1008 Write 4 Done Sent Done false
125422024-04-17T17:18:23.452Z 10 Acked 1009 Read 10 Done Sent Done false
125432024-04-17T17:18:23.453Z 11 Acked 1010 Write 3 Done Sent Done false
125442024-04-17T17:18:23.453Z 12 Acked 1011 Write 4 Done Sent Done false
125452024-04-17T17:18:23.453Z 13 Acked 1012 Write 6 Done Sent Done false
125462024-04-17T17:18:23.453Z 14 Acked 1013 Read 2 Done Sent Done false
125472024-04-17T17:18:23.453Z 15 Acked 1014 Flush 0 Done Sent Done false
125482024-04-17T17:18:23.453Z 16 Acked 1015 Flush 0 Done Sent Done false
125492024-04-17T17:18:23.453Z 17 Acked 1016 Write 1 Done Sent Done false
125502024-04-17T17:18:23.453Z 18 Acked 1017 Read 5 Done Sent Done false
125512024-04-17T17:18:23.453Z 19 Acked 1018 Read 8 Done Sent Done false
125522024-04-17T17:18:23.453Z 20 Acked 1019 Write 7 Done Sent Done false
125532024-04-17T17:18:23.453Z 21 Acked 1020 Read 7 Done Sent Done false
125542024-04-17T17:18:23.453Z 22 Acked 1021 Write 5 Done Sent Done false
125552024-04-17T17:18:23.453Z 23 Acked 1022 Read 2 Done Sent Done false
125562024-04-17T17:18:23.453Z 24 Acked 1023 Write 4 Done Sent Done false
125572024-04-17T17:18:23.453Z 25 Acked 1024 Write 10 Done Sent Done false
125582024-04-17T17:18:23.453Z 26 Acked 1025 Read 6 Done Sent Done false
125592024-04-17T17:18:23.453Z 27 Acked 1026 Write 3 Done Sent Done false
125602024-04-17T17:18:23.453Z 28 Acked 1027 Read 7 Done Sent Done false
125612024-04-17T17:18:23.453Z 29 Acked 1028 Read 3 Done Sent Done false
125622024-04-17T17:18:23.453Z 30 Acked 1029 Write 10 Sent Sent Sent false
125632024-04-17T17:18:23.453Z STATES DS:0 DS:1 DS:2 TOTAL
125642024-04-17T17:18:23.453Z New 0 0 0 0
125652024-04-17T17:18:23.453Z Sent 1 30 1 32
125662024-04-17T17:18:23.453Z Done 29 0 29 58
125672024-04-17T17:18:23.454Z Skipped 0 0 0 0
125682024-04-17T17:18:23.454Z Error 0 0 0 0
125692024-04-17T17:18:23.454ZLast Flush: 1015 0 1015
125702024-04-17T17:18:23.454ZDownstairs last five completed:
125712024-04-17T17:18:23.454ZUpstairs last five completed: 30 29 28 27 26
125722024-04-17T17:18:23.454Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:18:22.057712778Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1635}
125732024-04-17T17:18:23.454ZWrote out file "/tmp/test_repair_verify.data"
125742024-04-17T17:18:23.458Z
125752024-04-17T17:18:23.458ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
125762024-04-17T17:18:23.472ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
125772024-04-17T17:18:23.472Z 0 000-019 114 114 114 207 207 207 F F F
125782024-04-17T17:18:23.472Z 1 020-039 114 114 114 207 207 207 F F F
125792024-04-17T17:18:23.472Z 2 040-059 114 114 114 207 207 207 F F F
125802024-04-17T17:18:23.472Z 3 060-079 112 112 112 202 202 202 T F T <---
125812024-04-17T17:18:23.472Z 4 080-099 114 114 114 207 207 207 T F T <---
125822024-04-17T17:18:23.472Z 5 100-119 97 97 97 172 172 172 F F F
125832024-04-17T17:18:23.472Z 6 120-139 98 98 98 173 173 173 F F F
125842024-04-17T17:18:23.472Z 7 140-159 110 110 110 200 200 200 F F F
125852024-04-17T17:18:23.476Z 8 160-179 116 110 116 208 200 208 F F F <---
125862024-04-17T17:18:23.476Z 9 180-199 116 114 116 208 207 208 T F T <---
125872024-04-17T17:18:23.476Z 10 200-219 114 114 114 207 207 207 F F F
125882024-04-17T17:18:23.476Z 11 220-239 106 106 106 192 192 192 F F F
125892024-04-17T17:18:23.476Z 12 240-259 108 108 108 195 195 195 F F F
125902024-04-17T17:18:23.476Z 13 260-279 112 112 112 203 203 203 F F F
125912024-04-17T17:18:23.476Z 14 280-299 116 114 116 208 207 208 F F F <---
125922024-04-17T17:18:23.476Z 15 300-319 116 114 116 208 207 208 F F F <---
125932024-04-17T17:18:23.476Z 16 320-339 116 112 116 208 203 208 F F F <---
125942024-04-17T17:18:23.476Z 17 340-359 110 110 110 199 199 199 T F T <---
125952024-04-17T17:18:23.476Z 18 360-379 116 110 116 208 200 208 F F F <---
125962024-04-17T17:18:23.476Z 19 380-399 108 108 108 197 197 197 T F T <---
125972024-04-17T17:18:23.476Z 20 400-419 109 109 109 198 198 198 F F F
125982024-04-17T17:18:23.476Z 21 420-439 116 111 116 208 201 208 F F F <---
125992024-04-17T17:18:23.476Z 22 440-459 116 114 116 208 207 208 F F F <---
126002024-04-17T17:18:23.477Z 23 460-479 112 112 112 205 205 205 F F F
126012024-04-17T17:18:23.477Z 24 480-499 114 114 114 207 207 207 F F F
126022024-04-17T17:18:23.477Z 25 500-519 114 114 114 207 207 207 F F F
126032024-04-17T17:18:23.477Z 26 520-539 116 114 116 208 207 208 T F T <---
126042024-04-17T17:18:23.477Z 27 540-559 114 114 114 207 207 207 F F F
126052024-04-17T17:18:23.477Z 28 560-579 113 113 113 206 206 206 F F F
126062024-04-17T17:18:23.477Z 29 580-599 114 114 114 207 207 207 F F F
126072024-04-17T17:18:23.477ZMax gen: 116, Max flush: 208
126082024-04-17T17:18:23.477ZError: Difference in extent metadata found!
126092024-04-17T17:18:23.477ZA Difference in extent metadata is expected here
126102024-04-17T17:18:23.477ZOn loop 58
126112024-04-17T17:18:23.477Z
126122024-04-17T17:18:23.480ZVerifying data now
126132024-04-17T17:18:28.086ZNo change after verify
126142024-04-17T17:18:28.087ZLoop: 58 Downstairs dump after verify (and repair):
126152024-04-17T17:18:28.101ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
126162024-04-17T17:18:28.101Z 0 000-019 114 114 114 207 207 207 F F F
126172024-04-17T17:18:28.105Z 1 020-039 114 114 114 207 207 207 F F F
126182024-04-17T17:18:28.105Z 2 040-059 114 114 114 207 207 207 F F F
126192024-04-17T17:18:28.105Z 3 060-079 117 117 117 209 209 209 F F F
126202024-04-17T17:18:28.105Z 4 080-099 117 117 117 209 209 209 F F F
126212024-04-17T17:18:28.105Z 5 100-119 97 97 97 172 172 172 F F F
126222024-04-17T17:18:28.105Z 6 120-139 98 98 98 173 173 173 F F F
126232024-04-17T17:18:28.105Z 7 140-159 110 110 110 200 200 200 F F F
126242024-04-17T17:18:28.105Z 8 160-179 116 116 116 208 208 208 F F F
126252024-04-17T17:18:28.105Z 9 180-199 117 117 117 209 209 209 F F F
126262024-04-17T17:18:28.105Z 10 200-219 114 114 114 207 207 207 F F F
126272024-04-17T17:18:28.105Z 11 220-239 106 106 106 192 192 192 F F F
126282024-04-17T17:18:28.105Z 12 240-259 108 108 108 195 195 195 F F F
126292024-04-17T17:18:28.105Z 13 260-279 112 112 112 203 203 203 F F F
126302024-04-17T17:18:28.105Z 14 280-299 116 116 116 208 208 208 F F F
126312024-04-17T17:18:28.105Z 15 300-319 116 116 116 208 208 208 F F F
126322024-04-17T17:18:28.105Z 16 320-339 116 116 116 208 208 208 F F F
126332024-04-17T17:18:28.105Z 17 340-359 117 117 117 209 209 209 F F F
126342024-04-17T17:18:28.105Z 18 360-379 116 116 116 208 208 208 F F F
126352024-04-17T17:18:28.105Z 19 380-399 117 117 117 209 209 209 F F F
126362024-04-17T17:18:28.105Z 20 400-419 109 109 109 198 198 198 F F F
126372024-04-17T17:18:28.106Z 21 420-439 116 116 116 208 208 208 F F F
126382024-04-17T17:18:28.106Z 22 440-459 116 116 116 208 208 208 F F F
126392024-04-17T17:18:28.106Z 23 460-479 112 112 112 205 205 205 F F F
126402024-04-17T17:18:28.106Z 24 480-499 114 114 114 207 207 207 F F F
126412024-04-17T17:18:28.106Z 25 500-519 114 114 114 207 207 207 F F F
126422024-04-17T17:18:28.106Z 26 520-539 117 117 117 209 209 209 F F F
126432024-04-17T17:18:28.106Z 27 540-559 114 114 114 207 207 207 F F F
126442024-04-17T17:18:28.106Z 28 560-579 113 113 113 206 206 206 F F F
126452024-04-17T17:18:28.106Z 29 580-599 114 114 114 207 207 207 F F F
126462024-04-17T17:18:28.106ZMax gen: 117, Max flush: 209
126472024-04-17T17:18:28.109Z
126482024-04-17T17:18:28.109ZBegin loop 59
126492024-04-17T17:18:28.110ZDownstairs to restart: 1
126502024-04-17T17:18:28.124Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.732840245Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643}
126512024-04-17T17:18:28.124Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.73308573Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643}
126522024-04-17T17:18:28.124Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.733103271Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643}
126532024-04-17T17:18:28.138Z{"msg":"Crucible 102e5350-40f3-4287-b399-ed822dee7684 has session id: c3ba4036-8e32-4369-91c3-1c503c20a6f7","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.733133602Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126542024-04-17T17:18:28.138Z{Crucible runtime is spawned
126552024-04-17T17:18:28.138Z"msg":"Upstairs opts: Upstairs UUID: 102e5350-40f3-4287-b399-ed822dee7684, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.733209757Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126562024-04-17T17:18:28.138Z{"msg":"Crucible stats registered with UUID: 102e5350-40f3-4287-b399-ed822dee7684","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.73324902Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126572024-04-17T17:18:28.139Z{"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.733271571Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643}
126582024-04-17T17:18:28.139Z{"msg":"102e5350-40f3-4287-b399-ed822dee7684 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.733340855Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126592024-04-17T17:18:28.139Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.733376057Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"client":"0","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126602024-04-17T17:18:28.139Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.733392718Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"client":"0","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126612024-04-17T17:18:28.139Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.733413939Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"client":"1","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126622024-04-17T17:18:28.139Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.73342835Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"client":"1","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126632024-04-17T17:18:28.140Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.733459742Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"client":"2","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126642024-04-17T17:18:28.140Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.733497064Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"client":"2","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126652024-04-17T17:18:28.140Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.733520816Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"":"io task","client":"0","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126662024-04-17T17:18:28.140Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.733550858Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"":"io task","client":"1","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126672024-04-17T17:18:28.140Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.733575419Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"":"io task","client":"2","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126682024-04-17T17:18:28.141Z{"msg":"ds_connection connected from Ok(127.0.0.1:48879)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.733633883Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"":"io task","client":"1","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126692024-04-17T17:18:28.141Z{"msg":"ds_connection connected from Ok(127.0.0.1:37046)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.733733799Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"":"io task","client":"2","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126702024-04-17T17:18:28.141Z{"msg":"ds_connection connected from Ok(127.0.0.1:54210)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.733795302Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"":"io task","client":"0","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126712024-04-17T17:18:28.141Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.734362387Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"client":"0","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126722024-04-17T17:18:28.141Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.73441025Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"client":"0","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126732024-04-17T17:18:28.141Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.734439361Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"client":"2","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126742024-04-17T17:18:28.141Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.734475494Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"client":"2","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126752024-04-17T17:18:28.142Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.734989635Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"client":"2","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126762024-04-17T17:18:28.142Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.735037808Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"client":"0","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126772024-04-17T17:18:28.142Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.735202938Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"client":"1","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126782024-04-17T17:18:28.142Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.735258971Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"client":"1","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126792024-04-17T17:18:28.142Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.735523897Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"client":"2","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126802024-04-17T17:18:28.143Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.735556919Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"client":"2","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126812024-04-17T17:18:28.143Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.73557147Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126822024-04-17T17:18:28.143Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.735719829Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"client":"0","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126832024-04-17T17:18:28.143Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.735761452Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"client":"0","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126842024-04-17T17:18:28.143Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.735777683Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126852024-04-17T17:18:28.143Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.735806464Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"client":"1","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126862024-04-17T17:18:28.144Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.736269323Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"client":"1","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126872024-04-17T17:18:28.144Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.736305425Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"client":"1","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126882024-04-17T17:18:28.144Z{"msg":"[0]R flush_numbers[0..12]: [207, 207, 207, 209, 209, 172, 173, 200, 208, 209, 207, 192]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.736323046Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"":"downstairs","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126892024-04-17T17:18:28.144Z{"msg":"[0]R generation[0..12]: [114, 114, 114, 117, 117, 97, 98, 110, 116, 117, 114, 106]","v":0,"Wait for a query_work_queue command to finish before sending IO
126902024-04-17T17:18:28.144Zname":"crucible","level":30,"time":"2024-04-17T17:18:26.736467084Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"":"downstairs","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126912024-04-17T17:18:28.144Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.736511047Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"":"downstairs","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126922024-04-17T17:18:28.144Z{"msg":"[1]R flush_numbers[0..12]: [207, 207, 207, 209, 209, 172, 173, 200, 208, 209, 207, 192]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.736528348Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"":"downstairs","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126932024-04-17T17:18:28.144Z{"msg":"[1]R generation[0..12]: [114, 114, 114, 117, 117, 97, 98, 110, 116, 117, 114, 106]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.736543579Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"":"downstairs","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126942024-04-17T17:18:28.144Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.73655872Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"":"downstairs","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126952024-04-17T17:18:28.144Z{"msg":"[2]R flush_numbers[0..12]: [207, 207, 207, 209, 209, 172, 173, 200, 208, 209, 207, 192]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.736572821Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"":"downstairs","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126962024-04-17T17:18:28.144Z{"msg":"[2]R generation[0..12]: [114, 114, 114, 117, 117, 97, 98, 110, 116, 117, 114, 106]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.736586532Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"":"downstairs","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126972024-04-17T17:18:28.144Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.736618534Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"":"downstairs","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126982024-04-17T17:18:28.144Z{"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.736633054Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"":"downstairs","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
126992024-04-17T17:18:28.145Z{"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.736654146Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"":"downstairs","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
127002024-04-17T17:18:28.145Z{"msg":"Next flush: 210","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.736667457Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"":"downstairs","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
127012024-04-17T17:18:28.145Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.736680027Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"":"downstairs","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
127022024-04-17T17:18:28.145Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.736700039Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
127032024-04-17T17:18:28.145Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.736712059Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"client":"0","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
127042024-04-17T17:18:28.145Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.73672511Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"client":"1","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
127052024-04-17T17:18:28.145Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.736738621Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"client":"2","":"downstairs","session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
127062024-04-17T17:18:28.145Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.736751652Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
127072024-04-17T17:18:28.145Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.736763733Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
127082024-04-17T17:18:28.145Z{"msg":"102e5350-40f3-4287-b399-ed822dee7684 is now active with session: c3ba4036-8e32-4369-91c3-1c503c20a6f7","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.736788884Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
127092024-04-17T17:18:28.145Z,"pid":1643,"session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
127102024-04-17T17:18:28.145Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.736809515Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643,"session_id":"c3ba4036-8e32-4369-91c3-1c503c20a6f7"}
127112024-04-17T17:18:28.145Z{"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:26.736839727Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643}
127122024-04-17T17:18:28.159ZLoading write count information from file "/tmp/test_repair_verify.data"
127132024-04-17T17:18:28.159ZRun Repair workload
127142024-04-17T17:18:28.159Z01/30 Write block 262 len 16384 data: 10 10 9 7
127152024-04-17T17:18:28.159Z02/30 Write block 574 len 24576 data: 12 12 12 11 10 12
127162024-04-17T17:18:28.159Z03/30 Write block 501 len 16384 data: 10 10 10 11
127172024-04-17T17:18:28.159Z04/30 Read block 147 len 16384
127182024-04-17T17:18:28.160Z05/30 Write block 398 len 36864 data: 9 9 8 8 7 8 8 10 10
127192024-04-17T17:18:28.160Z06/30 Read block 446 len 40960
127202024-04-17T17:18:28.160Z07/30 Read block 556 len 16384
127212024-04-17T17:18:28.160Z08/30 Write block 394 len 8192 data: 11 7
127222024-04-17T17:18:28.160Z09/30 Flush
127232024-04-17T17:18:28.160Z10/30 Write block 160 len 36864 data: 10 11 10 12 10 8 10 11 10
127242024-04-17T17:18:28.160Z11/30 Write block 242 len 32768 data: 10 11 10 10 12 12 12 9
127252024-04-17T17:18:28.160Z12/30 Write block 510 len 20480 data: 8 7 9 5 9
127262024-04-17T17:18:28.160Z13/30 Write block 461 len 32768 data: 8 8 10 10 11 10 9 9
127272024-04-17T17:18:28.160Z14/30 Read block 2 len 40960
127282024-04-17T17:18:28.160Z15/30 Read block 524 len 36864
127292024-04-17T17:18:28.160Z16/30 Read block 189 len 4096
127302024-04-17T17:18:28.160Z17/30 Write block 115 len 28672 data: 5 5 6 4 8 9 9
127312024-04-17T17:18:28.161Z18/30 Write block 218 len 28672 data: 8 8 10 9 8 10 11
127322024-04-17T17:18:28.161Z19/30 Write block 63 len 32768 data: 12 13 11 10 10 9 7 7
127332024-04-17T17:18:28.161Z20/30 Read block 64 len 32768
127342024-04-17T17:18:28.161Z21/30 Read block 371 len 32768
127352024-04-17T17:18:28.161Z22/30 Write block 348 len 8192 data: 9 9
127362024-04-17T17:18:28.161Z23/30 Read block 70 len 4096
127372024-04-17T17:18:28.167Z24/30 Write block 229 len 20480 data: 14 13 10 10 10
127382024-04-17T17:18:28.167Z25/30 Read block 107 len 40960
127392024-04-17T17:18:28.167Z26/30 Flush
127402024-04-17T17:18:28.167Z27/30 Write block 485 len 32768 data: 14 16 14 14 12 12 8 9
127412024-04-17T17:18:28.167Z28/30 Read block 17 len 24576
127422024-04-17T17:18:28.167Z29/30 Read block 30 len 32768
127432024-04-17T17:18:28.167Z30/30 Read block 479 len 20480
127442024-04-17T17:18:28.167Z----------------------------------------------------------------
127452024-04-17T17:18:28.167Z Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:30
127462024-04-17T17:18:28.167ZStates: Active Active Active
127472024-04-17T17:18:28.167ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
127482024-04-17T17:18:28.167Z 1 Acked 1000 Write 4 Done Done Done false
127492024-04-17T17:18:28.167Z 2 Acked 1001 Write 6 Done Done Done false
127502024-04-17T17:18:28.167Z 3 Acked 1002 Write 4 Done Done Done false
127512024-04-17T17:18:28.167Z 4 Acked 1003 Read 4 Done Done Done false
127522024-04-17T17:18:28.167Z 5 Acked 1004 Write 9 Done Done Done false
127532024-04-17T17:18:28.167Z 6 Acked 1005 Read 10 Done Done Done false
127542024-04-17T17:18:28.167Z 7 Acked 1006 Read 4 Done Sent Done false
127552024-04-17T17:18:28.167Z 8 Acked 1007 Write 2 Done Sent Done false
127562024-04-17T17:18:28.167Z 9 Acked 1008 Flush 0 Done Sent Done false
127572024-04-17T17:18:28.167Z 10 Acked 1009 Write 9 Done Sent Done false
127582024-04-17T17:18:28.167Z 11 Acked 1010 Write 8 Done Sent Done false
127592024-04-17T17:18:28.168Z 12 Acked 1011 Write 5 Done Sent Done false
127602024-04-17T17:18:28.168Z 13 Acked 1012 Write 8 Done Sent Done false
127612024-04-17T17:18:28.168Z 14 Acked 1013 Read 10 Done Sent Done false
127622024-04-17T17:18:28.168Z 15 Acked 1014 Read 9 Done Sent Done false
127632024-04-17T17:18:28.168Z 16 Acked 1015 Read 1 Done Sent Done false
127642024-04-17T17:18:28.168Z 17 Acked 1016 Write 7 Done Sent Done false
127652024-04-17T17:18:28.168Z 18 Acked 1017 Write 7 Done Sent Done false
127662024-04-17T17:18:28.168Z 19 Acked 1018 Write 8 Done Sent Done false
127672024-04-17T17:18:28.168Z 20 Acked 1019 Read 8 Done Sent Done false
127682024-04-17T17:18:28.168Z 21 Acked 1020 Read 8 Done Sent Done false
127692024-04-17T17:18:28.168Z 22 Acked 1021 Write 2 Done Sent Done false
127702024-04-17T17:18:28.168Z 23 Acked 1022 Read 1 Done Sent Done false
127712024-04-17T17:18:28.168Z 24 Acked 1023 Write 5 Done Sent Done false
127722024-04-17T17:18:28.168Z 25 Acked 1024 Read 10 Done Sent Done false
127732024-04-17T17:18:28.168Z 26 Acked 1025 Flush 0 Done Sent Done false
127742024-04-17T17:18:28.168Z 27 Acked 1026 Write 8 Done Sent Done false
127752024-04-17T17:18:28.168Z 28 Acked 1027 Read 6 Done Sent Done false
127762024-04-17T17:18:28.168Z 29 Acked 1028 Read 8 Done Sent Done false
127772024-04-17T17:18:28.168Z 30 Acked 1029 Read 5 Done Sent Done false
127782024-04-17T17:18:28.168Z STATES DS:0 DS:1 DS:2 TOTAL
127792024-04-17T17:18:28.168Z New 0 0 0 0
127802024-04-17T17:18:28.168Z Sent 0 24 0 24
127812024-04-17T17:18:28.168Z Done 30 6 30 66
127822024-04-17T17:18:28.168Z Skipped 0 0 0 0
127832024-04-17T17:18:28.169Z Error 0 0 0 0
127842024-04-17T17:18:28.169ZLast Flush: 1025 0 1025
127852024-04-17T17:18:28.169ZDownstairs last five completed:
127862024-04-17T17:18:28.169ZUpstairs last five completed: 30 29 28 27 26
127872024-04-17T17:18:28.169Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:18:26.773653518Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1643}
127882024-04-17T17:18:28.169ZWrote out file "/tmp/test_repair_verify.data"
127892024-04-17T17:18:28.172Z
127902024-04-17T17:18:28.172ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
127912024-04-17T17:18:28.188ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
127922024-04-17T17:18:28.188Z 0 000-019 114 114 114 207 207 207 F F F
127932024-04-17T17:18:28.192Z 1 020-039 114 114 114 207 207 207 F F F
127942024-04-17T17:18:28.192Z 2 040-059 114 114 114 207 207 207 F F F
127952024-04-17T17:18:28.192Z 3 060-079 118 117 118 211 209 211 F F F <---
127962024-04-17T17:18:28.192Z 4 080-099 117 117 117 209 209 209 F F F
127972024-04-17T17:18:28.192Z 5 100-119 118 97 118 211 172 211 F F F <---
127982024-04-17T17:18:28.192Z 6 120-139 118 98 118 211 173 211 F F F <---
127992024-04-17T17:18:28.192Z 7 140-159 110 110 110 200 200 200 F F F
128002024-04-17T17:18:28.192Z 8 160-179 118 116 118 211 208 211 F F F <---
128012024-04-17T17:18:28.192Z 9 180-199 117 117 117 209 209 209 F F F
128022024-04-17T17:18:28.192Z 10 200-219 118 114 118 211 207 211 F F F <---
128032024-04-17T17:18:28.192Z 11 220-239 118 106 118 211 192 211 F F F <---
128042024-04-17T17:18:28.192Z 12 240-259 118 108 118 211 195 211 F F F <---
128052024-04-17T17:18:28.192Z 13 260-279 118 112 118 210 203 210 F T F <---
128062024-04-17T17:18:28.192Z 14 280-299 116 116 116 208 208 208 F F F
128072024-04-17T17:18:28.192Z 15 300-319 116 116 116 208 208 208 F F F
128082024-04-17T17:18:28.192Z 16 320-339 116 116 116 208 208 208 F F F
128092024-04-17T17:18:28.193Z 17 340-359 118 117 118 211 209 211 F F F <---
128102024-04-17T17:18:28.193Z 18 360-379 116 116 116 208 208 208 F F F
128112024-04-17T17:18:28.193Z 19 380-399 118 117 118 210 209 210 F T F <---
128122024-04-17T17:18:28.193Z 20 400-419 118 109 118 210 198 210 F T F <---
128132024-04-17T17:18:28.193Z 21 420-439 116 116 116 208 208 208 F F F
128142024-04-17T17:18:28.193Z 22 440-459 116 116 116 208 208 208 F F F
128152024-04-17T17:18:28.193Z 23 460-479 118 112 118 211 205 211 F F F <---
128162024-04-17T17:18:28.193Z 24 480-499 114 114 114 207 207 207 T F T <---
128172024-04-17T17:18:28.193Z 25 500-519 118 114 118 211 207 211 F T F <---
128182024-04-17T17:18:28.193Z 26 520-539 117 117 117 209 209 209 F F F
128192024-04-17T17:18:28.193Z 27 540-559 114 114 114 207 207 207 F F F
128202024-04-17T17:18:28.193Z 28 560-579 118 113 118 210 206 210 F T F <---
128212024-04-17T17:18:28.193Z 29 580-599 114 114 114 207 207 207 F F F
128222024-04-17T17:18:28.193ZMax gen: 118, Max flush: 211
128232024-04-17T17:18:28.193ZError: Difference in extent metadata found!
128242024-04-17T17:18:28.197ZA Difference in extent metadata is expected here
128252024-04-17T17:18:28.197ZOn loop 59
128262024-04-17T17:18:28.197Z
128272024-04-17T17:18:28.197ZVerifying data now
128282024-04-17T17:18:42.869ZNo change after verify
128292024-04-17T17:18:42.873ZLoop: 59 Downstairs dump after verify (and repair):
128302024-04-17T17:18:42.884ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
128312024-04-17T17:18:42.884Z 0 000-019 114 114 114 207 207 207 F F F
128322024-04-17T17:18:42.889Z 1 020-039 114 114 114 207 207 207 F F F
128332024-04-17T17:18:42.889Z 2 040-059 114 114 114 207 207 207 F F F
128342024-04-17T17:18:42.889Z 3 060-079 118 118 118 211 211 211 F F F
128352024-04-17T17:18:42.889Z 4 080-099 117 117 117 209 209 209 F F F
128362024-04-17T17:18:42.889Z 5 100-119 118 118 118 211 211 211 F F F
128372024-04-17T17:18:42.889Z 6 120-139 118 118 118 211 211 211 F F F
128382024-04-17T17:18:42.889Z 7 140-159 110 110 110 200 200 200 F F F
128392024-04-17T17:18:42.889Z 8 160-179 118 118 118 211 211 211 F F F
128402024-04-17T17:18:42.890Z 9 180-199 117 117 117 209 209 209 F F F
128412024-04-17T17:18:42.890Z 10 200-219 118 118 118 211 211 211 F F F
128422024-04-17T17:18:42.890Z 11 220-239 118 118 118 211 211 211 F F F
128432024-04-17T17:18:42.890Z 12 240-259 118 118 118 211 211 211 F F F
128442024-04-17T17:18:42.890Z 13 260-279 118 118 118 210 210 210 F F F
128452024-04-17T17:18:42.890Z 14 280-299 116 116 116 208 208 208 F F F
128462024-04-17T17:18:42.890Z 15 300-319 116 116 116 208 208 208 F F F
128472024-04-17T17:18:42.890Z 16 320-339 116 116 116 208 208 208 F F F
128482024-04-17T17:18:42.890Z 17 340-359 118 118 118 211 211 211 F F F
128492024-04-17T17:18:42.890Z 18 360-379 116 116 116 208 208 208 F F F
128502024-04-17T17:18:42.890Z 19 380-399 118 118 118 210 210 210 F F F
128512024-04-17T17:18:42.890Z 20 400-419 118 118 118 210 210 210 F F F
128522024-04-17T17:18:42.891Z 21 420-439 116 116 116 208 208 208 F F F
128532024-04-17T17:18:42.891Z 22 440-459 116 116 116 208 208 208 F F F
128542024-04-17T17:18:42.891Z 23 460-479 118 118 118 211 211 211 F F F
128552024-04-17T17:18:42.891Z 24 480-499 119 119 119 212 212 212 F F F
128562024-04-17T17:18:42.891Z 25 500-519 118 118 118 211 211 211 F F F
128572024-04-17T17:18:42.891Z 26 520-539 117 117 117 209 209 209 F F F
128582024-04-17T17:18:42.891Z 27 540-559 114 114 114 207 207 207 F F F
128592024-04-17T17:18:42.891Z 28 560-579 118 118 118 210 210 210 F F F
128602024-04-17T17:18:42.891Z 29 580-599 114 114 114 207 207 207 F F F
128612024-04-17T17:18:42.891ZMax gen: 119, Max flush: 212
128622024-04-17T17:18:42.891Z
128632024-04-17T17:18:42.891ZBegin loop 60
128642024-04-17T17:18:42.892ZDownstairs to restart: 0
128652024-04-17T17:18:42.908Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.516869371Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651}
128662024-04-17T17:18:42.909Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.517145758Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651}
128672024-04-17T17:18:42.909Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0Crucible runtime is spawned
128682024-04-17T17:18:42.916Z,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.517203102Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651}
128692024-04-17T17:18:42.916Z{"msg":"Crucible 60aa77d1-b482-424a-a019-2f3047f84adb has session id: 8ff0f102-c895-4f65-9700-070bb6430452","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.517256545Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
128702024-04-17T17:18:42.917Z{"msg":"Upstairs opts: Upstairs UUID: 60aa77d1-b482-424a-a019-2f3047f84adb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.517313718Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
128712024-04-17T17:18:42.917Z{"msg":"Crucible stats registered with UUID: 60aa77d1-b482-424a-a019-2f3047f84adb","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.517360041Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
128722024-04-17T17:18:42.917Z{"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.517401614Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651}
128732024-04-17T17:18:42.917Z{"msg":"60aa77d1-b482-424a-a019-2f3047f84adb active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.517429165Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
128742024-04-17T17:18:42.917Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.517456407Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"client":"0","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
128752024-04-17T17:18:42.917Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.517488229Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"client":"0","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
128762024-04-17T17:18:42.917Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.517551963Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"client":"1","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
128772024-04-17T17:18:42.917Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.517624647Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"client":"1","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
128782024-04-17T17:18:42.917Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.517651579Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"client":"2","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
128792024-04-17T17:18:42.917Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.517686711Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"client":"2","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
128802024-04-17T17:18:42.917Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.517711533Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"":"io task","client":"2","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
128812024-04-17T17:18:42.917Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.517747545Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"":"io task","client":"0","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
128822024-04-17T17:18:42.917Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.517774557Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"":"io task","client":"1","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
128832024-04-17T17:18:42.917Z{"msg":"ds_connection connected from Ok(127.0.0.1:37465)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.517811329Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"":"io task","client":"2","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
128842024-04-17T17:18:42.917Z{"msg":"ds_connection connected from Ok(127.0.0.1:49568)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.51783682Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"":"io task","client":"0","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
128852024-04-17T17:18:42.917Z{"msg":"ds_connection connected from Ok(127.0.0.1:52354)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.517859971Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"":"io task","client":"1","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
128862024-04-17T17:18:42.917Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.518366012Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"client":"2","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
128872024-04-17T17:18:42.917Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.518407325Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"client":"2","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
128882024-04-17T17:18:42.917Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.518424586Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"client":"1","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
128892024-04-17T17:18:42.917Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.518443777Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"client":"1","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
128902024-04-17T17:18:42.917Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.518849031Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"client":"0","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
128912024-04-17T17:18:42.917Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.518907755Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"client":"0","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
128922024-04-17T17:18:42.918Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.518958628Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"client":"2","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
128932024-04-17T17:18:42.918Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.51898484Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"client":"1","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
128942024-04-17T17:18:42.918Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.519445608Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"client":"2","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
128952024-04-17T17:18:42.918Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.519493661Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"client":"2","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
128962024-04-17T17:18:42.918Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.519520102Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
128972024-04-17T17:18:42.918Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.519553554Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"client":"0","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
128982024-04-17T17:18:42.918Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.519592737Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"client":"1","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
128992024-04-17T17:18:42.918Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.519638279Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"client":"1","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
129002024-04-17T17:18:42.918Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.519662381Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
129012024-04-17T17:18:42.918Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.520023803Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"client":"0","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
129022024-04-17T17:18:42.918Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.520056055Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"client":"0","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
129032024-04-17T17:18:42.918Z{"msg":"[0]R flush_numbers[0..12]: [207, 207, 207, 211, 209, 211, 211, 200, 211, 209, 211, 211]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.520071486Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"":"downstairs","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
129042024-04-17T17:18:42.918Z{"msg":"[0]R generation[0..12]: [114, 114, 114, 118, 117, 118, 118, 110, 118, 117, 118, 118]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.520086866Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"":"downstairs","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
129052024-04-17T17:18:42.918Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.520109048Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"":"downstairs","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
129062024-04-17T17:18:42.918Z{"msg":"[1]R flush_numbers[0..12]: [207, 207, 207, 211, 209, 211, 211, 200, 211, 209, 211, 211]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.520123039Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"":"downstairs","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
129072024-04-17T17:18:42.918Z{"msg":"[1]R generation[0..12]: [114, 114, 114, 118, 117, 118, 118, 110, 118, 117, 118, 118]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.52013668Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"":"downstairs","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
129082024-04-17T17:18:42.918Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.520157671Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"":"downstairs","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
129092024-04-17T17:18:42.918Z{"msg":"[2]R flush_numbers[0..12]: [207, 207, 207, 211, 209, 211, 211, 200, 211, 209, 211, 211]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.520171872Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"":"downstairs","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
129102024-04-17T17:18:42.918Z{"msg":"[2]R generation[0..12]: [114, 114, 114, 118, 117, 118, 118, 110, 118, 117, 118, 118]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.520186713Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"":"downstairs","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
129112024-04-17T17:18:42.918Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.520200533Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"":"downstairs","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
129122024-04-17T17:18:42.918Z{"msg":"Max found gen is 120","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO
129132024-04-17T17:18:42.918Zlevel":30,"time":"2024-04-17T17:18:41.520222875Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"":"downstairs","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
129142024-04-17T17:18:42.918Z{"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.520236756Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"":"downstairs","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
129152024-04-17T17:18:42.919Z{"msg":"Next flush: 213","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.520249647Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"":"downstairs","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
129162024-04-17T17:18:42.919Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.520263517Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"":"downstairs","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
129172024-04-17T17:18:42.919Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.520284249Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
129182024-04-17T17:18:42.919Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.520296829Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"client":"0","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
129192024-04-17T17:18:42.919Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.52030984Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"client":"1","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
129202024-04-17T17:18:42.919Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.520329721Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"client":"2","":"downstairs","session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
129212024-04-17T17:18:42.919Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.520342962Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
129222024-04-17T17:18:42.919Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.520354943Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
129232024-04-17T17:18:42.919Z{"msg":"60aa77d1-b482-424a-a019-2f3047f84adb is now active with session: 8ff0f102-c895-4f65-9700-070bb6430452","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.520374474Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
129242024-04-17T17:18:42.919Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.520386415Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651,"session_id":"8ff0f102-c895-4f65-9700-070bb6430452"}
129252024-04-17T17:18:42.919Z{"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:41.520398245Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651}
129262024-04-17T17:18:42.919ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
129272024-04-17T17:18:42.919ZLoading write count information from file "/tmp/test_repair_verify.data"
129282024-04-17T17:18:42.919ZRun Repair workload
129292024-04-17T17:18:42.923Z01/30 Write block 326 len 28672 data: 8 7 7 7 7 8 7
129302024-04-17T17:18:42.923Z02/30 Write block 145 len 28672 data: 9 8 6 6 8 7 5
129312024-04-17T17:18:42.923Z03/30 Read block 171 len 24576
129322024-04-17T17:18:42.923Z04/30 Write block 206 len 20480 data: 15 17 9 10 10
129332024-04-17T17:18:42.923Z05/30 Write block 90 len 20480 data: 13 14 15 14 12
129342024-04-17T17:18:42.923Z06/30 Read block 155 len 40960
129352024-04-17T17:18:42.923Z07/30 Read block 1 len 16384
129362024-04-17T17:18:42.924Z08/30 Read block 433 len 8192
129372024-04-17T17:18:42.924Z09/30 Read block 138 len 20480
129382024-04-17T17:18:42.924Z10/30 Write block 289 len 24576 data: 8 9 8 7 9 8
129392024-04-17T17:18:42.924Z11/30 Read block 107 len 24576
129402024-04-17T17:18:42.928Z12/30 Flush
129412024-04-17T17:18:42.928Z13/30 Write block 188 len 16384 data: 13 12 10 8
129422024-04-17T17:18:42.928Z14/30 Write block 145 len 12288 data: 10 9 7
129432024-04-17T17:18:42.928Z15/30 Flush
129442024-04-17T17:18:42.932Z16/30 Write block 271 len 12288 data: 10 10 12
129452024-04-17T17:18:42.932Z17/30 Read block 174 len 12288
129462024-04-17T17:18:42.932Z18/30 Write block 483 len 4096 data: 19
129472024-04-17T17:18:42.936Z19/30 Read block 216 len 28672
129482024-04-17T17:18:42.936Z20/30 Read block 331 len 32768
129492024-04-17T17:18:42.936Z21/30 Read block 471 len 16384
129502024-04-17T17:18:42.936Z22/30 Read block 364 len 4096
129512024-04-17T17:18:42.936Z23/30 Flush
129522024-04-17T17:18:42.943Z24/30 Write block 176 len 40960 data: 8 7 10 9 6 6 6 6 5 4
129532024-04-17T17:18:42.943Z25/30 Write block 469 len 40960 data: 10 11 12 11 11 12 11 12 16 17
129542024-04-17T17:18:42.943Z26/30 Write block 99 len 20480 data: 13 13 11 10 12
129552024-04-17T17:18:42.947Z27/30 Write block 360 len 24576 data: 7 7 7 6 7 6
129562024-04-17T17:18:42.947Z28/30 Read block 198 len 20480
129572024-04-17T17:18:42.948Z29/30 Write block 510 len 20480 data: 9 8 10 6 10
129582024-04-17T17:18:42.948Z30/30 Read block 179 len 8192
129592024-04-17T17:18:42.948Z----------------------------------------------------------------
129602024-04-17T17:18:42.948Z Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:30
129612024-04-17T17:18:42.948ZStates: Active Active Active
129622024-04-17T17:18:42.948ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
129632024-04-17T17:18:42.948Z 1 Acked 1000 Write 7 Done Done Done false
129642024-04-17T17:18:42.948Z 2 Acked 1001 Write 7 Done Done Done false
129652024-04-17T17:18:42.948Z 3 Acked 1002 Read 6 Done Done Done false
129662024-04-17T17:18:42.948Z 4 Acked 1003 Write 5 Done Done Done false
129672024-04-17T17:18:42.948Z 5 Acked 1004 Write 5 Done Done Done false
129682024-04-17T17:18:42.948Z 6 Acked 1005 Read 10 Done Done Done false
129692024-04-17T17:18:42.948Z 7 Acked 1006 Read 4 Done Done Done false
129702024-04-17T17:18:42.948Z 8 Acked 1007 Read 2 Done Done Done false
129712024-04-17T17:18:42.948Z 9 Acked 1008 Read 5 Done Done Done false
129722024-04-17T17:18:42.948Z 10 Acked 1009 Write 6 Done Done Done false
129732024-04-17T17:18:42.948Z 11 Acked 1010 Read 6 Sent Done Done false
129742024-04-17T17:18:42.948Z 12 Acked 1011 Flush 0 Sent Done Done false
129752024-04-17T17:18:42.948Z 13 Acked 1012 Write 4 Sent Done Done false
129762024-04-17T17:18:42.948Z 14 Acked 1013 Write 3 Sent Done Done false
129772024-04-17T17:18:42.948Z 15 Acked 1014 Flush 0 Sent Done Done false
129782024-04-17T17:18:42.948Z 16 Acked 1015 Write 3 Sent Done Done false
129792024-04-17T17:18:42.948Z 17 Acked 1016 Read 3 Sent Done Done false
129802024-04-17T17:18:42.949Z 18 Acked 1017 Write 1 Sent Done Done false
129812024-04-17T17:18:42.949Z 19 Acked 1018 Read 7 Sent Done Done false
129822024-04-17T17:18:42.949Z 20 Acked 1019 Read 8 Sent Done Done false
129832024-04-17T17:18:42.949Z 21 Acked 1020 Read 4 Sent Done Done false
129842024-04-17T17:18:42.949Z 22 Acked 1021 Read 1 Sent Done Done false
129852024-04-17T17:18:42.949Z 23 Acked 1022 Flush 0 Sent Done Done false
129862024-04-17T17:18:42.949Z 24 Acked 1023 Write 10 Sent Done Done false
129872024-04-17T17:18:42.949Z 25 Acked 1024 Write 10 Sent Done Done false
129882024-04-17T17:18:42.949Z 26 Acked 1025 Write 5 Sent Done Done false
129892024-04-17T17:18:42.949Z 27 Acked 1026 Write 6 Sent Done Done false
129902024-04-17T17:18:42.949Z 28 Acked 1027 Read 5 Sent Done Done false
129912024-04-17T17:18:42.949Z 29 Acked 1028 Write 5 Sent Done Done false
129922024-04-17T17:18:42.949Z 30 Acked 1029 Read 2 Sent Done Sent false
129932024-04-17T17:18:42.949Z STATES DS:0 DS:1 DS:2 TOTAL
129942024-04-17T17:18:42.949Z New 0 0 0 0
129952024-04-17T17:18:42.949Z Sent 20 0 1 21
129962024-04-17T17:18:42.949Z Done 10 30 29 69
129972024-04-17T17:18:42.949Z Skipped 0 0 0 0
129982024-04-17T17:18:42.949Z Error 0 0 0 0
129992024-04-17T17:18:42.949ZLast Flush: 0 1022 1022
130002024-04-17T17:18:42.949ZDownstairs last five completed:
130012024-04-17T17:18:42.949ZUpstairs last five completed: 30 29 28 27 26
130022024-04-17T17:18:42.949Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:18:41.553154231Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1651}
130032024-04-17T17:18:42.950ZWrote out file "/tmp/test_repair_verify.data"
130042024-04-17T17:18:42.955Z
130052024-04-17T17:18:42.955ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
130062024-04-17T17:18:42.969ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
130072024-04-17T17:18:42.969Z 0 000-019 114 114 114 207 207 207 F F F
130082024-04-17T17:18:42.970Z 1 020-039 114 114 114 207 207 207 F F F
130092024-04-17T17:18:42.970Z 2 040-059 114 114 114 207 207 207 F F F
130102024-04-17T17:18:42.970Z 3 060-079 118 118 118 211 211 211 F F F
130112024-04-17T17:18:42.970Z 4 080-099 117 120 120 209 213 213 T T T <---
130122024-04-17T17:18:42.970Z 5 100-119 118 118 118 211 211 211 F T T <---
130132024-04-17T17:18:42.970Z 6 120-139 118 118 118 211 211 211 F F F
130142024-04-17T17:18:42.970Z 7 140-159 110 120 120 200 214 214 T F F <---
130152024-04-17T17:18:42.970Z 8 160-179 118 118 118 211 211 211 F T T <---
130162024-04-17T17:18:42.974Z 9 180-199 117 120 120 209 214 214 F T T <---
130172024-04-17T17:18:42.974Z 10 200-219 118 120 120 211 213 213 T F F <---
130182024-04-17T17:18:42.974Z 11 220-239 118 118 118 211 211 211 F F F
130192024-04-17T17:18:42.974Z 12 240-259 118 118 118 211 211 211 F F F
130202024-04-17T17:18:42.975Z 13 260-279 118 120 120 210 215 215 F F F <---
130212024-04-17T17:18:42.975Z 14 280-299 116 120 120 208 213 213 T F F <---
130222024-04-17T17:18:42.975Z 15 300-319 116 116 116 208 208 208 F F F
130232024-04-17T17:18:42.975Z 16 320-339 116 120 120 208 213 213 T F F <---
130242024-04-17T17:18:42.975Z 17 340-359 118 118 118 211 211 211 F F F
130252024-04-17T17:18:42.975Z 18 360-379 116 116 116 208 208 208 F T T <---
130262024-04-17T17:18:42.975Z 19 380-399 118 118 118 210 210 210 F F F
130272024-04-17T17:18:42.975Z 20 400-419 118 118 118 210 210 210 F F F
130282024-04-17T17:18:42.975Z 21 420-439 116 116 116 208 208 208 F F F
130292024-04-17T17:18:42.975Z 22 440-459 116 116 116 208 208 208 F F F
130302024-04-17T17:18:42.975Z 23 460-479 118 118 118 211 211 211 F T T <---
130312024-04-17T17:18:42.975Z 24 480-499 119 120 120 212 215 215 F F F <---
130322024-04-17T17:18:42.975Z 25 500-519 118 118 118 211 211 211 F T T <---
130332024-04-17T17:18:42.975Z 26 520-539 117 117 117 209 209 209 F F F
130342024-04-17T17:18:42.975Z 27 540-559 114 114 114 207 207 207 F F F
130352024-04-17T17:18:42.975Z 28 560-579 118 118 118 210 210 210 F F F
130362024-04-17T17:18:42.975Z 29 580-599 114 114 114 207 207 207 F F F
130372024-04-17T17:18:42.975ZMax gen: 120, Max flush: 215
130382024-04-17T17:18:42.975ZError: Difference in extent metadata found!
130392024-04-17T17:18:42.975ZA Difference in extent metadata is expected here
130402024-04-17T17:18:42.975ZOn loop 60
130412024-04-17T17:18:42.975Z
130422024-04-17T17:18:42.981ZVerifying data now
130432024-04-17T17:18:57.575ZNo change after verify
130442024-04-17T17:18:57.575ZLoop: 60 Downstairs dump after verify (and repair):
130452024-04-17T17:18:57.590ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
130462024-04-17T17:18:57.590Z 0 000-019 114 114 114 207 207 207 F F F
130472024-04-17T17:18:57.590Z 1 020-039 114 114 114 207 207 207 F F F
130482024-04-17T17:18:57.590Z 2 040-059 114 114 114 207 207 207 F F F
130492024-04-17T17:18:57.590Z 3 060-079 118 118 118 211 211 211 F F F
130502024-04-17T17:18:57.590Z 4 080-099 121 121 121 216 216 216 F F F
130512024-04-17T17:18:57.594Z 5 100-119 121 121 121 216 216 216 F F F
130522024-04-17T17:18:57.594Z 6 120-139 118 118 118 211 211 211 F F F
130532024-04-17T17:18:57.594Z 7 140-159 120 120 120 214 214 214 F F F
130542024-04-17T17:18:57.594Z 8 160-179 121 121 121 216 216 216 F F F
130552024-04-17T17:18:57.594Z 9 180-199 121 121 121 216 216 216 F F F
130562024-04-17T17:18:57.594Z 10 200-219 120 120 120 213 213 213 F F F
130572024-04-17T17:18:57.594Z 11 220-239 118 118 118 211 211 211 F F F
130582024-04-17T17:18:57.594Z 12 240-259 118 118 118 211 211 211 F F F
130592024-04-17T17:18:57.594Z 13 260-279 120 120 120 215 215 215 F F F
130602024-04-17T17:18:57.594Z 14 280-299 120 120 120 213 213 213 F F F
130612024-04-17T17:18:57.595Z 15 300-319 116 116 116 208 208 208 F F F
130622024-04-17T17:18:57.595Z 16 320-339 120 120 120 213 213 213 F F F
130632024-04-17T17:18:57.595Z 17 340-359 118 118 118 211 211 211 F F F
130642024-04-17T17:18:57.595Z 18 360-379 121 121 121 216 216 216 F F F
130652024-04-17T17:18:57.595Z 19 380-399 118 118 118 210 210 210 F F F
130662024-04-17T17:18:57.595Z 20 400-419 118 118 118 210 210 210 F F F
130672024-04-17T17:18:57.595Z 21 420-439 116 116 116 208 208 208 F F F
130682024-04-17T17:18:57.595Z 22 440-459 116 116 116 208 208 208 F F F
130692024-04-17T17:18:57.595Z 23 460-479 121 121 121 216 216 216 F F F
130702024-04-17T17:18:57.595Z 24 480-499 120 120 120 215 215 215 F F F
130712024-04-17T17:18:57.595Z 25 500-519 121 121 121 216 216 216 F F F
130722024-04-17T17:18:57.595Z 26 520-539 117 117 117 209 209 209 F F F
130732024-04-17T17:18:57.595Z 27 540-559 114 114 114 207 207 207 F F F
130742024-04-17T17:18:57.595Z 28 560-579 118 118 118 210 210 210 F F F
130752024-04-17T17:18:57.595Z 29 580-599 114 114 114 207 207 207 F F F
130762024-04-17T17:18:57.595ZMax gen: 121, Max flush: 216
130772024-04-17T17:18:57.595Z
130782024-04-17T17:18:57.595ZBegin loop 61
130792024-04-17T17:18:57.595ZDownstairs to restart: 2
130802024-04-17T17:18:57.614Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.22252486Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659}
130812024-04-17T17:18:57.614Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.222773405Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659}
130822024-04-17T17:18:57.621Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.222807607Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659}
130832024-04-17T17:18:57.621Z{"msg":"Crucible edef72dd-a337-4cab-9f77-d8a2413ef0fb has session id: aaca3bba-3017-415d-aee1-ed96b5e207f4","v":0,"name":"crucible","level":30Crucible runtime is spawned
130842024-04-17T17:18:57.621Z,"time":"2024-04-17T17:18:56.22285568Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
130852024-04-17T17:18:57.621Z{"msg":"Upstairs opts: Upstairs UUID: edef72dd-a337-4cab-9f77-d8a2413ef0fb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.222969516Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
130862024-04-17T17:18:57.622Z{"msg":"Crucible stats registered with UUID: edef72dd-a337-4cab-9f77-d8a2413ef0fb","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.223014999Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
130872024-04-17T17:18:57.622Z{"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.223043401Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659}
130882024-04-17T17:18:57.622Z{"msg":"edef72dd-a337-4cab-9f77-d8a2413ef0fb active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.223081743Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
130892024-04-17T17:18:57.622Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.223130626Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"client":"0","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
130902024-04-17T17:18:57.622Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.223169689Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"client":"0","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
130912024-04-17T17:18:57.622Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.223206261Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"client":"1","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
130922024-04-17T17:18:57.622Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.223231192Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"client":"1","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
130932024-04-17T17:18:57.622Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.223263464Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"client":"2","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
130942024-04-17T17:18:57.622Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.223288626Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"client":"2","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
130952024-04-17T17:18:57.622Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.223320578Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"":"io task","client":"2","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
130962024-04-17T17:18:57.622Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.22336155Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"":"io task","client":"0","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
130972024-04-17T17:18:57.622Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.223387942Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"":"io task","client":"1","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
130982024-04-17T17:18:57.622Z{"msg":"ds_connection connected from Ok(127.0.0.1:52438)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.223413194Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"":"io task","client":"1","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
130992024-04-17T17:18:57.622Z{"msg":"ds_connection connected from Ok(127.0.0.1:61058)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.223547662Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"":"io task","client":"0","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131002024-04-17T17:18:57.622Z{"msg":"ds_connection connected from Ok(127.0.0.1:55967)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.223585014Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"":"io task","client":"2","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131012024-04-17T17:18:57.622Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.224084834Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"client":"0","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131022024-04-17T17:18:57.622Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.224120946Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"client":"0","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131032024-04-17T17:18:57.622Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.224139167Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"client":"1","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131042024-04-17T17:18:57.622Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.224163089Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"client":"1","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131052024-04-17T17:18:57.622Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.224432505Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"client":"2","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131062024-04-17T17:18:57.622Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.224480748Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"client":"2","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131072024-04-17T17:18:57.622Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.224794717Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"client":"1","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131082024-04-17T17:18:57.623Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.224853291Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"client":"0","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131092024-04-17T17:18:57.623Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.225140888Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"client":"2","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131102024-04-17T17:18:57.623Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.225324699Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"client":"0","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131112024-04-17T17:18:57.623Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.225372102Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"client":"0","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131122024-04-17T17:18:57.623Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.225407524Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131132024-04-17T17:18:57.623Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.225439446Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"client":"1","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131142024-04-17T17:18:57.623Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.225472478Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"client":"1","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131152024-04-17T17:18:57.623Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.22549575Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131162024-04-17T17:18:57.623Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.225676711Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"client":"2","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131172024-04-17T17:18:57.623Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.225712243Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"client":"2","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131182024-04-17T17:18:57.623Z{"msg":"[0]R flush_numbers[0..12]: [207, 207, 207, 211, 216, 216, 211, 214, 216, 216, 213, 211]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.225731504Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"":"downstairs","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131192024-04-17T17:18:57.623Z{"msg":"[0]R generation[0..12]: [114, 114, 114, 118, 121, 121, 118, 120, 121, 121, 120, 118]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.225762446Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"":"downstairs","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131202024-04-17T17:18:57.623Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.225793588Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"":"downstairs","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131212024-04-17T17:18:57.623Z{"msg":"[1]R flush_numbers[0..12]: [207, 207, 207, 211, 216, 216, 211, 214, 216, 216, 213, 211]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.22582666Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"":"downstairs","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131222024-04-17T17:18:57.623ZWait for a query_work_queue command to finish before sending IO
131232024-04-17T17:18:57.623Z{"msg":"[1]R generation[0..12]: [114, 114, 114, 118, 121, 121, 118, 120, 121, 121, 120, 118]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.225860652Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"":"downstairs","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131242024-04-17T17:18:57.623Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.225893194Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"":"downstairs","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131252024-04-17T17:18:57.623Z{"msg":"[2]R flush_numbers[0..12]: [207, 207, 207, 211, 216, 216, 211, 214, 216, 216, 213, 211]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.225925486Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"":"downstairs","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131262024-04-17T17:18:57.623Z{"msg":"[2]R generation[0..12]: [114, 114, 114, 118, 121, 121, 118, 120, 121, 121, 120, 118]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.225961038Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"":"downstairs","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131272024-04-17T17:18:57.623Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.225985469Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"":"downstairs","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131282024-04-17T17:18:57.623Z{"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.226023392Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"":"downstairs","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131292024-04-17T17:18:57.623Z{"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.226046403Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"":"downstairs","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131302024-04-17T17:18:57.623Z{"msg":"Next flush: 217","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.226077545Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"":"downstairs","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131312024-04-17T17:18:57.623Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.226101686Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"":"downstairs","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131322024-04-17T17:18:57.624Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.226135848Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131332024-04-17T17:18:57.624Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.226170261Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"client":"0","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131342024-04-17T17:18:57.624ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
131352024-04-17T17:18:57.624Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.226209073Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"client":"1","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131362024-04-17T17:18:57.624Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.226244865Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"client":"2","":"downstairs","session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131372024-04-17T17:18:57.624Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.226268087Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131382024-04-17T17:18:57.624Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.226280917Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131392024-04-17T17:18:57.624Z{"msg":"edef72dd-a337-4cab-9f77-d8a2413ef0fb is now active with session: aaca3bba-3017-415d-aee1-ed96b5e207f4","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.226300258Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131402024-04-17T17:18:57.624Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.226312659Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659,"session_id":"aaca3bba-3017-415d-aee1-ed96b5e207f4"}
131412024-04-17T17:18:57.624Z{"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:18:56.22633114Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659}
131422024-04-17T17:18:57.624ZLoading write count information from file "/tmp/test_repair_verify.data"
131432024-04-17T17:18:57.624ZRun Repair workload
131442024-04-17T17:18:57.628Z01/30 Write block 568 len 32768 data: 10 9 7 7 9 10 13 13
131452024-04-17T17:18:57.628Z02/30 Write block 166 len 8192 data: 11 12
131462024-04-17T17:18:57.628Z03/30 Flush
131472024-04-17T17:18:57.628Z04/30 Write block 285 len 8192 data: 7 5
131482024-04-17T17:18:57.628Z05/30 Write block 318 len 4096 data: 13
131492024-04-17T17:18:57.628Z06/30 Write block 145 len 20480 data: 11 10 8 7 9
131502024-04-17T17:18:57.629Z07/30 Read block 288 len 20480
131512024-04-17T17:18:57.629Z08/30 Read block 573 len 36864
131522024-04-17T17:18:57.629Z09/30 Write block 382 len 16384 data: 8 9 8 10
131532024-04-17T17:18:57.629Z10/30 Read block 20 len 40960
131542024-04-17T17:18:57.629Z11/30 Read block 164 len 20480
131552024-04-17T17:18:57.632Z12/30 Write block 517 len 16384 data: 6 7 6 6
131562024-04-17T17:18:57.632Z13/30 Read block 507 len 4096
131572024-04-17T17:18:57.632Z14/30 Read block 221 len 28672
131582024-04-17T17:18:57.632Z15/30 Read block 290 len 16384
131592024-04-17T17:18:57.632Z16/30 Write block 465 len 36864 data: 12 11 10 10 11 12 13 12 12
131602024-04-17T17:18:57.632Z17/30 Read block 479 len 20480
131612024-04-17T17:18:57.636Z18/30 Flush
131622024-04-17T17:18:57.639Z19/30 Write block 542 len 32768 data: 13 13 12 9 9 9 12 14
131632024-04-17T17:18:57.639Z20/30 Read block 327 len 12288
131642024-04-17T17:18:57.639Z21/30 Write block 172 len 32768 data: 6 7 5 8 9 8 11 10
131652024-04-17T17:18:57.643Z22/30 Write block 432 len 32768 data: 8 11 10 10 10 10 10 11
131662024-04-17T17:18:57.643Z23/30 Write block 202 len 16384 data: 15 16 17 16
131672024-04-17T17:18:57.643Z24/30 Write block 36 len 16384 data: 14 12 10 10
131682024-04-17T17:18:57.643Z25/30 Read block 456 len 16384
131692024-04-17T17:18:57.646Z26/30 Read block 591 len 4096
131702024-04-17T17:18:57.646Z27/30 Write block 162 len 16384 data: 11 13 11 9
131712024-04-17T17:18:57.646Z28/30 Read block 216 len 20480
131722024-04-17T17:18:57.646Z29/30 Write block 121 len 40960 data: 10 8 8 8 9 9 9 7 8 10
131732024-04-17T17:18:57.653Z30/30 Read block 55 len 32768
131742024-04-17T17:18:57.653Z----------------------------------------------------------------
131752024-04-17T17:18:57.653Z Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:30
131762024-04-17T17:18:57.653ZStates: Active Active Active
131772024-04-17T17:18:57.653ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
131782024-04-17T17:18:57.653Z 1 Acked 1000 Write 8 Done Done Done false
131792024-04-17T17:18:57.653Z 2 Acked 1001 Write 2 Done Done Sent false
131802024-04-17T17:18:57.653Z 3 Acked 1002 Flush 0 Done Done Sent false
131812024-04-17T17:18:57.653Z 4 Acked 1003 Write 2 Done Done Sent false
131822024-04-17T17:18:57.653Z 5 Acked 1004 Write 1 Done Done Sent false
131832024-04-17T17:18:57.653Z 6 Acked 1005 Write 5 Done Done Sent false
131842024-04-17T17:18:57.653Z 7 Acked 1006 Read 5 Done Done Sent false
131852024-04-17T17:18:57.653Z 8 Acked 1007 Read 9 Done Done Sent false
131862024-04-17T17:18:57.653Z 9 Acked 1008 Write 4 Done Done Sent false
131872024-04-17T17:18:57.653Z 10 Acked 1009 Read 10 Done Done Sent false
131882024-04-17T17:18:57.653Z 11 Acked 1010 Read 5 Done Done Sent false
131892024-04-17T17:18:57.654Z 12 Acked 1011 Write 4 Done Done Sent false
131902024-04-17T17:18:57.654Z 13 Acked 1012 Read 1 Done Done Sent false
131912024-04-17T17:18:57.654Z 14 Acked 1013 Read 7 Done Done Sent false
131922024-04-17T17:18:57.654Z 15 Acked 1014 Read 4 Done Done Sent false
131932024-04-17T17:18:57.654Z 16 Acked 1015 Write 9 Done Done Sent false
131942024-04-17T17:18:57.654Z 17 Acked 1016 Read 5 Done Done Sent false
131952024-04-17T17:18:57.654Z 18 Acked 1017 Flush 0 Done Done Sent false
131962024-04-17T17:18:57.654Z 19 Acked 1018 Write 8 Done Done Sent false
131972024-04-17T17:18:57.654Z 20 Acked 1019 Read 3 Done Done Sent false
131982024-04-17T17:18:57.654Z 21 Acked 1020 Write 8 Done Done Sent false
131992024-04-17T17:18:57.654Z 22 Acked 1021 Write 8 Done Done Sent false
132002024-04-17T17:18:57.654Z 23 Acked 1022 Write 4 Done Done Sent false
132012024-04-17T17:18:57.654Z 24 Acked 1023 Write 4 Done Done Sent false
132022024-04-17T17:18:57.654Z 25 Acked 1024 Read 4 Done Done Sent false
132032024-04-17T17:18:57.654Z 26 Acked 1025 Read 1 Done Done Sent false
132042024-04-17T17:18:57.654Z 27 Acked 1026 Write 4 Done Done Sent false
132052024-04-17T17:18:57.654Z 28 Acked 1027 Read 5 Done Done Sent false
132062024-04-17T17:18:57.654Z 29 Acked 1028 Write 10 Done Done Sent false
132072024-04-17T17:18:57.654Z 30 Acked 1029 Read 8 Done Sent Sent false
132082024-04-17T17:18:57.654Z STATES DS:0 DS:1 DS:2 TOTAL
132092024-04-17T17:18:57.654Z New 0 0 0 0
132102024-04-17T17:18:57.654Z Sent 0 1 29 30
132112024-04-17T17:18:57.654Z Done 30 29 1 60
132122024-04-17T17:18:57.654Z Skipped 0 0 0 0
132132024-04-17T17:18:57.655Z Error 0 0 0 0
132142024-04-17T17:18:57.655ZLast Flush: 1017 1017 0
132152024-04-17T17:18:57.655ZDownstairs last five completed:
132162024-04-17T17:18:57.655ZUpstairs last five completed: 30 29 28 27 26
132172024-04-17T17:18:57.655Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:18:56.257083774Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1659}
132182024-04-17T17:18:57.655ZWrote out file "/tmp/test_repair_verify.data"
132192024-04-17T17:18:57.658Z
132202024-04-17T17:18:57.658ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
132212024-04-17T17:18:57.673ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
132222024-04-17T17:18:57.673Z 0 000-019 114 114 114 207 207 207 F F F
132232024-04-17T17:18:57.673Z 1 020-039 114 114 114 207 207 207 T T F <---
132242024-04-17T17:18:57.673Z 2 040-059 114 114 114 207 207 207 F F F
132252024-04-17T17:18:57.673Z 3 060-079 118 118 118 211 211 211 F F F
132262024-04-17T17:18:57.673Z 4 080-099 121 121 121 216 216 216 F F F
132272024-04-17T17:18:57.673Z 5 100-119 121 121 121 216 216 216 F F F
132282024-04-17T17:18:57.673Z 6 120-139 118 118 118 211 211 211 T T F <---
132292024-04-17T17:18:57.673Z 7 140-159 122 122 120 218 218 214 F F F <---
132302024-04-17T17:18:57.673Z 8 160-179 122 122 121 217 217 216 T T F <---
132312024-04-17T17:18:57.677Z 9 180-199 121 121 121 216 216 216 F F F
132322024-04-17T17:18:57.677Z 10 200-219 120 120 120 213 213 213 T T F <---
132332024-04-17T17:18:57.677Z 11 220-239 118 118 118 211 211 211 F F F
132342024-04-17T17:18:57.677Z 12 240-259 118 118 118 211 211 211 F F F
132352024-04-17T17:18:57.677Z 13 260-279 120 120 120 215 215 215 F F F
132362024-04-17T17:18:57.677Z 14 280-299 122 122 120 218 218 213 F F F <---
132372024-04-17T17:18:57.677Z 15 300-319 122 122 116 218 218 208 F F F <---
132382024-04-17T17:18:57.677Z 16 320-339 120 120 120 213 213 213 F F F
132392024-04-17T17:18:57.677Z 17 340-359 118 118 118 211 211 211 F F F
132402024-04-17T17:18:57.677Z 18 360-379 121 121 121 216 216 216 F F F
132412024-04-17T17:18:57.678Z 19 380-399 122 122 118 218 218 210 F F F <---
132422024-04-17T17:18:57.678Z 20 400-419 118 118 118 210 210 210 F F F
132432024-04-17T17:18:57.678Z 21 420-439 116 116 116 208 208 208 T T F <---
132442024-04-17T17:18:57.678Z 22 440-459 116 116 116 208 208 208 F F F
132452024-04-17T17:18:57.678Z 23 460-479 122 122 121 218 218 216 F F F <---
132462024-04-17T17:18:57.678Z 24 480-499 120 120 120 215 215 215 F F F
132472024-04-17T17:18:57.678Z 25 500-519 122 122 121 218 218 216 F F F <---
132482024-04-17T17:18:57.678Z 26 520-539 122 122 117 218 218 209 F F F <---
132492024-04-17T17:18:57.678Z 27 540-559 114 114 114 207 207 207 T T F <---
132502024-04-17T17:18:57.678Z 28 560-579 122 122 118 217 217 210 F F T <---
132512024-04-17T17:18:57.678Z 29 580-599 114 114 114 207 207 207 F F F
132522024-04-17T17:18:57.678ZMax gen: 122, Max flush: 218
132532024-04-17T17:18:57.678ZError: Difference in extent metadata found!
132542024-04-17T17:18:57.678ZA Difference in extent metadata is expected here
132552024-04-17T17:18:57.678ZOn loop 61
132562024-04-17T17:18:57.678Z
132572024-04-17T17:18:57.682ZVerifying data now
132582024-04-17T17:19:02.341ZNo change after verify
132592024-04-17T17:19:02.341ZLoop: 61 Downstairs dump after verify (and repair):
132602024-04-17T17:19:02.356ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
132612024-04-17T17:19:02.356Z 0 000-019 114 114 114 207 207 207 F F F
132622024-04-17T17:19:02.360Z 1 020-039 123 123 123 219 219 219 F F F
132632024-04-17T17:19:02.360Z 2 040-059 114 114 114 207 207 207 F F F
132642024-04-17T17:19:02.360Z 3 060-079 118 118 118 211 211 211 F F F
132652024-04-17T17:19:02.360Z 4 080-099 121 121 121 216 216 216 F F F
132662024-04-17T17:19:02.360Z 5 100-119 121 121 121 216 216 216 F F F
132672024-04-17T17:19:02.360Z 6 120-139 123 123 123 219 219 219 F F F
132682024-04-17T17:19:02.360Z 7 140-159 122 122 122 218 218 218 F F F
132692024-04-17T17:19:02.360Z 8 160-179 123 123 123 219 219 219 F F F
132702024-04-17T17:19:02.361Z 9 180-199 121 121 121 216 216 216 F F F
132712024-04-17T17:19:02.361Z 10 200-219 123 123 123 219 219 219 F F F
132722024-04-17T17:19:02.361Z 11 220-239 118 118 118 211 211 211 F F F
132732024-04-17T17:19:02.361Z 12 240-259 118 118 118 211 211 211 F F F
132742024-04-17T17:19:02.361Z 13 260-279 120 120 120 215 215 215 F F F
132752024-04-17T17:19:02.361Z 14 280-299 122 122 122 218 218 218 F F F
132762024-04-17T17:19:02.361Z 15 300-319 122 122 122 218 218 218 F F F
132772024-04-17T17:19:02.361Z 16 320-339 120 120 120 213 213 213 F F F
132782024-04-17T17:19:02.361Z 17 340-359 118 118 118 211 211 211 F F F
132792024-04-17T17:19:02.361Z 18 360-379 121 121 121 216 216 216 F F F
132802024-04-17T17:19:02.361Z 19 380-399 122 122 122 218 218 218 F F F
132812024-04-17T17:19:02.361Z 20 400-419 118 118 118 210 210 210 F F F
132822024-04-17T17:19:02.361Z 21 420-439 123 123 123 219 219 219 F F F
132832024-04-17T17:19:02.361Z 22 440-459 116 116 116 208 208 208 F F F
132842024-04-17T17:19:02.361Z 23 460-479 122 122 122 218 218 218 F F F
132852024-04-17T17:19:02.361Z 24 480-499 120 120 120 215 215 215 F F F
132862024-04-17T17:19:02.361Z 25 500-519 122 122 122 218 218 218 F F F
132872024-04-17T17:19:02.361Z 26 520-539 122 122 122 218 218 218 F F F
132882024-04-17T17:19:02.361Z 27 540-559 123 123 123 219 219 219 F F F
132892024-04-17T17:19:02.361Z 28 560-579 122 122 122 217 217 217 F F F
132902024-04-17T17:19:02.361Z 29 580-599 114 114 114 207 207 207 F F F
132912024-04-17T17:19:02.361ZMax gen: 123, Max flush: 219
132922024-04-17T17:19:02.361Z
132932024-04-17T17:19:02.362ZBegin loop 62
132942024-04-17T17:19:02.362ZDownstairs to restart: 1
132952024-04-17T17:19:02.377Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.986200723Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667}
132962024-04-17T17:19:02.377Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.986428697Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667}
132972024-04-17T17:19:02.387Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.986461808Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667}
132982024-04-17T17:19:02.387Z{"msg":"Crucible 5cd4087a-f0d8-424d-a0a4-d4b54de5d7af has session id: 00501fea-4683-47ad-9e32-59f3ae4e2c18","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.9864859Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
132992024-04-17T17:19:02.388Z{"msg":"Upstairs opts: Upstairs UUID: 5cd4087a-f0d8-424d-a0a4-d4b54de5d7af, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.986553984Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133002024-04-17T17:19:02.388Z{"msg":"Crucible runtime is spawned
133012024-04-17T17:19:02.388ZCrucible stats registered with UUID: 5cd4087a-f0d8-424d-a0a4-d4b54de5d7af","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.986608228Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133022024-04-17T17:19:02.388Z{"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.986661551Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667}
133032024-04-17T17:19:02.388Z{"msg":"5cd4087a-f0d8-424d-a0a4-d4b54de5d7af active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.986808049Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133042024-04-17T17:19:02.388Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.986903565Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"client":"0","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133052024-04-17T17:19:02.388Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.986970209Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"client":"0","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133062024-04-17T17:19:02.388Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.987025163Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"client":"1","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133072024-04-17T17:19:02.388Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.987062805Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"client":"1","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133082024-04-17T17:19:02.388Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.987088257Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"client":"2","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133092024-04-17T17:19:02.388Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.987115468Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"client":"2","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133102024-04-17T17:19:02.388Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.987201424Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"":"io task","client":"2","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133112024-04-17T17:19:02.388Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.987237496Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"":"io task","client":"0","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133122024-04-17T17:19:02.388Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.987274228Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"":"io task","client":"1","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133132024-04-17T17:19:02.388Z{"msg":"ds_connection connected from Ok(127.0.0.1:52991)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.987301859Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"":"io task","client":"1","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133142024-04-17T17:19:02.388Z{"msg":"ds_connection connected from Ok(127.0.0.1:55752)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.987346262Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"":"io task","client":"2","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133152024-04-17T17:19:02.388Z{"msg":"ds_connection connected from Ok(127.0.0.1:58924)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.987388825Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"":"io task","client":"0","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133162024-04-17T17:19:02.388Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.987832402Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"client":"0","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133172024-04-17T17:19:02.388Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.987881265Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"client":"0","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133182024-04-17T17:19:02.388Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.987922357Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"client":"1","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133192024-04-17T17:19:02.388Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.98796257Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"client":"1","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133202024-04-17T17:19:02.388Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.987991631Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"client":"2","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133212024-04-17T17:19:02.388Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.988028303Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"client":"2","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133222024-04-17T17:19:02.389Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.988482081Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"client":"1","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133232024-04-17T17:19:02.389Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.988520343Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"client":"2","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133242024-04-17T17:19:02.389Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.988537074Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"client":"0","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133252024-04-17T17:19:02.389Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.989036875Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"client":"1","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133262024-04-17T17:19:02.389Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.989076817Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"client":"1","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133272024-04-17T17:19:02.389Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.989107719Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133282024-04-17T17:19:02.389Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.989136871Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"client":"0","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133292024-04-17T17:19:02.389Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.989161652Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"client":"0","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133302024-04-17T17:19:02.389Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.989188924Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133312024-04-17T17:19:02.389Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.989210415Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"client":"2","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133322024-04-17T17:19:02.389Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.989238377Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"client":"2"Wait for a query_work_queue command to finish before sending IO
133332024-04-17T17:19:02.389Z,"":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133342024-04-17T17:19:02.389Z{"msg":"[0]R flush_numbers[0..12]: [207, 219, 207, 211, 216, 216, 219, 218, 219, 216, 219, 211]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.989267729Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"":"downstairs","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133352024-04-17T17:19:02.389Z{"msg":"[0]R generation[0..12]: [114, 123, 114, 118, 121, 121, 123, 122, 123, 121, 123, 118]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.989300541Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"":"downstairs","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133362024-04-17T17:19:02.389Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.989326582Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"":"downstairs","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133372024-04-17T17:19:02.389Z{"msg":"[1]R flush_numbers[0..12]: [207, 219, 207, 211, 216, 216, 219, 218, 219, 216, 219, 211]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.989359184Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"":"downstairs","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133382024-04-17T17:19:02.389Z{"msg":"[1]R generation[0..12]: [114, 123, 114, 118, 121, 121, 123, 122, 123, 121, 123, 118]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.989397937Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"":"downstairs","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133392024-04-17T17:19:02.389Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.989423118Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"":"downstairs","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133402024-04-17T17:19:02.389Z{"msg":"[2]R flush_numbers[0..12]: [207, 219, 207, 211, 216, 216, 219, 218, 219, 216, 219, 211]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.9894588Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"":"downstairs","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133412024-04-17T17:19:02.389Z{"msg":"[2]R generation[0..12]: [114, 123, 114, 118, 121, 121, 123, 122, 123, 121, 123, 118]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.989498143Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"":"downstairs","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133422024-04-17T17:19:02.389Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
133432024-04-17T17:19:02.389Z,"time":"2024-04-17T17:19:00.989540255Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"":"downstairs","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133442024-04-17T17:19:02.390Z{"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.989577708Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"":"downstairs","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133452024-04-17T17:19:02.390Z{"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.989594549Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"":"downstairs","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133462024-04-17T17:19:02.390Z{"msg":"Next flush: 220","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.98961514Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"":"downstairs","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133472024-04-17T17:19:02.390Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.989628931Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"":"downstairs","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133482024-04-17T17:19:02.390Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.989650172Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133492024-04-17T17:19:02.390Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.989662313Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"client":"0","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133502024-04-17T17:19:02.390Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.989682564Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"client":"1","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133512024-04-17T17:19:02.390Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.989696075Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"client":"2","":"downstairs","session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133522024-04-17T17:19:02.390Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.989715546Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133532024-04-17T17:19:02.390Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.989728866Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133542024-04-17T17:19:02.390Z{"msg":"5cd4087a-f0d8-424d-a0a4-d4b54de5d7af is now active with session: 00501fea-4683-47ad-9e32-59f3ae4e2c18","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.989748258Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133552024-04-17T17:19:02.390Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.989760308Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667,"session_id":"00501fea-4683-47ad-9e32-59f3ae4e2c18"}
133562024-04-17T17:19:02.390Z{"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:00.98977933Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667}
133572024-04-17T17:19:02.390ZLoading write count information from file "/tmp/test_repair_verify.data"
133582024-04-17T17:19:02.390ZRun Repair workload
133592024-04-17T17:19:02.395Z01/30 Write block 63 len 40960 data: 13 14 12 11 11 10 8 8 8 8
133602024-04-17T17:19:02.395Z02/30 Read block 589 len 28672
133612024-04-17T17:19:02.395Z03/30 Write block 494 len 32768 data: 8 9 7 8 11 10 11 11
133622024-04-17T17:19:02.395Z04/30 Read block 130 len 28672
133632024-04-17T17:19:02.395Z05/30 Write block 136 len 32768 data: 10 9 8 8 7 6 6 7
133642024-04-17T17:19:02.395Z06/30 Read block 564 len 28672
133652024-04-17T17:19:02.395Z07/30 Write block 401 len 16384 data: 9 8 9 9
133662024-04-17T17:19:02.396Z08/30 Read block 48 len 12288
133672024-04-17T17:19:02.396Z09/30 Read block 551 len 32768
133682024-04-17T17:19:02.396Z10/30 Write block 50 len 36864 data: 7 8 8 9 13 11 11 13 14
133692024-04-17T17:19:02.396Z11/30 Write block 416 len 8192 data: 12 12
133702024-04-17T17:19:02.396Z12/30 Read block 575 len 4096
133712024-04-17T17:19:02.396Z13/30 Read block 249 len 16384
133722024-04-17T17:19:02.396Z14/30 Read block 560 len 4096
133732024-04-17T17:19:02.400Z15/30 Write block 210 len 36864 data: 11 11 11 11 11 13 11 8 9
133742024-04-17T17:19:02.400Z16/30 Write block 334 len 28672 data: 8 9 11 13 12 12 11
133752024-04-17T17:19:02.400Z17/30 Write block 255 len 12288 data: 7 11 11
133762024-04-17T17:19:02.400Z18/30 Read block 34 len 36864
133772024-04-17T17:19:02.400Z19/30 Read block 124 len 20480
133782024-04-17T17:19:02.404Z20/30 Read block 25 len 12288
133792024-04-17T17:19:02.404Z21/30 Read block 86 len 8192
133802024-04-17T17:19:02.404Z22/30 Read block 9 len 12288
133812024-04-17T17:19:02.404Z23/30 Read block 144 len 20480
133822024-04-17T17:19:02.404Z24/30 Flush
133832024-04-17T17:19:02.409Z25/30 Write block 301 len 16384 data: 8 7 9 10
133842024-04-17T17:19:02.417Z26/30 Write block 110 len 28672 data: 9 10 8 7 5 6 6
133852024-04-17T17:19:02.417Z27/30 Write block 285 len 4096 data: 8
133862024-04-17T17:19:02.417Z28/30 Write block 289 len 20480 data: 9 10 9 8 10
133872024-04-17T17:19:02.417Z29/30 Write block 563 len 8192 data: 11 13
133882024-04-17T17:19:02.417Z30/30 Write block 539 len 24576 data: 10 10 11 14 14 13
133892024-04-17T17:19:02.417Z----------------------------------------------------------------
133902024-04-17T17:19:02.417Z Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:30
133912024-04-17T17:19:02.417ZStates: Active Active Active
133922024-04-17T17:19:02.417ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
133932024-04-17T17:19:02.417Z 1 Acked 1000 Write 10 Done Done Done false
133942024-04-17T17:19:02.417Z 2 Acked 1001 Read 7 Done Done Done false
133952024-04-17T17:19:02.417Z 3 Acked 1002 Write 8 Done Sent Done false
133962024-04-17T17:19:02.417Z 4 Acked 1003 Read 7 Done Sent Done false
133972024-04-17T17:19:02.417Z 5 Acked 1004 Write 8 Done Sent Done false
133982024-04-17T17:19:02.417Z 6 Acked 1005 Read 7 Done Sent Done false
133992024-04-17T17:19:02.417Z 7 Acked 1006 Write 4 Done Sent Done false
134002024-04-17T17:19:02.417Z 8 Acked 1007 Read 3 Done Sent Done false
134012024-04-17T17:19:02.417Z 9 Acked 1008 Read 8 Done Sent Done false
134022024-04-17T17:19:02.417Z 10 Acked 1009 Write 9 Done Sent Done false
134032024-04-17T17:19:02.417Z 11 Acked 1010 Write 2 Done Sent Done false
134042024-04-17T17:19:02.417Z 12 Acked 1011 Read 1 Done Sent Done false
134052024-04-17T17:19:02.417Z 13 Acked 1012 Read 4 Done Sent Done false
134062024-04-17T17:19:02.417Z 14 Acked 1013 Read 1 Done Sent Done false
134072024-04-17T17:19:02.418Z 15 Acked 1014 Write 9 Done Sent Done false
134082024-04-17T17:19:02.418Z 16 Acked 1015 Write 7 Done Sent Done false
134092024-04-17T17:19:02.418Z 17 Acked 1016 Write 3 Done Sent Done false
134102024-04-17T17:19:02.418Z 18 Acked 1017 Read 9 Done Sent Done false
134112024-04-17T17:19:02.418Z 19 Acked 1018 Read 5 Done Sent Done false
134122024-04-17T17:19:02.418Z 20 Acked 1019 Read 3 Done Sent Done false
134132024-04-17T17:19:02.418Z 21 Acked 1020 Read 2 Done Sent Done false
134142024-04-17T17:19:02.418Z 22 Acked 1021 Read 3 Done Sent Done false
134152024-04-17T17:19:02.418Z 23 Acked 1022 Read 5 Done Sent Done false
134162024-04-17T17:19:02.418Z 24 Acked 1023 Flush 0 Done Sent Done false
134172024-04-17T17:19:02.418Z 25 Acked 1024 Write 4 Done Sent Done false
134182024-04-17T17:19:02.418Z 26 Acked 1025 Write 7 Done Sent Done false
134192024-04-17T17:19:02.418Z 27 Acked 1026 Write 1 Done Sent Done false
134202024-04-17T17:19:02.418Z 28 Acked 1027 Write 5 Done Sent Done false
134212024-04-17T17:19:02.418Z 29 Acked 1028 Write 2 Done Sent Done false
134222024-04-17T17:19:02.418Z 30 Acked 1029 Write 6 Sent Sent Sent false
134232024-04-17T17:19:02.418Z STATES DS:0 DS:1 DS:2 TOTAL
134242024-04-17T17:19:02.418Z New 0 0 0 0
134252024-04-17T17:19:02.418Z Sent 1 28 1 30
134262024-04-17T17:19:02.418Z Done 29 2 29 60
134272024-04-17T17:19:02.418Z Skipped 0 0 0 0
134282024-04-17T17:19:02.418Z Error 0 0 0 0
134292024-04-17T17:19:02.418ZLast Flush: 1023 0 1023
134302024-04-17T17:19:02.419ZDownstairs last five completed:
134312024-04-17T17:19:02.419ZUpstairs last five completed: 30 29 28 27 26
134322024-04-17T17:19:02.419Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:19:01.021877065Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1667}
134332024-04-17T17:19:02.419ZWrote out file "/tmp/test_repair_verify.data"
134342024-04-17T17:19:02.423Z
134352024-04-17T17:19:02.423ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
134362024-04-17T17:19:02.438ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
134372024-04-17T17:19:02.438Z 0 000-019 114 114 114 207 207 207 F F F
134382024-04-17T17:19:02.438Z 1 020-039 123 123 123 219 219 219 F F F
134392024-04-17T17:19:02.438Z 2 040-059 124 114 124 220 207 220 F F F <---
134402024-04-17T17:19:02.438Z 3 060-079 124 118 124 220 211 220 F T F <---
134412024-04-17T17:19:02.438Z 4 080-099 121 121 121 216 216 216 F F F
134422024-04-17T17:19:02.438Z 5 100-119 121 121 121 216 216 216 T F T <---
134432024-04-17T17:19:02.438Z 6 120-139 124 123 124 220 219 220 F F F <---
134442024-04-17T17:19:02.438Z 7 140-159 124 122 124 220 218 220 F F F <---
134452024-04-17T17:19:02.443Z 8 160-179 123 123 123 219 219 219 F F F
134462024-04-17T17:19:02.443Z 9 180-199 121 121 121 216 216 216 F F F
134472024-04-17T17:19:02.443Z 10 200-219 124 123 124 220 219 220 F F F <---
134482024-04-17T17:19:02.443Z 11 220-239 118 118 118 211 211 211 F F F
134492024-04-17T17:19:02.443Z 12 240-259 124 118 124 220 211 220 F F F <---
134502024-04-17T17:19:02.443Z 13 260-279 120 120 120 215 215 215 F F F
134512024-04-17T17:19:02.443Z 14 280-299 122 122 122 218 218 218 T F T <---
134522024-04-17T17:19:02.444Z 15 300-319 122 122 122 218 218 218 T F T <---
134532024-04-17T17:19:02.444Z 16 320-339 124 120 124 220 213 220 F F F <---
134542024-04-17T17:19:02.444Z 17 340-359 124 118 124 220 211 220 F F F <---
134552024-04-17T17:19:02.444Z 18 360-379 121 121 121 216 216 216 F F F
134562024-04-17T17:19:02.444Z 19 380-399 122 122 122 218 218 218 F F F
134572024-04-17T17:19:02.444Z 20 400-419 124 118 124 220 210 220 F F F <---
134582024-04-17T17:19:02.444Z 21 420-439 123 123 123 219 219 219 F F F
134592024-04-17T17:19:02.444Z 22 440-459 116 116 116 208 208 208 F F F
134602024-04-17T17:19:02.444Z 23 460-479 122 122 122 218 218 218 F F F
134612024-04-17T17:19:02.444Z 24 480-499 124 120 124 220 215 220 F F F <---
134622024-04-17T17:19:02.444Z 25 500-519 124 122 124 220 218 220 F F F <---
134632024-04-17T17:19:02.444Z 26 520-539 122 122 122 218 218 218 T F T <---
134642024-04-17T17:19:02.444Z 27 540-559 123 123 123 219 219 219 T F T <---
134652024-04-17T17:19:02.444Z 28 560-579 122 122 122 217 217 217 T F T <---
134662024-04-17T17:19:02.444Z 29 580-599 114 114 114 207 207 207 F F F
134672024-04-17T17:19:02.444ZMax gen: 124, Max flush: 220
134682024-04-17T17:19:02.444ZError: Difference in extent metadata found!
134692024-04-17T17:19:02.444ZA Difference in extent metadata is expected here
134702024-04-17T17:19:02.444ZOn loop 62
134712024-04-17T17:19:02.444Z
134722024-04-17T17:19:02.448ZVerifying data now
134732024-04-17T17:19:07.231ZNo change after verify
134742024-04-17T17:19:07.231ZLoop: 62 Downstairs dump after verify (and repair):
134752024-04-17T17:19:07.246ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
134762024-04-17T17:19:07.246Z 0 000-019 114 114 114 207 207 207 F F F
134772024-04-17T17:19:07.249Z 1 020-039 123 123 123 219 219 219 F F F
134782024-04-17T17:19:07.249Z 2 040-059 124 124 124 220 220 220 F F F
134792024-04-17T17:19:07.249Z 3 060-079 124 124 124 220 220 220 F F F
134802024-04-17T17:19:07.250Z 4 080-099 121 121 121 216 216 216 F F F
134812024-04-17T17:19:07.250Z 5 100-119 125 125 125 221 221 221 F F F
134822024-04-17T17:19:07.250Z 6 120-139 124 124 124 220 220 220 F F F
134832024-04-17T17:19:07.250Z 7 140-159 124 124 124 220 220 220 F F F
134842024-04-17T17:19:07.250Z 8 160-179 123 123 123 219 219 219 F F F
134852024-04-17T17:19:07.250Z 9 180-199 121 121 121 216 216 216 F F F
134862024-04-17T17:19:07.250Z 10 200-219 124 124 124 220 220 220 F F F
134872024-04-17T17:19:07.250Z 11 220-239 118 118 118 211 211 211 F F F
134882024-04-17T17:19:07.250Z 12 240-259 124 124 124 220 220 220 F F F
134892024-04-17T17:19:07.250Z 13 260-279 120 120 120 215 215 215 F F F
134902024-04-17T17:19:07.250Z 14 280-299 125 125 125 221 221 221 F F F
134912024-04-17T17:19:07.250Z 15 300-319 125 125 125 221 221 221 F F F
134922024-04-17T17:19:07.250Z 16 320-339 124 124 124 220 220 220 F F F
134932024-04-17T17:19:07.250Z 17 340-359 124 124 124 220 220 220 F F F
134942024-04-17T17:19:07.250Z 18 360-379 121 121 121 216 216 216 F F F
134952024-04-17T17:19:07.250Z 19 380-399 122 122 122 218 218 218 F F F
134962024-04-17T17:19:07.250Z 20 400-419 124 124 124 220 220 220 F F F
134972024-04-17T17:19:07.250Z 21 420-439 123 123 123 219 219 219 F F F
134982024-04-17T17:19:07.250Z 22 440-459 116 116 116 208 208 208 F F F
134992024-04-17T17:19:07.250Z 23 460-479 122 122 122 218 218 218 F F F
135002024-04-17T17:19:07.250Z 24 480-499 124 124 124 220 220 220 F F F
135012024-04-17T17:19:07.250Z 25 500-519 124 124 124 220 220 220 F F F
135022024-04-17T17:19:07.251Z 26 520-539 125 125 125 221 221 221 F F F
135032024-04-17T17:19:07.251Z 27 540-559 125 125 125 221 221 221 F F F
135042024-04-17T17:19:07.251Z 28 560-579 125 125 125 221 221 221 F F F
135052024-04-17T17:19:07.251Z 29 580-599 114 114 114 207 207 207 F F F
135062024-04-17T17:19:07.251ZMax gen: 125, Max flush: 221
135072024-04-17T17:19:07.254Z
135082024-04-17T17:19:07.254ZBegin loop 63
135092024-04-17T17:19:07.254ZDownstairs to restart: 0
135102024-04-17T17:19:07.269Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.878186752Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675}
135112024-04-17T17:19:07.269Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.878449478Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675}
135122024-04-17T17:19:07.269Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.878471229Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675}
135132024-04-17T17:19:07.277Z{"msg":"Crucible 9289267d-975d-484e-ab49-bfe8b5dd0005 has session id: 0bd554b0-23ed-4e81-8ed5-25f4900d6648","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.878504491Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135142024-04-17T17:19:07.277Z{"msg":"Upstairs opts: Upstairs UUID: 9289267d-975d-484e-ab49-bfe8b5dd0005, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"Crucible runtime is spawned
135152024-04-17T17:19:07.278Zcrucible","level":30,"time":"2024-04-17T17:19:05.878604387Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135162024-04-17T17:19:07.278Z{"msg":"Crucible stats registered with UUID: 9289267d-975d-484e-ab49-bfe8b5dd0005","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.87864271Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135172024-04-17T17:19:07.278Z{"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.87865724Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675}
135182024-04-17T17:19:07.278Z{"msg":"9289267d-975d-484e-ab49-bfe8b5dd0005 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.878753466Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135192024-04-17T17:19:07.278Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.878779518Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"client":"0","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135202024-04-17T17:19:07.278Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.878794609Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"client":"0","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135212024-04-17T17:19:07.278Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.87881518Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"client":"1","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135222024-04-17T17:19:07.278Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.878829061Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"client":"1","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135232024-04-17T17:19:07.278Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.878843372Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"client":"2","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135242024-04-17T17:19:07.278Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.878860073Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"client":"2","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135252024-04-17T17:19:07.278Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.878890615Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135262024-04-17T17:19:07.278Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.878933667Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135272024-04-17T17:19:07.278Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.87898393Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135282024-04-17T17:19:07.278Z{"msg":"ds_connection connected from Ok(127.0.0.1:62511)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.879042864Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135292024-04-17T17:19:07.278Z{"msg":"ds_connection connected from Ok(127.0.0.1:56651)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.879152221Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135302024-04-17T17:19:07.278Z{"msg":"ds_connection connected from Ok(127.0.0.1:65412)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.879197863Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135312024-04-17T17:19:07.278Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.879875474Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"client":"1","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135322024-04-17T17:19:07.278Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.879928738Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"client":"1","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135332024-04-17T17:19:07.278Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.879956929Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"client":"2","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135342024-04-17T17:19:07.278Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.880002712Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"client":"2","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135352024-04-17T17:19:07.278Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.880232826Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"client":"0","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135362024-04-17T17:19:07.278Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.880281729Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"client":"0","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135372024-04-17T17:19:07.278Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.880583827Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"client":"2","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135382024-04-17T17:19:07.279Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.880734346Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"client":"1","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135392024-04-17T17:19:07.279Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.880975351Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"client":"0","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135402024-04-17T17:19:07.279Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.881145231Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"client":"2","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135412024-04-17T17:19:07.279Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.881180104Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"client":"2","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135422024-04-17T17:19:07.279Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.881253348Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135432024-04-17T17:19:07.279Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.881299931Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"client":"1","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135442024-04-17T17:19:07.279Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.881347634Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"client":"1","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135452024-04-17T17:19:07.279Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.881390636Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135462024-04-17T17:19:07.279Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.881434779Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"client":"0","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135472024-04-17T17:19:07.279Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.881466191Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"client":"0","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135482024-04-17T17:19:07.279Z{"msg":"[0]R flush_numbers[0..12]: [207, 219, 220, 220, 216, 221, 220, 220, 219, 216, 220, 211]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.881505663Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"":"downstairs","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135492024-04-17T17:19:07.279Z{"msg":"[0]R generation[0..12]: [114, 123, 124, 124, 121, 125, 124, 124, 123, 121, 124, 118]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.881529555Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"":"downstairs","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135502024-04-17T17:19:07.279Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.881567597Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"":"downstairs","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135512024-04-17T17:19:07.279ZWait for a query_work_queue command to finish before sending IO
135522024-04-17T17:19:07.279Z{"msg":"[1]R flush_numbers[0..12]: [207, 219, 220, 220, 216, 221, 220, 220, 219, 216, 220, 211]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.88160937Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"":"downstairs","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135532024-04-17T17:19:07.279Z{"msg":"[1]R generation[0..12]: [114, 123, 124, 124, 121, 125, 124, 124, 123, 121, 124, 118]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.881649172Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"":"downstairs","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135542024-04-17T17:19:07.279Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.881680394Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"":"downstairs","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135552024-04-17T17:19:07.279Z{"msg":"[2]R flush_numbers[0..12]: [207, 219, 220, 220, 216, 221, 220, 220, 219, 216, 220, 211]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.881715986Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"":"downstairs","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135562024-04-17T17:19:07.279Z{"msg":"[2]R generation[0..12]: [114, 123, 124, 124, 121, 125, 124, 124, 123, 121, 124, 118]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.881748548Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"":"downstairs","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135572024-04-17T17:19:07.279Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.88178065Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"":"downstairs","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135582024-04-17T17:19:07.279Z{"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.881812252Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"":"downstairs","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135592024-04-17T17:19:07.279Z{"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.881841853Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"":"downstairs","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135602024-04-17T17:19:07.280Z{"msg":"Next flush: 222","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.881873426Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"":"downstairs","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135612024-04-17T17:19:07.280Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.881904317Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"":"downstairs","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135622024-04-17T17:19:07.280Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
135632024-04-17T17:19:07.280ZNo downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.88193931Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135642024-04-17T17:19:07.280Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.881970211Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"client":"0","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135652024-04-17T17:19:07.280Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.882013814Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"client":"1","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135662024-04-17T17:19:07.280Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.882044396Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"client":"2","":"downstairs","session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135672024-04-17T17:19:07.280Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.882073498Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135682024-04-17T17:19:07.280Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.88210478Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135692024-04-17T17:19:07.280Z{"msg":"9289267d-975d-484e-ab49-bfe8b5dd0005 is now active with session: 0bd554b0-23ed-4e81-8ed5-25f4900d6648","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.882135671Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135702024-04-17T17:19:07.280Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.882165903Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675,"session_id":"0bd554b0-23ed-4e81-8ed5-25f4900d6648"}
135712024-04-17T17:19:07.280Z{"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:05.882193985Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675}
135722024-04-17T17:19:07.280ZLoading write count information from file "/tmp/test_repair_verify.data"
135732024-04-17T17:19:07.280ZRun Repair workload
135742024-04-17T17:19:07.284Z01/30 Write block 266 len 24576 data: 7 7 7 10 9 11
135752024-04-17T17:19:07.284Z02/30 Write block 496 len 32768 data: 8 9 12 11 12 12 11 11
135762024-04-17T17:19:07.284Z03/30 Write block 45 len 20480 data: 9 9 7 6 8
135772024-04-17T17:19:07.284Z04/30 Write block 49 len 32768 data: 9 8 9 9 10 14 12 12
135782024-04-17T17:19:07.284Z05/30 Flush
135792024-04-17T17:19:07.284Z06/30 Write block 235 len 20480 data: 13 11 9 7 9
135802024-04-17T17:19:07.284Z07/30 Write block 429 len 36864 data: 9 6 7 9 12 11 11 11 11
135812024-04-17T17:19:07.288Z08/30 Write block 480 len 28672 data: 17 16 17 20 17 15 17
135822024-04-17T17:19:07.288Z09/30 Write block 182 len 4096 data: 7
135832024-04-17T17:19:07.288Z10/30 Read block 106 len 28672
135842024-04-17T17:19:07.288Z11/30 Flush
135852024-04-17T17:19:07.291Z12/30 Flush
135862024-04-17T17:19:07.291Z13/30 Write block 380 len 40960 data: 8 7 9 10 9 11 9 11 9 10
135872024-04-17T17:19:07.295Z14/30 Write block 303 len 36864 data: 10 11 11 10 8 9 8 9 8
135882024-04-17T17:19:07.295Z15/30 Read block 163 len 16384
135892024-04-17T17:19:07.295Z16/30 Read block 271 len 12288
135902024-04-17T17:19:07.295Z17/30 Write block 42 len 12288 data: 11 10 7
135912024-04-17T17:19:07.295Z18/30 Read block 330 len 20480
135922024-04-17T17:19:07.298Z19/30 Write block 289 len 40960 data: 10 11 10 9 11 9 6 4 5 6
135932024-04-17T17:19:07.299Z20/30 Write block 494 len 12288 data: 9 10 9
135942024-04-17T17:19:07.299Z21/30 Read block 543 len 16384
135952024-04-17T17:19:07.299Z22/30 Flush
135962024-04-17T17:19:07.302Z23/30 Write block 108 len 36864 data: 10 9 10 11 9 8 6 7 7
135972024-04-17T17:19:07.306Z24/30 Write block 302 len 32768 data: 8 11 12 12 11 9 10 9
135982024-04-17T17:19:07.306Z25/30 Read block 550 len 12288
135992024-04-17T17:19:07.306Z26/30 Read block 252 len 16384
136002024-04-17T17:19:07.306Z27/30 Write block 308 len 16384 data: 11 10 10 9
136012024-04-17T17:19:07.306Z28/30 Read block 116 len 40960
136022024-04-17T17:19:07.313Z29/30 Write block 296 len 40960 data: 5 6 7 7 7 9 9 12 13 13
136032024-04-17T17:19:07.313Z30/30 Read block 151 len 8192
136042024-04-17T17:19:07.313Z----------------------------------------------------------------
136052024-04-17T17:19:07.313Z Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:25
136062024-04-17T17:19:07.313ZStates: Active Active Active
136072024-04-17T17:19:07.313ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
136082024-04-17T17:19:07.313Z 6 Acked 1005 Write 5 Done Done Done false
136092024-04-17T17:19:07.313Z 7 Acked 1006 Write 9 Done Done Done false
136102024-04-17T17:19:07.313Z 8 Acked 1007 Write 7 Sent Done Done false
136112024-04-17T17:19:07.313Z 9 Acked 1008 Write 1 Sent Done Done false
136122024-04-17T17:19:07.314Z 10 Acked 1009 Read 7 Sent Done Done false
136132024-04-17T17:19:07.314Z 11 Acked 1010 Flush 0 Sent Done Done false
136142024-04-17T17:19:07.314Z 12 Acked 1011 Flush 0 Sent Done Done false
136152024-04-17T17:19:07.314Z 13 Acked 1012 Write 10 Sent Done Done false
136162024-04-17T17:19:07.314Z 14 Acked 1013 Write 9 Sent Done Done false
136172024-04-17T17:19:07.314Z 15 Acked 1014 Read 4 Sent Done Done false
136182024-04-17T17:19:07.314Z 16 Acked 1015 Read 3 Sent Done Done false
136192024-04-17T17:19:07.314Z 17 Acked 1016 Write 3 Sent Done Done false
136202024-04-17T17:19:07.314Z 18 Acked 1017 Read 5 Sent Done Done false
136212024-04-17T17:19:07.314Z 19 Acked 1018 Write 10 Sent Done Done false
136222024-04-17T17:19:07.314Z 20 Acked 1019 Write 3 Sent Done Done false
136232024-04-17T17:19:07.314Z 21 Acked 1020 Read 4 Sent Done Done false
136242024-04-17T17:19:07.314Z 22 Acked 1021 Flush 0 Sent Done Done false
136252024-04-17T17:19:07.314Z 23 Acked 1022 Write 9 Sent Done Done false
136262024-04-17T17:19:07.314Z 24 Acked 1023 Write 8 Sent Done Done false
136272024-04-17T17:19:07.314Z 25 Acked 1024 Read 3 Sent Done Done false
136282024-04-17T17:19:07.314Z 26 Acked 1025 Read 4 Sent Done Done false
136292024-04-17T17:19:07.314Z 27 Acked 1026 Write 4 Sent Done Done false
136302024-04-17T17:19:07.314Z 28 Acked 1027 Read 10 Sent Done Done false
136312024-04-17T17:19:07.314Z 29 Acked 1028 Write 10 Sent Done Done false
136322024-04-17T17:19:07.314Z 30 Acked 1029 Read 2 Sent Done Sent false
136332024-04-17T17:19:07.314Z STATES DS:0 DS:1 DS:2 TOTAL
136342024-04-17T17:19:07.314Z New 0 0 0 0
136352024-04-17T17:19:07.315Z Sent 23 0 1 24
136362024-04-17T17:19:07.315Z Done 2 25 24 51
136372024-04-17T17:19:07.315Z Skipped 0 0 0 0
136382024-04-17T17:19:07.315Z Error 0 0 0 0
136392024-04-17T17:19:07.315ZLast Flush: 1004 1021 1021
136402024-04-17T17:19:07.315ZDownstairs last five completed: 1004 1003 1002 1001 1000
136412024-04-17T17:19:07.315ZUpstairs last five completed: 30 29 28 27 26
136422024-04-17T17:19:07.315Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:19:05.918095181Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1675}
136432024-04-17T17:19:07.315ZWrote out file "/tmp/test_repair_verify.data"
136442024-04-17T17:19:07.319Z
136452024-04-17T17:19:07.319ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
136462024-04-17T17:19:07.342ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
136472024-04-17T17:19:07.342Z 0 000-019 114 114 114 207 207 207 F F F
136482024-04-17T17:19:07.342Z 1 020-039 123 123 123 219 219 219 F F F
136492024-04-17T17:19:07.342Z 2 040-059 126 126 126 222 225 225 F F F <---
136502024-04-17T17:19:07.342Z 3 060-079 124 124 124 220 220 220 F F F
136512024-04-17T17:19:07.342Z 4 080-099 121 121 121 216 216 216 F F F
136522024-04-17T17:19:07.342Z 5 100-119 125 125 125 221 221 221 F T T <---
136532024-04-17T17:19:07.343Z 6 120-139 124 124 124 220 220 220 F F F
136542024-04-17T17:19:07.343Z 7 140-159 124 124 124 220 220 220 F F F
136552024-04-17T17:19:07.343Z 8 160-179 123 123 123 219 219 219 F F F
136562024-04-17T17:19:07.349Z 9 180-199 121 126 126 216 223 223 F F F <---
136572024-04-17T17:19:07.349Z 10 200-219 124 124 124 220 220 220 F F F
136582024-04-17T17:19:07.349Z 11 220-239 118 126 126 211 223 223 T F F <---
136592024-04-17T17:19:07.349Z 12 240-259 124 124 124 220 220 220 F F F
136602024-04-17T17:19:07.349Z 13 260-279 126 126 126 222 222 222 F F F
136612024-04-17T17:19:07.349Z 14 280-299 125 126 126 221 225 225 F T T <---
136622024-04-17T17:19:07.350Z 15 300-319 125 126 126 221 225 225 F T T <---
136632024-04-17T17:19:07.350Z 16 320-339 124 124 124 220 220 220 F F F
136642024-04-17T17:19:07.350Z 17 340-359 124 124 124 220 220 220 F F F
136652024-04-17T17:19:07.350Z 18 360-379 121 121 121 216 216 216 F F F
136662024-04-17T17:19:07.350Z 19 380-399 122 126 126 218 225 225 F F F <---
136672024-04-17T17:19:07.350Z 20 400-419 124 124 124 220 220 220 F F F
136682024-04-17T17:19:07.350Z 21 420-439 123 126 126 219 223 223 T F F <---
136692024-04-17T17:19:07.350Z 22 440-459 116 116 116 208 208 208 F F F
136702024-04-17T17:19:07.351Z 23 460-479 122 122 122 218 218 218 F F F
136712024-04-17T17:19:07.351Z 24 480-499 126 126 126 222 225 225 F F F <---
136722024-04-17T17:19:07.351Z 25 500-519 126 126 126 222 222 222 F F F
136732024-04-17T17:19:07.351Z 26 520-539 125 125 125 221 221 221 F F F
136742024-04-17T17:19:07.351Z 27 540-559 125 125 125 221 221 221 F F F
136752024-04-17T17:19:07.351Z 28 560-579 125 125 125 221 221 221 F F F
136762024-04-17T17:19:07.351Z 29 580-599 114 114 114 207 207 207 F F F
136772024-04-17T17:19:07.351ZMax gen: 126, Max flush: 225
136782024-04-17T17:19:07.351ZError: Difference in extent metadata found!
136792024-04-17T17:19:07.351ZA Difference in extent metadata is expected here
136802024-04-17T17:19:07.352ZOn loop 63
136812024-04-17T17:19:07.352Z
136822024-04-17T17:19:07.352ZVerifying data now
136832024-04-17T17:19:21.769ZNo change after verify
136842024-04-17T17:19:21.769ZLoop: 63 Downstairs dump after verify (and repair):
136852024-04-17T17:19:21.784ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
136862024-04-17T17:19:21.784Z 0 000-019 114 114 114 207 207 207 F F F
136872024-04-17T17:19:21.788Z 1 020-039 123 123 123 219 219 219 F F F
136882024-04-17T17:19:21.788Z 2 040-059 126 126 126 225 225 225 F F F
136892024-04-17T17:19:21.788Z 3 060-079 124 124 124 220 220 220 F F F
136902024-04-17T17:19:21.788Z 4 080-099 121 121 121 216 216 216 F F F
136912024-04-17T17:19:21.788Z 5 100-119 127 127 127 226 226 226 F F F
136922024-04-17T17:19:21.788Z 6 120-139 124 124 124 220 220 220 F F F
136932024-04-17T17:19:21.788Z 7 140-159 124 124 124 220 220 220 F F F
136942024-04-17T17:19:21.788Z 8 160-179 123 123 123 219 219 219 F F F
136952024-04-17T17:19:21.788Z 9 180-199 126 126 126 223 223 223 F F F
136962024-04-17T17:19:21.788Z 10 200-219 124 124 124 220 220 220 F F F
136972024-04-17T17:19:21.788Z 11 220-239 126 126 126 223 223 223 F F F
136982024-04-17T17:19:21.788Z 12 240-259 124 124 124 220 220 220 F F F
136992024-04-17T17:19:21.788Z 13 260-279 126 126 126 222 222 222 F F F
137002024-04-17T17:19:21.788Z 14 280-299 127 127 127 226 226 226 F F F
137012024-04-17T17:19:21.788Z 15 300-319 127 127 127 226 226 226 F F F
137022024-04-17T17:19:21.788Z 16 320-339 124 124 124 220 220 220 F F F
137032024-04-17T17:19:21.788Z 17 340-359 124 124 124 220 220 220 F F F
137042024-04-17T17:19:21.789Z 18 360-379 121 121 121 216 216 216 F F F
137052024-04-17T17:19:21.789Z 19 380-399 126 126 126 225 225 225 F F F
137062024-04-17T17:19:21.789Z 20 400-419 124 124 124 220 220 220 F F F
137072024-04-17T17:19:21.789Z 21 420-439 126 126 126 223 223 223 F F F
137082024-04-17T17:19:21.789Z 22 440-459 116 116 116 208 208 208 F F F
137092024-04-17T17:19:21.789Z 23 460-479 122 122 122 218 218 218 F F F
137102024-04-17T17:19:21.789Z 24 480-499 126 126 126 225 225 225 F F F
137112024-04-17T17:19:21.789Z 25 500-519 126 126 126 222 222 222 F F F
137122024-04-17T17:19:21.789Z 26 520-539 125 125 125 221 221 221 F F F
137132024-04-17T17:19:21.789Z 27 540-559 125 125 125 221 221 221 F F F
137142024-04-17T17:19:21.789Z 28 560-579 125 125 125 221 221 221 F F F
137152024-04-17T17:19:21.789Z 29 580-599 114 114 114 207 207 207 F F F
137162024-04-17T17:19:21.789ZMax gen: 127, Max flush: 226
137172024-04-17T17:19:21.789Z
137182024-04-17T17:19:21.789ZBegin loop 64
137192024-04-17T17:19:21.789ZDownstairs to restart: 0
137202024-04-17T17:19:21.806Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.415093855Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683}
137212024-04-17T17:19:21.806Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.415309318Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683}
137222024-04-17T17:19:21.813Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.41534116Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683}
137232024-04-17T17:19:21.813Z{"msg":"Crucible 80022ccb-1eb4-49bf-abf9-2f12a1de0bc5 has session id: 7d3037eb-ae38-46a0-bc05-f8f947c87b4f","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.415374362Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137242024-04-17T17:19:21.813Z{Crucible runtime is spawned
137252024-04-17T17:19:21.813Z"msg":"Upstairs opts: Upstairs UUID: 80022ccb-1eb4-49bf-abf9-2f12a1de0bc5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.415433165Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137262024-04-17T17:19:21.813Z{"msg":"Crucible stats registered with UUID: 80022ccb-1eb4-49bf-abf9-2f12a1de0bc5","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.415463737Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137272024-04-17T17:19:21.814Z{"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.415481908Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683}
137282024-04-17T17:19:21.814Z{"msg":"80022ccb-1eb4-49bf-abf9-2f12a1de0bc5 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.415659089Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137292024-04-17T17:19:21.814Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.415690851Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"client":"0","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137302024-04-17T17:19:21.814Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.415768456Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"client":"0","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137312024-04-17T17:19:21.814Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.415804888Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"client":"1","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137322024-04-17T17:19:21.814Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.4158477Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"client":"1","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137332024-04-17T17:19:21.814Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.415912414Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"client":"2","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137342024-04-17T17:19:21.814Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.415960307Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"client":"2","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137352024-04-17T17:19:21.814Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.415993019Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137362024-04-17T17:19:21.814Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.416071814Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137372024-04-17T17:19:21.814Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.416116797Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137382024-04-17T17:19:21.814Z{"msg":"ds_connection connected from Ok(127.0.0.1:38647)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.416155919Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137392024-04-17T17:19:21.814Z{"msg":"ds_connection connected from Ok(127.0.0.1:51412)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.416194942Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137402024-04-17T17:19:21.814Z{"msg":"ds_connection connected from Ok(127.0.0.1:60504)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.416241064Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137412024-04-17T17:19:21.814Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.416917845Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"client":"1","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137422024-04-17T17:19:21.814Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.416953768Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"client":"1","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137432024-04-17T17:19:21.814Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.416969848Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"client":"0","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137442024-04-17T17:19:21.814Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.416988439Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"client":"0","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137452024-04-17T17:19:21.814Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.417026632Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"client":"2","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137462024-04-17T17:19:21.814Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.417054354Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"client":"2","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137472024-04-17T17:19:21.814Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.417569535Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"client":"2","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137482024-04-17T17:19:21.814Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.417617508Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"client":"0","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137492024-04-17T17:19:21.815Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.417668901Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"client":"1","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137502024-04-17T17:19:21.815Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.418068375Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"client":"2","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137512024-04-17T17:19:21.815Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.418111818Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"client":"2","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137522024-04-17T17:19:21.815Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.41814814Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137532024-04-17T17:19:21.815Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.418180622Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"client":"1","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137542024-04-17T17:19:21.815Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.418208664Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"client":"1","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137552024-04-17T17:19:21.815Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.418247046Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137562024-04-17T17:19:21.815Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.418285348Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"client":"0","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137572024-04-17T17:19:21.815Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.41831956Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"client":"0","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137582024-04-17T17:19:21.815Z{"msg":"[0]R flush_numbers[0..12]: [207, 219, 225, 220, 216, 226, 220, 220, 219, 223, 220, 223]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.418354882Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"":"downstairs","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137592024-04-17T17:19:21.815Z{"msg":"[0]R generation[0..12]: [114, 123, 126, 124, 121, 127, 124, 124, 123, 126, 124, 126]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.418391555Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"":"downstairs","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137602024-04-17T17:19:21.815Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.418426897Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"":"downstairs","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137612024-04-17T17:19:21.815Z{"msg":"[1]R flush_numbers[0..12]: [207, 219, 225, 220, 216, 226, 220, 220, 219, 223, 220, 223]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.418462879Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"":"downstairs","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137622024-04-17T17:19:21.815Z{Wait for a query_work_queue command to finish before sending IO
137632024-04-17T17:19:21.815Z"msg":"[1]R generation[0..12]: [114, 123, 126, 124, 121, 127, 124, 124, 123, 126, 124, 126]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.418501701Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"":"downstairs","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137642024-04-17T17:19:21.815Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.418527903Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"":"downstairs","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137652024-04-17T17:19:21.815Z{"msg":"[2]R flush_numbers[0..12]: [207, 219, 225, 220, 216, 226, 220, 220, 219, 223, 220, 223]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.418563955Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"":"downstairs","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137662024-04-17T17:19:21.815Z{"msg":"[2]R generation[0..12]: [114, 123, 126, 124, 121, 127, 124, 124, 123, 126, 124, 126]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.418605548Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"":"downstairs","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137672024-04-17T17:19:21.815Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.41864394Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"":"downstairs","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137682024-04-17T17:19:21.815Z{"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.418681072Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"":"downstairs","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137692024-04-17T17:19:21.815Z{"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.418716854Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"":"downstairs","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137702024-04-17T17:19:21.815Z{"msg":"Next flush: 227","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.418754657Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"":"downstairs","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137712024-04-17T17:19:21.815Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.418790769Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"":"downstairs","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137722024-04-17T17:19:21.815Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.418827741Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137732024-04-17T17:19:21.816ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
137742024-04-17T17:19:21.816Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.418865313Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"client":"0","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137752024-04-17T17:19:21.816Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.418887415Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"client":"1","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137762024-04-17T17:19:21.816Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.418901055Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"client":"2","":"downstairs","session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137772024-04-17T17:19:21.816Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.418922597Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137782024-04-17T17:19:21.816Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.418936108Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137792024-04-17T17:19:21.816Z{"msg":"80022ccb-1eb4-49bf-abf9-2f12a1de0bc5 is now active with session: 7d3037eb-ae38-46a0-bc05-f8f947c87b4f","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.418948708Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137802024-04-17T17:19:21.816Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.418968059Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683,"session_id":"7d3037eb-ae38-46a0-bc05-f8f947c87b4f"}
137812024-04-17T17:19:21.816Z{"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:20.41898131Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683}
137822024-04-17T17:19:21.821ZLoading write count information from file "/tmp/test_repair_verify.data"
137832024-04-17T17:19:21.821ZRun Repair workload
137842024-04-17T17:19:21.821Z01/30 Write block 350 len 20480 data: 10 6 6 6 6
137852024-04-17T17:19:21.821Z02/30 Write block 545 len 32768 data: 10 10 10 13 15 11 12 13
137862024-04-17T17:19:21.821Z03/30 Write block 544 len 28672 data: 14 11 11 11 14 16 12
137872024-04-17T17:19:21.821Z04/30 Read block 165 len 24576
137882024-04-17T17:19:21.821Z05/30 Write block 549 len 28672 data: 17 13 13 14 11 11 12
137892024-04-17T17:19:21.821Z06/30 Write block 207 len 28672 data: 18 10 11 12 12 12 12
137902024-04-17T17:19:21.821Z07/30 Read block 354 len 8192
137912024-04-17T17:19:21.826Z08/30 Write block 67 len 36864 data: 12 11 9 9 9 9 6 8 9
137922024-04-17T17:19:21.826Z09/30 Read block 71 len 32768
137932024-04-17T17:19:21.826Z10/30 Write block 439 len 24576 data: 12 9 9 11 11 11
137942024-04-17T17:19:21.826Z11/30 Write block 281 len 40960 data: 6 7 7 8 9 6 6 7 11 12
137952024-04-17T17:19:21.826Z12/30 Read block 585 len 32768
137962024-04-17T17:19:21.826Z13/30 Read block 528 len 24576
137972024-04-17T17:19:21.830Z14/30 Write block 125 len 12288 data: 10 10 10
137982024-04-17T17:19:21.830Z15/30 Read block 483 len 12288
137992024-04-17T17:19:21.830Z16/30 Flush
138002024-04-17T17:19:21.833Z17/30 Write block 464 len 28672 data: 11 13 12 11 11 12 13
138012024-04-17T17:19:21.834Z18/30 Read block 35 len 8192
138022024-04-17T17:19:21.834Z19/30 Read block 419 len 32768
138032024-04-17T17:19:21.834Z20/30 Read block 32 len 36864
138042024-04-17T17:19:21.840Z21/30 Flush
138052024-04-17T17:19:21.840Z22/30 Write block 146 len 40960 data: 11 9 8 10 8 6 6 5 8 8
138062024-04-17T17:19:21.844Z23/30 Read block 236 len 12288
138072024-04-17T17:19:21.844Z24/30 Write block 86 len 20480 data: 9 13 11 13 14
138082024-04-17T17:19:21.844Z25/30 Read block 594 len 24576
138092024-04-17T17:19:21.844Z26/30 Read block 533 len 32768
138102024-04-17T17:19:21.844Z27/30 Read block 440 len 4096
138112024-04-17T17:19:21.844Z28/30 Write block 446 len 16384 data: 8 6 7 8
138122024-04-17T17:19:21.844Z29/30 Write block 435 len 20480 data: 12 12 12 11 13
138132024-04-17T17:19:21.844Z30/30 Write block 29 len 12288 data: 12 10 12
138142024-04-17T17:19:21.848Z----------------------------------------------------------------
138152024-04-17T17:19:21.848Z Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:30
138162024-04-17T17:19:21.848ZStates: Active Active Active
138172024-04-17T17:19:21.849ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
138182024-04-17T17:19:21.849Z 1 Acked 1000 Write 5 Done Done Done false
138192024-04-17T17:19:21.849Z 2 Acked 1001 Write 8 Done Done Done false
138202024-04-17T17:19:21.849Z 3 Acked 1002 Write 7 Sent Done Done false
138212024-04-17T17:19:21.849Z 4 Acked 1003 Read 6 Sent Done Done false
138222024-04-17T17:19:21.849Z 5 Acked 1004 Write 7 Sent Done Done false
138232024-04-17T17:19:21.849Z 6 Acked 1005 Write 7 Sent Done Done false
138242024-04-17T17:19:21.849Z 7 Acked 1006 Read 2 Sent Done Done false
138252024-04-17T17:19:21.849Z 8 Acked 1007 Write 9 Sent Done Done false
138262024-04-17T17:19:21.849Z 9 Acked 1008 Read 8 Sent Done Done false
138272024-04-17T17:19:21.849Z 10 Acked 1009 Write 6 Sent Done Done false
138282024-04-17T17:19:21.849Z 11 Acked 1010 Write 10 Sent Done Done false
138292024-04-17T17:19:21.849Z 12 Acked 1011 Read 8 Sent Done Done false
138302024-04-17T17:19:21.849Z 13 Acked 1012 Read 6 Sent Done Done false
138312024-04-17T17:19:21.849Z 14 Acked 1013 Write 3 Sent Done Done false
138322024-04-17T17:19:21.849Z 15 Acked 1014 Read 3 Sent Done Done false
138332024-04-17T17:19:21.849Z 16 Acked 1015 Flush 0 Sent Done Done false
138342024-04-17T17:19:21.849Z 17 Acked 1016 Write 7 Sent Done Done false
138352024-04-17T17:19:21.850Z 18 Acked 1017 Read 2 Sent Done Done false
138362024-04-17T17:19:21.850Z 19 Acked 1018 Read 8 Sent Done Done false
138372024-04-17T17:19:21.850Z 20 Acked 1019 Read 9 Sent Done Done false
138382024-04-17T17:19:21.850Z 21 Acked 1020 Flush 0 Sent Done Done false
138392024-04-17T17:19:21.850Z 22 Acked 1021 Write 10 Sent Done Done false
138402024-04-17T17:19:21.850Z 23 Acked 1022 Read 3 Sent Done Done false
138412024-04-17T17:19:21.850Z 24 Acked 1023 Write 5 Sent Done Done false
138422024-04-17T17:19:21.850Z 25 Acked 1024 Read 6 Sent Done Done false
138432024-04-17T17:19:21.850Z 26 Acked 1025 Read 8 Sent Done Done false
138442024-04-17T17:19:21.850Z 27 Acked 1026 Read 1 Sent Done Done false
138452024-04-17T17:19:21.850Z 28 Acked 1027 Write 4 Sent Done Done false
138462024-04-17T17:19:21.850Z 29 Acked 1028 Write 5 Sent Done Done false
138472024-04-17T17:19:21.850Z 30 Acked 1029 Write 3 Sent Sent Sent false
138482024-04-17T17:19:21.850Z STATES DS:0 DS:1 DS:2 TOTAL
138492024-04-17T17:19:21.850Z New 0 0 0 0
138502024-04-17T17:19:21.850Z Sent 28 1 1 30
138512024-04-17T17:19:21.850Z Done 2 29 29 60
138522024-04-17T17:19:21.850Z Skipped 0 0 0 0
138532024-04-17T17:19:21.850Z Error 0 0 0 0
138542024-04-17T17:19:21.851ZLast Flush: 0 1020 1020
138552024-04-17T17:19:21.851ZDownstairs last five completed:
138562024-04-17T17:19:21.851ZUpstairs last five completed: 30 29 28 27 26
138572024-04-17T17:19:21.851Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:19:20.453663722Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1683}
138582024-04-17T17:19:21.851ZWrote out file "/tmp/test_repair_verify.data"
138592024-04-17T17:19:21.855Z
138602024-04-17T17:19:21.855ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
138612024-04-17T17:19:21.870ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
138622024-04-17T17:19:21.870Z 0 000-019 114 114 114 207 207 207 F F F
138632024-04-17T17:19:21.870Z 1 020-039 123 123 123 219 219 219 F T T <---
138642024-04-17T17:19:21.870Z 2 040-059 126 126 126 225 225 225 F F F
138652024-04-17T17:19:21.874Z 3 060-079 124 128 128 220 227 227 F F F <---
138662024-04-17T17:19:21.874Z 4 080-099 121 121 121 216 216 216 F T T <---
138672024-04-17T17:19:21.874Z 5 100-119 127 127 127 226 226 226 F F F
138682024-04-17T17:19:21.874Z 6 120-139 124 128 128 220 227 227 F F F <---
138692024-04-17T17:19:21.874Z 7 140-159 124 124 124 220 220 220 F T T <---
138702024-04-17T17:19:21.874Z 8 160-179 123 123 123 219 219 219 F F F
138712024-04-17T17:19:21.874Z 9 180-199 126 126 126 223 223 223 F F F
138722024-04-17T17:19:21.874Z 10 200-219 124 128 128 220 227 227 F F F <---
138732024-04-17T17:19:21.874Z 11 220-239 126 126 126 223 223 223 F F F
138742024-04-17T17:19:21.874Z 12 240-259 124 124 124 220 220 220 F F F
138752024-04-17T17:19:21.874Z 13 260-279 126 126 126 222 222 222 F F F
138762024-04-17T17:19:21.874Z 14 280-299 127 128 128 226 227 227 F F F <---
138772024-04-17T17:19:21.874Z 15 300-319 127 127 127 226 226 226 F F F
138782024-04-17T17:19:21.875Z 16 320-339 124 124 124 220 220 220 F F F
138792024-04-17T17:19:21.875Z 17 340-359 124 128 128 220 227 227 T F F <---
138802024-04-17T17:19:21.875Z 18 360-379 121 121 121 216 216 216 F F F
138812024-04-17T17:19:21.875Z 19 380-399 126 126 126 225 225 225 F F F
138822024-04-17T17:19:21.875Z 20 400-419 124 124 124 220 220 220 F F F
138832024-04-17T17:19:21.875Z 21 420-439 126 128 128 223 227 227 F T T <---
138842024-04-17T17:19:21.875Z 22 440-459 116 128 128 208 227 227 F T T <---
138852024-04-17T17:19:21.875Z 23 460-479 122 128 128 218 228 228 F F F <---
138862024-04-17T17:19:21.875Z 24 480-499 126 126 126 225 225 225 F F F
138872024-04-17T17:19:21.875Z 25 500-519 126 126 126 222 222 222 F F F
138882024-04-17T17:19:21.875Z 26 520-539 125 125 125 221 221 221 F F F
138892024-04-17T17:19:21.875Z 27 540-559 125 128 128 221 227 227 T F F <---
138902024-04-17T17:19:21.875Z 28 560-579 125 125 125 221 221 221 F F F
138912024-04-17T17:19:21.875Z 29 580-599 114 114 114 207 207 207 F F F
138922024-04-17T17:19:21.875ZMax gen: 128, Max flush: 228
138932024-04-17T17:19:21.875ZError: Difference in extent metadata found!
138942024-04-17T17:19:21.875ZA Difference in extent metadata is expected here
138952024-04-17T17:19:21.875ZOn loop 64
138962024-04-17T17:19:21.875Z
138972024-04-17T17:19:21.879ZVerifying data now
138982024-04-17T17:19:32.438ZNo change after verify
138992024-04-17T17:19:32.438ZLoop: 64 Downstairs dump after verify (and repair):
139002024-04-17T17:19:32.453ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
139012024-04-17T17:19:32.453Z 0 000-019 114 114 114 207 207 207 F F F
139022024-04-17T17:19:32.453Z 1 020-039 129 129 129 229 229 229 F F F
139032024-04-17T17:19:32.453Z 2 040-059 126 126 126 225 225 225 F F F
139042024-04-17T17:19:32.453Z 3 060-079 128 128 128 227 227 227 F F F
139052024-04-17T17:19:32.453Z 4 080-099 129 129 129 229 229 229 F F F
139062024-04-17T17:19:32.453Z 5 100-119 127 127 127 226 226 226 F F F
139072024-04-17T17:19:32.457Z 6 120-139 128 128 128 227 227 227 F F F
139082024-04-17T17:19:32.457Z 7 140-159 129 129 129 229 229 229 F F F
139092024-04-17T17:19:32.457Z 8 160-179 123 123 123 219 219 219 F F F
139102024-04-17T17:19:32.457Z 9 180-199 126 126 126 223 223 223 F F F
139112024-04-17T17:19:32.457Z 10 200-219 128 128 128 227 227 227 F F F
139122024-04-17T17:19:32.457Z 11 220-239 126 126 126 223 223 223 F F F
139132024-04-17T17:19:32.457Z 12 240-259 124 124 124 220 220 220 F F F
139142024-04-17T17:19:32.458Z 13 260-279 126 126 126 222 222 222 F F F
139152024-04-17T17:19:32.458Z 14 280-299 128 128 128 227 227 227 F F F
139162024-04-17T17:19:32.458Z 15 300-319 127 127 127 226 226 226 F F F
139172024-04-17T17:19:32.458Z 16 320-339 124 124 124 220 220 220 F F F
139182024-04-17T17:19:32.458Z 17 340-359 128 128 128 227 227 227 F F F
139192024-04-17T17:19:32.458Z 18 360-379 121 121 121 216 216 216 F F F
139202024-04-17T17:19:32.458Z 19 380-399 126 126 126 225 225 225 F F F
139212024-04-17T17:19:32.458Z 20 400-419 124 124 124 220 220 220 F F F
139222024-04-17T17:19:32.458Z 21 420-439 129 129 129 229 229 229 F F F
139232024-04-17T17:19:32.458Z 22 440-459 129 129 129 229 229 229 F F F
139242024-04-17T17:19:32.458Z 23 460-479 128 128 128 228 228 228 F F F
139252024-04-17T17:19:32.458Z 24 480-499 126 126 126 225 225 225 F F F
139262024-04-17T17:19:32.458Z 25 500-519 126 126 126 222 222 222 F F F
139272024-04-17T17:19:32.458Z 26 520-539 125 125 125 221 221 221 F F F
139282024-04-17T17:19:32.458Z 27 540-559 128 128 128 227 227 227 F F F
139292024-04-17T17:19:32.458Z 28 560-579 125 125 125 221 221 221 F F F
139302024-04-17T17:19:32.458Z 29 580-599 114 114 114 207 207 207 F F F
139312024-04-17T17:19:32.458ZMax gen: 129, Max flush: 229
139322024-04-17T17:19:32.458Z
139332024-04-17T17:19:32.458ZBegin loop 65
139342024-04-17T17:19:32.458ZDownstairs to restart: 2
139352024-04-17T17:19:32.476Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.085381629Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691}
139362024-04-17T17:19:32.483Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.085635905Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691}
139372024-04-17T17:19:32.483Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.085672377Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691}
139382024-04-17T17:19:32.483Z{"msg":"Crucible 0476076e-99b9-4e4a-8f53-81609bd3ef77 has session id: 3708b3ba-df4f-4f9d-be5c-fba1ee90c913","v":0,"name":"crucible","level":30Crucible runtime is spawned
139392024-04-17T17:19:32.483Z,"time":"2024-04-17T17:19:31.085697078Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139402024-04-17T17:19:32.483Z{"msg":"Upstairs opts: Upstairs UUID: 0476076e-99b9-4e4a-8f53-81609bd3ef77, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.085751532Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139412024-04-17T17:19:32.483Z{"msg":"Crucible stats registered with UUID: 0476076e-99b9-4e4a-8f53-81609bd3ef77","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.085784594Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139422024-04-17T17:19:32.483Z{"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.085798775Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691}
139432024-04-17T17:19:32.483Z{"msg":"0476076e-99b9-4e4a-8f53-81609bd3ef77 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.085858908Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139442024-04-17T17:19:32.483Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.085883589Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"client":"0","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139452024-04-17T17:19:32.483Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.085898501Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"client":"0","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139462024-04-17T17:19:32.483Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.085913811Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"client":"1","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139472024-04-17T17:19:32.483Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.085934543Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"client":"1","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139482024-04-17T17:19:32.483Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.085960344Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"client":"2","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139492024-04-17T17:19:32.483Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.085986186Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"client":"2","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139502024-04-17T17:19:32.483Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.086032058Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139512024-04-17T17:19:32.483Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.086074901Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139522024-04-17T17:19:32.483Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.086101873Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139532024-04-17T17:19:32.483Z{"msg":"ds_connection connected from Ok(127.0.0.1:40768)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.086270513Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139542024-04-17T17:19:32.483Z{"msg":"ds_connection connected from Ok(127.0.0.1:47897)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.086329637Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139552024-04-17T17:19:32.483Z{"msg":"ds_connection connected from Ok(127.0.0.1:50870)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.086366139Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139562024-04-17T17:19:32.484Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.087075512Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"client":"1","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139572024-04-17T17:19:32.484Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.087106484Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"client":"1","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139582024-04-17T17:19:32.484Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.087130775Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"client":"0","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139592024-04-17T17:19:32.484Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.087171158Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"client":"0","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139602024-04-17T17:19:32.484Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.087435464Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"client":"2","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139612024-04-17T17:19:32.484Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.087466196Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"client":"2","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139622024-04-17T17:19:32.484Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.087756783Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"client":"0","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139632024-04-17T17:19:32.484Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.087806966Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"client":"1","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139642024-04-17T17:19:32.484Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.088117905Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"client":"2","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139652024-04-17T17:19:32.484Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.088298056Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"client":"1","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139662024-04-17T17:19:32.484Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.088338478Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"client":"1","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139672024-04-17T17:19:32.484Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.08836446Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139682024-04-17T17:19:32.484Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.088395562Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"client":"0","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139692024-04-17T17:19:32.484Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.088420533Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"client":"0","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139702024-04-17T17:19:32.484Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.088448135Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139712024-04-17T17:19:32.484Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.088587384Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"client":"2","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139722024-04-17T17:19:32.484Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.088629356Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"client":"2","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139732024-04-17T17:19:32.484Z{"msg":"[0]R flush_numbers[0..12]: [207, 229, 225, 227, 229, 226, 227, 229, 219, 223, 227, 223]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.088645817Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"":"downstairs","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139742024-04-17T17:19:32.484Z{"msg":"[0]R generation[0..12]: [114, 129, 126, 128, 129, 127, 128, 129, 123, 126, 128, 126]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.088730002Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"":"downstairs","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139752024-04-17T17:19:32.484Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.088769444Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"":"downstairs","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139762024-04-17T17:19:32.484Z{"msg":"Wait for a query_work_queue command to finish before sending IO
139772024-04-17T17:19:32.484Z[1]R flush_numbers[0..12]: [207, 229, 225, 227, 229, 226, 227, 229, 219, 223, 227, 223]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.088798696Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"":"downstairs","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139782024-04-17T17:19:32.484Z{"msg":"[1]R generation[0..12]: [114, 129, 126, 128, 129, 127, 128, 129, 123, 126, 128, 126]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.088835739Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"":"downstairs","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139792024-04-17T17:19:32.485Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.088877011Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"":"downstairs","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139802024-04-17T17:19:32.485Z{"msg":"[2]R flush_numbers[0..12]: [207, 229, 225, 227, 229, 226, 227, 229, 219, 223, 227, 223]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.088918224Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"":"downstairs","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139812024-04-17T17:19:32.485Z{"msg":"[2]R generation[0..12]: [114, 129, 126, 128, 129, 127, 128, 129, 123, 126, 128, 126]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.088957696Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"":"downstairs","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139822024-04-17T17:19:32.485Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.088996128Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"":"downstairs","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139832024-04-17T17:19:32.485Z{"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.0890354Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"":"downstairs","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139842024-04-17T17:19:32.485Z{"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.089052432Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"":"downstairs","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139852024-04-17T17:19:32.485Z{"msg":"Next flush: 230","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.089073683Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"":"downstairs","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139862024-04-17T17:19:32.485Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.089087464Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"":"downstairs","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139872024-04-17T17:19:32.485Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.089112845Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139882024-04-17T17:19:32.485Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.089126946Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"client":"0","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139892024-04-17T17:19:32.485Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.089140607Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"client":"1","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139902024-04-17T17:19:32.485Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.089160928Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"client":"2","":"downstairs","session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139912024-04-17T17:19:32.485Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.089174309Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139922024-04-17T17:19:32.489Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.08918626Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"session_id"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
139932024-04-17T17:19:32.489Z:"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139942024-04-17T17:19:32.489Z{"msg":"0476076e-99b9-4e4a-8f53-81609bd3ef77 is now active with session: 3708b3ba-df4f-4f9d-be5c-fba1ee90c913","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.089213221Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139952024-04-17T17:19:32.489Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.089225142Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691,"session_id":"3708b3ba-df4f-4f9d-be5c-fba1ee90c913"}
139962024-04-17T17:19:32.489Z{"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:31.089237783Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691}
139972024-04-17T17:19:32.489ZLoading write count information from file "/tmp/test_repair_verify.data"
139982024-04-17T17:19:32.489ZRun Repair workload
139992024-04-17T17:19:32.489Z01/30 Write block 501 len 12288 data: 13 12 12
140002024-04-17T17:19:32.489Z02/30 Read block 526 len 4096
140012024-04-17T17:19:32.489Z03/30 Read block 91 len 20480
140022024-04-17T17:19:32.489Z04/30 Flush
140032024-04-17T17:19:32.489Z05/30 Write block 376 len 4096 data: 7
140042024-04-17T17:19:32.489Z06/30 Read block 152 len 4096
140052024-04-17T17:19:32.490Z07/30 Write block 491 len 28672 data: 9 10 8 10 11 10 10
140062024-04-17T17:19:32.490Z08/30 Read block 229 len 16384
140072024-04-17T17:19:32.490Z09/30 Write block 479 len 8192 data: 17 18
140082024-04-17T17:19:32.493Z10/30 Read block 491 len 36864
140092024-04-17T17:19:32.493Z11/30 Read block 544 len 36864
140102024-04-17T17:19:32.493Z12/30 Read block 72 len 16384
140112024-04-17T17:19:32.493Z13/30 Read block 257 len 20480
140122024-04-17T17:19:32.493Z14/30 Write block 525 len 12288 data: 8 10 10
140132024-04-17T17:19:32.493Z15/30 Write block 156 len 20480 data: 10 11 9 11 11
140142024-04-17T17:19:32.497Z16/30 Read block 168 len 28672
140152024-04-17T17:19:32.497Z17/30 Read block 159 len 28672
140162024-04-17T17:19:32.497Z18/30 Read block 576 len 32768
140172024-04-17T17:19:32.500Z19/30 Write block 172 len 40960 data: 7 8 6 9 10 9 12 11 7 7
140182024-04-17T17:19:32.500Z20/30 Write block 580 len 28672 data: 10 11 11 12 13 12 13
140192024-04-17T17:19:32.500Z21/30 Read block 540 len 4096
140202024-04-17T17:19:32.500Z22/30 Read block 85 len 8192
140212024-04-17T17:19:32.504Z23/30 Write block 184 len 12288 data: 6 5 6
140222024-04-17T17:19:32.504Z24/30 Flush
140232024-04-17T17:19:32.504Z25/30 Write block 404 len 8192 data: 10 11
140242024-04-17T17:19:32.507Z26/30 Read block 595 len 12288
140252024-04-17T17:19:32.507Z27/30 Write block 320 len 12288 data: 10 9 5
140262024-04-17T17:19:32.507Z28/30 Read block 235 len 32768
140272024-04-17T17:19:32.508Z29/30 Read block 551 len 32768
140282024-04-17T17:19:32.511Z30/30 Write block 419 len 24576 data: 12 11 7 9 8 9
140292024-04-17T17:19:32.511Z----------------------------------------------------------------
140302024-04-17T17:19:32.512Z Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:30
140312024-04-17T17:19:32.512ZStates: Active Active Active
140322024-04-17T17:19:32.512ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
140332024-04-17T17:19:32.512Z 1 Acked 1000 Write 3 Done Done Sent false
140342024-04-17T17:19:32.512Z 2 Acked 1001 Read 1 Done Done Sent false
140352024-04-17T17:19:32.512Z 3 Acked 1002 Read 5 Done Done Sent false
140362024-04-17T17:19:32.512Z 4 Acked 1003 Flush 0 Done Done Sent false
140372024-04-17T17:19:32.512Z 5 Acked 1004 Write 1 Done Done Sent false
140382024-04-17T17:19:32.512Z 6 Acked 1005 Read 1 Done Done Sent false
140392024-04-17T17:19:32.512Z 7 Acked 1006 Write 7 Done Done Sent false
140402024-04-17T17:19:32.512Z 8 Acked 1007 Read 4 Done Done Sent false
140412024-04-17T17:19:32.512Z 9 Acked 1008 Write 2 Done Done Sent false
140422024-04-17T17:19:32.512Z 10 Acked 1009 Read 9 Done Done Sent false
140432024-04-17T17:19:32.512Z 11 Acked 1010 Read 9 Done Done Sent false
140442024-04-17T17:19:32.512Z 12 Acked 1011 Read 4 Done Done Sent false
140452024-04-17T17:19:32.512Z 13 Acked 1012 Read 5 Done Done Sent false
140462024-04-17T17:19:32.512Z 14 Acked 1013 Write 3 Done Done Sent false
140472024-04-17T17:19:32.512Z 15 Acked 1014 Write 5 Done Done Sent false
140482024-04-17T17:19:32.512Z 16 Acked 1015 Read 7 Done Done Sent false
140492024-04-17T17:19:32.512Z 17 Acked 1016 Read 7 Done Done Sent false
140502024-04-17T17:19:32.512Z 18 Acked 1017 Read 8 Done Done Sent false
140512024-04-17T17:19:32.512Z 19 Acked 1018 Write 10 Done Done Sent false
140522024-04-17T17:19:32.512Z 20 Acked 1019 Write 7 Done Done Sent false
140532024-04-17T17:19:32.513Z 21 Acked 1020 Read 1 Done Done Sent false
140542024-04-17T17:19:32.513Z 22 Acked 1021 Read 2 Done Done Sent false
140552024-04-17T17:19:32.513Z 23 Acked 1022 Write 3 Done Done Sent false
140562024-04-17T17:19:32.513Z 24 Acked 1023 Flush 0 Done Done Sent false
140572024-04-17T17:19:32.513Z 25 Acked 1024 Write 2 Done Done Sent false
140582024-04-17T17:19:32.513Z 26 Acked 1025 Read 3 Done Done Sent false
140592024-04-17T17:19:32.513Z 27 Acked 1026 Write 3 Done Done Sent false
140602024-04-17T17:19:32.513Z 28 Acked 1027 Read 8 Done Done Sent false
140612024-04-17T17:19:32.513Z 29 Acked 1028 Read 8 Done Done Sent false
140622024-04-17T17:19:32.513Z 30 Acked 1029 Write 6 Sent Sent Sent false
140632024-04-17T17:19:32.513Z STATES DS:0 DS:1 DS:2 TOTAL
140642024-04-17T17:19:32.513Z New 0 0 0 0
140652024-04-17T17:19:32.513Z Sent 1 1 30 32
140662024-04-17T17:19:32.513Z Done 29 29 0 58
140672024-04-17T17:19:32.513Z Skipped 0 0 0 0
140682024-04-17T17:19:32.513Z Error 0 0 0 0
140692024-04-17T17:19:32.513ZLast Flush: 1023 1023 0
140702024-04-17T17:19:32.513ZDownstairs last five completed:
140712024-04-17T17:19:32.513ZUpstairs last five completed: 30 29 28 27 26
140722024-04-17T17:19:32.513Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:19:31.118207648Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1691}
140732024-04-17T17:19:32.513ZWrote out file "/tmp/test_repair_verify.data"
140742024-04-17T17:19:32.517Z
140752024-04-17T17:19:32.517ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
140762024-04-17T17:19:32.531ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
140772024-04-17T17:19:32.531Z 0 000-019 114 114 114 207 207 207 F F F
140782024-04-17T17:19:32.532Z 1 020-039 129 129 129 229 229 229 F F F
140792024-04-17T17:19:32.532Z 2 040-059 126 126 126 225 225 225 F F F
140802024-04-17T17:19:32.532Z 3 060-079 128 128 128 227 227 227 F F F
140812024-04-17T17:19:32.532Z 4 080-099 129 129 129 229 229 229 F F F
140822024-04-17T17:19:32.532Z 5 100-119 127 127 127 226 226 226 F F F
140832024-04-17T17:19:32.532Z 6 120-139 128 128 128 227 227 227 F F F
140842024-04-17T17:19:32.532Z 7 140-159 130 130 129 231 231 229 F F F <---
140852024-04-17T17:19:32.535Z 8 160-179 130 130 123 231 231 219 F F F <---
140862024-04-17T17:19:32.535Z 9 180-199 130 130 126 231 231 223 F F F <---
140872024-04-17T17:19:32.535Z 10 200-219 128 128 128 227 227 227 F F F
140882024-04-17T17:19:32.535Z 11 220-239 126 126 126 223 223 223 F F F
140892024-04-17T17:19:32.535Z 12 240-259 124 124 124 220 220 220 F F F
140902024-04-17T17:19:32.536Z 13 260-279 126 126 126 222 222 222 F F F
140912024-04-17T17:19:32.536Z 14 280-299 128 128 128 227 227 227 F F F
140922024-04-17T17:19:32.536Z 15 300-319 127 127 127 226 226 226 F F F
140932024-04-17T17:19:32.536Z 16 320-339 124 124 124 220 220 220 T T F <---
140942024-04-17T17:19:32.536Z 17 340-359 128 128 128 227 227 227 F F F
140952024-04-17T17:19:32.536Z 18 360-379 130 130 121 231 231 216 F F F <---
140962024-04-17T17:19:32.536Z 19 380-399 126 126 126 225 225 225 F F F
140972024-04-17T17:19:32.536Z 20 400-419 124 124 124 220 220 220 T T F <---
140982024-04-17T17:19:32.536Z 21 420-439 129 129 129 229 229 229 T T F <---
140992024-04-17T17:19:32.536Z 22 440-459 129 129 129 229 229 229 F F F
141002024-04-17T17:19:32.536Z 23 460-479 130 130 128 231 231 228 F F F <---
141012024-04-17T17:19:32.536Z 24 480-499 130 130 126 231 231 225 F F F <---
141022024-04-17T17:19:32.536Z 25 500-519 130 130 126 230 230 222 F F F <---
141032024-04-17T17:19:32.536Z 26 520-539 130 130 125 231 231 221 F F F <---
141042024-04-17T17:19:32.536Z 27 540-559 128 128 128 227 227 227 F F F
141052024-04-17T17:19:32.536Z 28 560-579 125 125 125 221 221 221 F F F
141062024-04-17T17:19:32.536Z 29 580-599 130 130 114 231 231 207 F F F <---
141072024-04-17T17:19:32.536ZMax gen: 130, Max flush: 231
141082024-04-17T17:19:32.536ZError: Difference in extent metadata found!
141092024-04-17T17:19:32.536ZA Difference in extent metadata is expected here
141102024-04-17T17:19:32.536ZOn loop 65
141112024-04-17T17:19:32.536Z
141122024-04-17T17:19:32.540ZVerifying data now
141132024-04-17T17:19:37.099ZNo change after verify
141142024-04-17T17:19:37.099ZLoop: 65 Downstairs dump after verify (and repair):
141152024-04-17T17:19:37.113ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
141162024-04-17T17:19:37.113Z 0 000-019 114 114 114 207 207 207 F F F
141172024-04-17T17:19:37.117Z 1 020-039 129 129 129 229 229 229 F F F
141182024-04-17T17:19:37.117Z 2 040-059 126 126 126 225 225 225 F F F
141192024-04-17T17:19:37.117Z 3 060-079 128 128 128 227 227 227 F F F
141202024-04-17T17:19:37.117Z 4 080-099 129 129 129 229 229 229 F F F
141212024-04-17T17:19:37.117Z 5 100-119 127 127 127 226 226 226 F F F
141222024-04-17T17:19:37.117Z 6 120-139 128 128 128 227 227 227 F F F
141232024-04-17T17:19:37.117Z 7 140-159 130 130 130 231 231 231 F F F
141242024-04-17T17:19:37.117Z 8 160-179 130 130 130 231 231 231 F F F
141252024-04-17T17:19:37.117Z 9 180-199 130 130 130 231 231 231 F F F
141262024-04-17T17:19:37.117Z 10 200-219 128 128 128 227 227 227 F F F
141272024-04-17T17:19:37.117Z 11 220-239 126 126 126 223 223 223 F F F
141282024-04-17T17:19:37.117Z 12 240-259 124 124 124 220 220 220 F F F
141292024-04-17T17:19:37.118Z 13 260-279 126 126 126 222 222 222 F F F
141302024-04-17T17:19:37.118Z 14 280-299 128 128 128 227 227 227 F F F
141312024-04-17T17:19:37.118Z 15 300-319 127 127 127 226 226 226 F F F
141322024-04-17T17:19:37.118Z 16 320-339 131 131 131 232 232 232 F F F
141332024-04-17T17:19:37.118Z 17 340-359 128 128 128 227 227 227 F F F
141342024-04-17T17:19:37.118Z 18 360-379 130 130 130 231 231 231 F F F
141352024-04-17T17:19:37.118Z 19 380-399 126 126 126 225 225 225 F F F
141362024-04-17T17:19:37.118Z 20 400-419 131 131 131 232 232 232 F F F
141372024-04-17T17:19:37.118Z 21 420-439 131 131 131 232 232 232 F F F
141382024-04-17T17:19:37.118Z 22 440-459 129 129 129 229 229 229 F F F
141392024-04-17T17:19:37.118Z 23 460-479 130 130 130 231 231 231 F F F
141402024-04-17T17:19:37.118Z 24 480-499 130 130 130 231 231 231 F F F
141412024-04-17T17:19:37.118Z 25 500-519 130 130 130 230 230 230 F F F
141422024-04-17T17:19:37.118Z 26 520-539 130 130 130 231 231 231 F F F
141432024-04-17T17:19:37.118Z 27 540-559 128 128 128 227 227 227 F F F
141442024-04-17T17:19:37.118Z 28 560-579 125 125 125 221 221 221 F F F
141452024-04-17T17:19:37.118Z 29 580-599 130 130 130 231 231 231 F F F
141462024-04-17T17:19:37.118ZMax gen: 131, Max flush: 232
141472024-04-17T17:19:37.122Z
141482024-04-17T17:19:37.122ZBegin loop 66
141492024-04-17T17:19:37.122ZDownstairs to restart: 1
141502024-04-17T17:19:37.137Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.746312475Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699}
141512024-04-17T17:19:37.137Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.746539059Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699}
141522024-04-17T17:19:37.137Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.746563741Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699}
141532024-04-17T17:19:37.142Z{"msg":"Crucible c3d8a5f3-89ff-4bca-bcc9-6b6831857d88 has session id: 4856506f-8d16-4b44-bc84-c78fba5ddf3e","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.746595342Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141542024-04-17T17:19:37.143Z{"msg":"Upstairs opts: Upstairs UUID: c3d8a5f3-89ff-4bca-bcc9-6b6831857d88, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
141552024-04-17T17:19:37.143Z,"time":"2024-04-17T17:19:35.746653156Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141562024-04-17T17:19:37.143Z{"msg":"Crucible stats registered with UUID: c3d8a5f3-89ff-4bca-bcc9-6b6831857d88","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.746685218Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141572024-04-17T17:19:37.143Z{"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.74671845Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699}
141582024-04-17T17:19:37.143Z{"msg":"c3d8a5f3-89ff-4bca-bcc9-6b6831857d88 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.746866239Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141592024-04-17T17:19:37.143Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.74689116Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"client":"0","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141602024-04-17T17:19:37.143Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.746913012Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"client":"0","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141612024-04-17T17:19:37.143Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.746947934Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"client":"1","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141622024-04-17T17:19:37.143Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.747017768Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"client":"1","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141632024-04-17T17:19:37.143Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.747059131Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"client":"2","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141642024-04-17T17:19:37.143Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.747084762Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"client":"2","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141652024-04-17T17:19:37.143Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.747121034Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141662024-04-17T17:19:37.143Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.747161307Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141672024-04-17T17:19:37.143Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.747186218Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141682024-04-17T17:19:37.143Z{"msg":"ds_connection connected from Ok(127.0.0.1:33804)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.74721974Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141692024-04-17T17:19:37.143Z{"msg":"ds_connection connected from Ok(127.0.0.1:45887)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.747299755Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141702024-04-17T17:19:37.143Z{"msg":"ds_connection connected from Ok(127.0.0.1:44906)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.747359259Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141712024-04-17T17:19:37.143Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.747947314Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"client":"0","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141722024-04-17T17:19:37.143Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.747982607Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"client":"0","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141732024-04-17T17:19:37.143Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.747998917Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"client":"2","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141742024-04-17T17:19:37.143Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.748013468Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"client":"2","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141752024-04-17T17:19:37.143Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.748515009Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"client":"2","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141762024-04-17T17:19:37.144Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.748554221Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"client":"0","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141772024-04-17T17:19:37.144Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.748658148Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"client":"1","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141782024-04-17T17:19:37.144Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.748723891Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"client":"1","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141792024-04-17T17:19:37.144Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.749150217Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"client":"2","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141802024-04-17T17:19:37.144Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.74918748Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"client":"2","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141812024-04-17T17:19:37.144Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.749205001Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141822024-04-17T17:19:37.144Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.749279475Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"client":"0","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141832024-04-17T17:19:37.144Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.749319788Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"client":"0","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141842024-04-17T17:19:37.144Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.74935169Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141852024-04-17T17:19:37.144Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.749469597Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"client":"1","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141862024-04-17T17:19:37.144Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.749884952Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"client":"1","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141872024-04-17T17:19:37.144Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.749926244Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"client":"1","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141882024-04-17T17:19:37.144Z{"msg":"[0]R flush_numbers[0..12]: [207, 229, 225, 227, 229, 226, 227, 231, 231, 231, 227, 223]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.749952896Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"":"downstairs","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141892024-04-17T17:19:37.144Z{"msg":"[0]R generation[0..12]: [114, 129, 126, 128, 129, 127, 128, 130, 130, 130, 128, 126]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.749981828Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"":"downstairs","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141902024-04-17T17:19:37.144Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.750006999Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"":"downstairs","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141912024-04-17T17:19:37.144Z{"msg":"[1]R flush_numbers[0..12]: [207, 229, 225, 227, 229, 226, 227, 231, 231, 231, 227, 223]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.750032061Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"":"downstairs","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141922024-04-17T17:19:37.144Z{"msg":"[1]R generation[0..12]: [114, 129, 126, 128, 129, 127, 128, 130, 130, 130, 128, 126]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.750056542Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"":"downstairs","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141932024-04-17T17:19:37.144ZWait for a query_work_queue command to finish before sending IO
141942024-04-17T17:19:37.144Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.750099165Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"":"downstairs","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141952024-04-17T17:19:37.144Z{"msg":"[2]R flush_numbers[0..12]: [207, 229, 225, 227, 229, 226, 227, 231, 231, 231, 227, 223]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.750128766Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"":"downstairs","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141962024-04-17T17:19:37.144Z{"msg":"[2]R generation[0..12]: [114, 129, 126, 128, 129, 127, 128, 130, 130, 130, 128, 126]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.750148898Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"":"downstairs","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141972024-04-17T17:19:37.144Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.750172759Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"":"downstairs","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141982024-04-17T17:19:37.144Z{"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.750202431Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"":"downstairs","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
141992024-04-17T17:19:37.145Z{"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.750225372Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"":"downstairs","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
142002024-04-17T17:19:37.145Z{"msg":"Next flush: 233","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.750251634Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"":"downstairs","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
142012024-04-17T17:19:37.145Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.750273975Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"":"downstairs","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
142022024-04-17T17:19:37.145Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.750301027Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
142032024-04-17T17:19:37.145Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.750320838Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"client":"0","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
142042024-04-17T17:19:37.145Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.750339369Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"client":"1","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
142052024-04-17T17:19:37.152Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.750358671Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"client":"2","":"downstairs","session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
142062024-04-17T17:19:37.152Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
142072024-04-17T17:19:37.152Z,"level":30,"time":"2024-04-17T17:19:35.750400053Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
142082024-04-17T17:19:37.152Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.750441196Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
142092024-04-17T17:19:37.152Z{"msg":"c3d8a5f3-89ff-4bca-bcc9-6b6831857d88 is now active with session: 4856506f-8d16-4b44-bc84-c78fba5ddf3e","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.750467147Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
142102024-04-17T17:19:37.152Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.750498709Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699,"session_id":"4856506f-8d16-4b44-bc84-c78fba5ddf3e"}
142112024-04-17T17:19:37.152Z{"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:35.75051923Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699}
142122024-04-17T17:19:37.152ZLoading write count information from file "/tmp/test_repair_verify.data"
142132024-04-17T17:19:37.152ZRun Repair workload
142142024-04-17T17:19:37.152Z01/30 Write block 293 len 12288 data: 12 10 7
142152024-04-17T17:19:37.152Z02/30 Read block 184 len 4096
142162024-04-17T17:19:37.152Z03/30 Read block 164 len 12288
142172024-04-17T17:19:37.152Z04/30 Read block 17 len 16384
142182024-04-17T17:19:37.152Z05/30 Flush
142192024-04-17T17:19:37.152Z06/30 Write block 353 len 20480 data: 7 7 5 5 6
142202024-04-17T17:19:37.152Z07/30 Flush
142212024-04-17T17:19:37.156Z08/30 Write block 47 len 32768 data: 8 7 10 9 10 10 11 15
142222024-04-17T17:19:37.156Z09/30 Read block 265 len 32768
142232024-04-17T17:19:37.156Z10/30 Read block 546 len 8192
142242024-04-17T17:19:37.156Z11/30 Write block 3 len 8192 data: 8 9
142252024-04-17T17:19:37.156Z12/30 Read block 214 len 4096
142262024-04-17T17:19:37.156Z13/30 Read block 489 len 32768
142272024-04-17T17:19:37.156Z14/30 Read block 121 len 8192
142282024-04-17T17:19:37.156Z15/30 Read block 0 len 32768
142292024-04-17T17:19:37.160Z16/30 Write block 12 len 40960 data: 9 10 11 8 6 9 8 9 10 13
142302024-04-17T17:19:37.160Z17/30 Read block 596 len 8192
142312024-04-17T17:19:37.160Z18/30 Read block 186 len 40960
142322024-04-17T17:19:37.160Z19/30 Write block 336 len 20480 data: 12 14 13 13 12
142332024-04-17T17:19:37.160Z20/30 Read block 226 len 8192
142342024-04-17T17:19:37.163Z21/30 Write block 439 len 20480 data: 14 10 10 12 12
142352024-04-17T17:19:37.163Z22/30 Write block 197 len 8192 data: 8 10
142362024-04-17T17:19:37.164Z23/30 Write block 268 len 24576 data: 8 11 10 12 11 13
142372024-04-17T17:19:37.164Z24/30 Read block 537 len 4096
142382024-04-17T17:19:37.167Z25/30 Flush
142392024-04-17T17:19:37.170Z26/30 Write block 240 len 20480 data: 9 12 11 12 11
142402024-04-17T17:19:37.170Z27/30 Write block 99 len 20480 data: 14 14 12 11 13
142412024-04-17T17:19:37.170Z28/30 Write block 158 len 16384 data: 10 12 12 12
142422024-04-17T17:19:37.170Z29/30 Read block 129 len 8192
142432024-04-17T17:19:37.174Z30/30 Write block 247 len 24576 data: 13 13 10 9 10 11
142442024-04-17T17:19:37.174Z----------------------------------------------------------------
142452024-04-17T17:19:37.174Z Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:30
142462024-04-17T17:19:37.174ZStates: Active Active Active
142472024-04-17T17:19:37.174ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
142482024-04-17T17:19:37.174Z 1 Acked 1000 Write 3 Done Done Done false
142492024-04-17T17:19:37.174Z 2 Acked 1001 Read 1 Done Done Done false
142502024-04-17T17:19:37.175Z 3 Acked 1002 Read 3 Done Sent Done false
142512024-04-17T17:19:37.175Z 4 Acked 1003 Read 4 Done Sent Done false
142522024-04-17T17:19:37.175Z 5 Acked 1004 Flush 0 Done Sent Done false
142532024-04-17T17:19:37.175Z 6 Acked 1005 Write 5 Done Sent Done false
142542024-04-17T17:19:37.175Z 7 Acked 1006 Flush 0 Done Sent Done false
142552024-04-17T17:19:37.175Z 8 Acked 1007 Write 8 Done Sent Done false
142562024-04-17T17:19:37.175Z 9 Acked 1008 Read 8 Done Sent Done false
142572024-04-17T17:19:37.175Z 10 Acked 1009 Read 2 Done Sent Done false
142582024-04-17T17:19:37.175Z 11 Acked 1010 Write 2 Done Sent Done false
142592024-04-17T17:19:37.175Z 12 Acked 1011 Read 1 Done Sent Done false
142602024-04-17T17:19:37.175Z 13 Acked 1012 Read 8 Done Sent Done false
142612024-04-17T17:19:37.175Z 14 Acked 1013 Read 2 Done Sent Done false
142622024-04-17T17:19:37.175Z 15 Acked 1014 Read 8 Done Sent Done false
142632024-04-17T17:19:37.175Z 16 Acked 1015 Write 10 Done Sent Done false
142642024-04-17T17:19:37.175Z 17 Acked 1016 Read 2 Done Sent Done false
142652024-04-17T17:19:37.175Z 18 Acked 1017 Read 10 Done Sent Done false
142662024-04-17T17:19:37.175Z 19 Acked 1018 Write 5 Done Sent Done false
142672024-04-17T17:19:37.175Z 20 Acked 1019 Read 2 Done Sent Done false
142682024-04-17T17:19:37.175Z 21 Acked 1020 Write 5 Done Sent Done false
142692024-04-17T17:19:37.175Z 22 Acked 1021 Write 2 Done Sent Done false
142702024-04-17T17:19:37.175Z 23 Acked 1022 Write 6 Done Sent Done false
142712024-04-17T17:19:37.175Z 24 Acked 1023 Read 1 Done Sent Done false
142722024-04-17T17:19:37.175Z 25 Acked 1024 Flush 0 Done Sent Done false
142732024-04-17T17:19:37.176Z 26 Acked 1025 Write 5 Done Sent Done false
142742024-04-17T17:19:37.176Z 27 Acked 1026 Write 5 Done Sent Done false
142752024-04-17T17:19:37.176Z 28 Acked 1027 Write 4 Done Sent Done false
142762024-04-17T17:19:37.176Z 29 Acked 1028 Read 2 Done Sent Done false
142772024-04-17T17:19:37.176Z 30 Acked 1029 Write 6 Sent Sent Sent false
142782024-04-17T17:19:37.176Z STATES DS:0 DS:1 DS:2 TOTAL
142792024-04-17T17:19:37.176Z New 0 0 0 0
142802024-04-17T17:19:37.176Z Sent 1 28 1 30
142812024-04-17T17:19:37.176Z Done 29 2 29 60
142822024-04-17T17:19:37.176Z Skipped 0 0 0 0
142832024-04-17T17:19:37.176Z Error 0 0 0 0
142842024-04-17T17:19:37.176ZLast Flush: 1024 0 1024
142852024-04-17T17:19:37.176ZDownstairs last five completed:
142862024-04-17T17:19:37.176ZUpstairs last five completed: 30 29 28 27 26
142872024-04-17T17:19:37.176Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:19:35.781286215Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1699}
142882024-04-17T17:19:37.176ZWrote out file "/tmp/test_repair_verify.data"
142892024-04-17T17:19:37.180Z
142902024-04-17T17:19:37.180ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
142912024-04-17T17:19:37.196ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
142922024-04-17T17:19:37.196Z 0 000-019 132 114 132 235 207 235 F F F <---
142932024-04-17T17:19:37.200Z 1 020-039 132 129 132 235 229 235 F F F <---
142942024-04-17T17:19:37.200Z 2 040-059 132 126 132 235 225 235 F F F <---
142952024-04-17T17:19:37.200Z 3 060-079 128 128 128 227 227 227 F F F
142962024-04-17T17:19:37.200Z 4 080-099 129 129 129 229 229 229 T F T <---
142972024-04-17T17:19:37.200Z 5 100-119 127 127 127 226 226 226 T F T <---
142982024-04-17T17:19:37.200Z 6 120-139 128 128 128 227 227 227 F F F
142992024-04-17T17:19:37.200Z 7 140-159 130 130 130 231 231 231 T F T <---
143002024-04-17T17:19:37.200Z 8 160-179 130 130 130 231 231 231 T F T <---
143012024-04-17T17:19:37.200Z 9 180-199 132 130 132 235 231 235 F F F <---
143022024-04-17T17:19:37.200Z 10 200-219 128 128 128 227 227 227 F F F
143032024-04-17T17:19:37.200Z 11 220-239 126 126 126 223 223 223 F F F
143042024-04-17T17:19:37.200Z 12 240-259 124 124 124 220 220 220 T F T <---
143052024-04-17T17:19:37.200Z 13 260-279 132 126 132 235 222 235 F F F <---
143062024-04-17T17:19:37.200Z 14 280-299 132 128 132 233 227 233 F T F <---
143072024-04-17T17:19:37.200Z 15 300-319 127 127 127 226 226 226 F F F
143082024-04-17T17:19:37.200Z 16 320-339 132 131 132 235 232 235 F F F <---
143092024-04-17T17:19:37.201Z 17 340-359 132 128 132 235 227 235 F F F <---
143102024-04-17T17:19:37.201Z 18 360-379 130 130 130 231 231 231 F F F
143112024-04-17T17:19:37.201Z 19 380-399 126 126 126 225 225 225 F F F
143122024-04-17T17:19:37.201Z 20 400-419 131 131 131 232 232 232 F F F
143132024-04-17T17:19:37.201Z 21 420-439 132 131 132 235 232 235 F F F <---
143142024-04-17T17:19:37.201Z 22 440-459 132 129 132 235 229 235 F F F <---
143152024-04-17T17:19:37.201Z 23 460-479 130 130 130 231 231 231 F F F
143162024-04-17T17:19:37.201Z 24 480-499 130 130 130 231 231 231 F F F
143172024-04-17T17:19:37.201Z 25 500-519 130 130 130 230 230 230 F F F
143182024-04-17T17:19:37.201Z 26 520-539 130 130 130 231 231 231 F F F
143192024-04-17T17:19:37.201Z 27 540-559 128 128 128 227 227 227 F F F
143202024-04-17T17:19:37.201Z 28 560-579 125 125 125 221 221 221 F F F
143212024-04-17T17:19:37.201Z 29 580-599 130 130 130 231 231 231 F F F
143222024-04-17T17:19:37.201ZMax gen: 132, Max flush: 235
143232024-04-17T17:19:37.201ZError: Difference in extent metadata found!
143242024-04-17T17:19:37.201ZA Difference in extent metadata is expected here
143252024-04-17T17:19:37.201ZOn loop 66
143262024-04-17T17:19:37.201Z
143272024-04-17T17:19:37.205ZVerifying data now
143282024-04-17T17:19:41.888ZNo change after verify
143292024-04-17T17:19:41.888ZLoop: 66 Downstairs dump after verify (and repair):
143302024-04-17T17:19:41.903ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
143312024-04-17T17:19:41.903Z 0 000-019 132 132 132 235 235 235 F F F
143322024-04-17T17:19:41.903Z 1 020-039 132 132 132 235 235 235 F F F
143332024-04-17T17:19:41.903Z 2 040-059 132 132 132 235 235 235 F F F
143342024-04-17T17:19:41.903Z 3 060-079 128 128 128 227 227 227 F F F
143352024-04-17T17:19:41.903Z 4 080-099 133 133 133 236 236 236 F F F
143362024-04-17T17:19:41.903Z 5 100-119 133 133 133 236 236 236 F F F
143372024-04-17T17:19:41.907Z 6 120-139 128 128 128 227 227 227 F F F
143382024-04-17T17:19:41.907Z 7 140-159 133 133 133 236 236 236 F F F
143392024-04-17T17:19:41.907Z 8 160-179 133 133 133 236 236 236 F F F
143402024-04-17T17:19:41.907Z 9 180-199 132 132 132 235 235 235 F F F
143412024-04-17T17:19:41.907Z 10 200-219 128 128 128 227 227 227 F F F
143422024-04-17T17:19:41.907Z 11 220-239 126 126 126 223 223 223 F F F
143432024-04-17T17:19:41.907Z 12 240-259 133 133 133 236 236 236 F F F
143442024-04-17T17:19:41.907Z 13 260-279 132 132 132 235 235 235 F F F
143452024-04-17T17:19:41.907Z 14 280-299 132 132 132 233 233 233 F F F
143462024-04-17T17:19:41.907Z 15 300-319 127 127 127 226 226 226 F F F
143472024-04-17T17:19:41.907Z 16 320-339 132 132 132 235 235 235 F F F
143482024-04-17T17:19:41.907Z 17 340-359 132 132 132 235 235 235 F F F
143492024-04-17T17:19:41.907Z 18 360-379 130 130 130 231 231 231 F F F
143502024-04-17T17:19:41.907Z 19 380-399 126 126 126 225 225 225 F F F
143512024-04-17T17:19:41.907Z 20 400-419 131 131 131 232 232 232 F F F
143522024-04-17T17:19:41.907Z 21 420-439 132 132 132 235 235 235 F F F
143532024-04-17T17:19:41.907Z 22 440-459 132 132 132 235 235 235 F F F
143542024-04-17T17:19:41.907Z 23 460-479 130 130 130 231 231 231 F F F
143552024-04-17T17:19:41.908Z 24 480-499 130 130 130 231 231 231 F F F
143562024-04-17T17:19:41.908Z 25 500-519 130 130 130 230 230 230 F F F
143572024-04-17T17:19:41.908Z 26 520-539 130 130 130 231 231 231 F F F
143582024-04-17T17:19:41.908Z 27 540-559 128 128 128 227 227 227 F F F
143592024-04-17T17:19:41.908Z 28 560-579 125 125 125 221 221 221 F F F
143602024-04-17T17:19:41.908Z 29 580-599 130 130 130 231 231 231 F F F
143612024-04-17T17:19:41.908ZMax gen: 133, Max flush: 236
143622024-04-17T17:19:41.908Z
143632024-04-17T17:19:41.908ZBegin loop 67
143642024-04-17T17:19:41.908ZDownstairs to restart: 2
143652024-04-17T17:19:41.927Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.536058568Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707}
143662024-04-17T17:19:41.927Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.536290002Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707}
143672024-04-17T17:19:41.927Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.536306663Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707}
143682024-04-17T17:19:41.934Z{"msg":"Crucible fc25e527-98f2-4fde-9a50-f38d20b2f2a4 has session id: 68eb42ac-ab48-4048-b1b2-269ba152ef42","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.536337725Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
143692024-04-17T17:19:41.934Z{Crucible runtime is spawned
143702024-04-17T17:19:41.934Z"msg":"Upstairs opts: Upstairs UUID: fc25e527-98f2-4fde-9a50-f38d20b2f2a4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.536447482Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
143712024-04-17T17:19:41.934Z{"msg":"Crucible stats registered with UUID: fc25e527-98f2-4fde-9a50-f38d20b2f2a4","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.536492925Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
143722024-04-17T17:19:41.934Z{"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.536518456Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707}
143732024-04-17T17:19:41.934Z{"msg":"fc25e527-98f2-4fde-9a50-f38d20b2f2a4 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.536624733Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
143742024-04-17T17:19:41.934Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.536661295Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"client":"0","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
143752024-04-17T17:19:41.934Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.536679626Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"client":"0","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
143762024-04-17T17:19:41.934Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.536711948Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"client":"1","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
143772024-04-17T17:19:41.934Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.53673679Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"client":"1","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
143782024-04-17T17:19:41.934Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.536773862Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"client":"2","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
143792024-04-17T17:19:41.934Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.536797203Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"client":"2","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
143802024-04-17T17:19:41.934Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.536827235Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
143812024-04-17T17:19:41.934Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.536859507Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
143822024-04-17T17:19:41.934Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.536887028Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
143832024-04-17T17:19:41.934Z{"msg":"ds_connection connected from Ok(127.0.0.1:49129)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.536960293Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
143842024-04-17T17:19:41.934Z{"msg":"ds_connection connected from Ok(127.0.0.1:49134)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.537011536Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
143852024-04-17T17:19:41.934Z{"msg":"ds_connection connected from Ok(127.0.0.1:48254)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.537057919Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
143862024-04-17T17:19:41.934Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.53774027Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"client":"0","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
143872024-04-17T17:19:41.934Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.537785233Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"client":"0","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
143882024-04-17T17:19:41.934Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.537801204Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"client":"1","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
143892024-04-17T17:19:41.934Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.537826096Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"client":"1","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
143902024-04-17T17:19:41.935Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.537863218Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"client":"2","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
143912024-04-17T17:19:41.935Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.537930792Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"client":"2","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
143922024-04-17T17:19:41.935Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.538303254Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"client":"0","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
143932024-04-17T17:19:41.935Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.538340037Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"client":"1","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
143942024-04-17T17:19:41.935Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.538372098Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"client":"2","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
143952024-04-17T17:19:41.935Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.538800495Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"client":"1","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
143962024-04-17T17:19:41.935Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.538837467Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"client":"1","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
143972024-04-17T17:19:41.935Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.538852318Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
143982024-04-17T17:19:41.935Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.538945683Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"client":"2","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
143992024-04-17T17:19:41.935Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.538972115Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"client":"2","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
144002024-04-17T17:19:41.935Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.538985936Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
144012024-04-17T17:19:41.935Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.538997836Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"client":"0","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
144022024-04-17T17:19:41.935Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.539010847Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"client":"0","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
144032024-04-17T17:19:41.935Z{"msg":"[0]R flush_numbers[0..12]: [235, 235, 235, 227, 236, 236, 227, 236, 236, 235, 227, 223]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.539031318Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"":"downstairs","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
144042024-04-17T17:19:41.935Z{"msg":"[0]R generation[0..12]: [132, 132, 132, 128, 133, 133, 128, 133, 133, 132, 128, 126]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.539045769Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"":"downstairs","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
144052024-04-17T17:19:41.935Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.53906034Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"":"downstairs","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
144062024-04-17T17:19:41.935Z{"msg":"[1]R flush_numbers[0..12]: [235, 235, 235, 227, 236, 236, 227, 236, 236, 235, 227, 223]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.539081012Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"":"downstairs","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
144072024-04-17T17:19:41.935Z{"msg":"[1]R generation[0..12]: [132, 132, 132, 128, 133, 133, 128, 133, 133, 132, 128, 126]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.539095102Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"":"downstairs","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
144082024-04-17T17:19:41.935Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.539109343Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"":"downstairs","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
144092024-04-17T17:19:41.935ZWait for a query_work_queue command to finish before sending IO
144102024-04-17T17:19:41.935Z{"msg":"[2]R flush_numbers[0..12]: [235, 235, 235, 227, 236, 236, 227, 236, 236, 235, 227, 223]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.539136105Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"":"downstairs","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
144112024-04-17T17:19:41.935Z{"msg":"[2]R generation[0..12]: [132, 132, 132, 128, 133, 133, 128, 133, 133, 132, 128, 126]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.539150676Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"":"downstairs","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
144122024-04-17T17:19:41.935Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.539171347Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"":"downstairs","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
144132024-04-17T17:19:41.936Z{"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.539184418Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"":"downstairs","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
144142024-04-17T17:19:41.936Z{"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.539197789Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"":"downstairs","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
144152024-04-17T17:19:41.936Z{"msg":"Next flush: 237","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.539210449Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"":"downstairs","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
144162024-04-17T17:19:41.936Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.53922312Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"":"downstairs","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
144172024-04-17T17:19:41.936Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.539236721Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
144182024-04-17T17:19:41.936Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.539248412Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"client":"0","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
144192024-04-17T17:19:41.936Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.539261192Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"client":"1","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
144202024-04-17T17:19:41.936Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.539281444Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"client":"2","":"downstairs","session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
144212024-04-17T17:19:41.936Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.539301805Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
144222024-04-17T17:19:41.936Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.539314826Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
144232024-04-17T17:19:41.936Z{"msg":"fc25e527-98f2-4fde-9a50-f38d20b2f2a4 is now active with session: 68eb42ac-ab48-4048-b1b2-269ba152ef42","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.539335087Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
144242024-04-17T17:19:41.936Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.539347318Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707,"session_id":"68eb42ac-ab48-4048-b1b2-269ba152ef42"}
144252024-04-17T17:19:41.936Z{"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:40.539359778Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707}
144262024-04-17T17:19:41.936ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
144272024-04-17T17:19:41.936ZLoading write count information from file "/tmp/test_repair_verify.data"
144282024-04-17T17:19:41.936ZRun Repair workload
144292024-04-17T17:19:41.940Z01/30 Write block 31 len 8192 data: 13 17
144302024-04-17T17:19:41.941Z02/30 Read block 150 len 36864
144312024-04-17T17:19:41.941Z03/30 Write block 180 len 12288 data: 8 8 8
144322024-04-17T17:19:41.941Z04/30 Read block 256 len 8192
144332024-04-17T17:19:41.941Z05/30 Write block 54 len 12288 data: 16 13 13
144342024-04-17T17:19:41.941Z06/30 Write block 433 len 16384 data: 13 12 13 13
144352024-04-17T17:19:41.941Z07/30 Read block 586 len 36864
144362024-04-17T17:19:41.941Z08/30 Read block 297 len 32768
144372024-04-17T17:19:41.941Z09/30 Flush
144382024-04-17T17:19:41.941Z10/30 Write block 359 len 12288 data: 7 8 8
144392024-04-17T17:19:41.946Z11/30 Flush
144402024-04-17T17:19:41.946Z12/30 Flush
144412024-04-17T17:19:41.946Z13/30 Write block 383 len 28672 data: 11 10 12 10 12 10 11
144422024-04-17T17:19:41.946Z14/30 Read block 596 len 16384
144432024-04-17T17:19:41.950Z15/30 Read block 233 len 8192
144442024-04-17T17:19:41.950Z16/30 Write block 474 len 4096 data: 13
144452024-04-17T17:19:41.950Z17/30 Flush
144462024-04-17T17:19:41.950Z18/30 Write block 455 len 28672 data: 7 7 7 9 10 11 9
144472024-04-17T17:19:41.950Z19/30 Read block 486 len 20480
144482024-04-17T17:19:41.954Z20/30 Read block 119 len 16384
144492024-04-17T17:19:41.954Z21/30 Read block 402 len 28672
144502024-04-17T17:19:41.954Z22/30 Write block 4 len 32768 data: 10 10 8 7 7 8 8 7
144512024-04-17T17:19:41.954Z23/30 Write block 73 len 12288 data: 7 9 10
144522024-04-17T17:19:41.959Z24/30 Write block 579 len 12288 data: 13 11 12
144532024-04-17T17:19:41.959Z25/30 Write block 75 len 40960 data: 11 10 10 9 8 11 9 11 14 10
144542024-04-17T17:19:41.959Z26/30 Flush
144552024-04-17T17:19:41.967Z27/30 Write block 314 len 20480 data: 13 12 11 12 14
144562024-04-17T17:19:41.967Z28/30 Read block 463 len 24576
144572024-04-17T17:19:41.968Z29/30 Write block 343 len 16384 data: 13 12 13 12
144582024-04-17T17:19:41.968Z30/30 Read block 354 len 24576
144592024-04-17T17:19:41.968Z----------------------------------------------------------------
144602024-04-17T17:19:41.968Z Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30
144612024-04-17T17:19:41.968ZStates: Active Active Active
144622024-04-17T17:19:41.968ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
144632024-04-17T17:19:41.968Z 1 Acked 1000 Write 2 Done Done Sent false
144642024-04-17T17:19:41.968Z 2 Acked 1001 Read 9 Done Done Sent false
144652024-04-17T17:19:41.969Z 3 Acked 1002 Write 3 Done Done Sent false
144662024-04-17T17:19:41.969Z 4 Acked 1003 Read 2 Done Done Sent false
144672024-04-17T17:19:41.969Z 5 Acked 1004 Write 3 Done Done Sent false
144682024-04-17T17:19:41.969Z 6 Acked 1005 Write 4 Done Done Sent false
144692024-04-17T17:19:41.969Z 7 Acked 1006 Read 9 Done Done Sent false
144702024-04-17T17:19:41.969Z 8 Acked 1007 Read 8 Done Done Sent false
144712024-04-17T17:19:41.969Z 9 Acked 1008 Flush 0 Done Done Sent false
144722024-04-17T17:19:41.970Z 10 Acked 1009 Write 3 Done Done Sent false
144732024-04-17T17:19:41.970Z 11 Acked 1010 Flush 0 Done Done Sent false
144742024-04-17T17:19:41.970Z 12 Acked 1011 Flush 0 Done Done Sent false
144752024-04-17T17:19:41.970Z 13 Acked 1012 Write 7 Done Done Sent false
144762024-04-17T17:19:41.970Z 14 Acked 1013 Read 4 Done Done Sent false
144772024-04-17T17:19:41.970Z 15 Acked 1014 Read 2 Done Done Sent false
144782024-04-17T17:19:41.971Z 16 Acked 1015 Write 1 Done Done Sent false
144792024-04-17T17:19:41.971Z 17 Acked 1016 Flush 0 Done Done Sent false
144802024-04-17T17:19:41.971Z 18 Acked 1017 Write 7 Done Done Sent false
144812024-04-17T17:19:41.971Z 19 Acked 1018 Read 5 Done Done Sent false
144822024-04-17T17:19:41.971Z 20 Acked 1019 Read 4 Done Done Sent false
144832024-04-17T17:19:41.971Z 21 Acked 1020 Read 7 Done Done Sent false
144842024-04-17T17:19:41.971Z 22 Acked 1021 Write 8 Done Done Sent false
144852024-04-17T17:19:41.972Z 23 Acked 1022 Write 3 Done Done Sent false
144862024-04-17T17:19:41.972Z 24 Acked 1023 Write 3 Done Done Sent false
144872024-04-17T17:19:41.972Z 25 Acked 1024 Write 10 Done Done Sent false
144882024-04-17T17:19:41.972Z 26 Acked 1025 Flush 0 Done Done Sent false
144892024-04-17T17:19:41.972Z 27 Acked 1026 Write 5 Done Done Sent false
144902024-04-17T17:19:41.972Z 28 Acked 1027 Read 6 Done Done Sent false
144912024-04-17T17:19:41.972Z 29 Acked 1028 Write 4 Done Done Sent false
144922024-04-17T17:19:41.973Z 30 Acked 1029 Read 6 Done Sent Sent false
144932024-04-17T17:19:41.973Z STATES DS:0 DS:1 DS:2 TOTAL
144942024-04-17T17:19:41.973Z New 0 0 0 0
144952024-04-17T17:19:41.973Z Sent 0 1 30 31
144962024-04-17T17:19:41.973Z Done 30 29 0 59
144972024-04-17T17:19:41.973Z Skipped 0 0 0 0
144982024-04-17T17:19:41.973Z Error 0 0 0 0
144992024-04-17T17:19:41.973ZLast Flush: 1025 1025 0
145002024-04-17T17:19:41.973ZDownstairs last five completed:
145012024-04-17T17:19:41.973ZUpstairs last five completed: 30 29 28 27 26
145022024-04-17T17:19:41.973Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:19:40.572432353Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1707}
145032024-04-17T17:19:41.973ZWrote out file "/tmp/test_repair_verify.data"
145042024-04-17T17:19:41.983Z
145052024-04-17T17:19:41.983ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
145062024-04-17T17:19:41.987ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
145072024-04-17T17:19:41.987Z 0 000-019 134 134 132 241 241 235 F F F <---
145082024-04-17T17:19:41.987Z 1 020-039 134 134 132 237 237 235 F F F <---
145092024-04-17T17:19:41.987Z 2 040-059 134 134 132 237 237 235 F F F <---
145102024-04-17T17:19:41.987Z 3 060-079 134 134 128 241 241 227 F F F <---
145112024-04-17T17:19:41.987Z 4 080-099 134 134 133 241 241 236 F F F <---
145122024-04-17T17:19:41.987Z 5 100-119 133 133 133 236 236 236 F F F
145132024-04-17T17:19:41.987Z 6 120-139 128 128 128 227 227 227 F F F
145142024-04-17T17:19:41.987Z 7 140-159 133 133 133 236 236 236 F F F
145152024-04-17T17:19:41.987Z 8 160-179 133 133 133 236 236 236 F F F
145162024-04-17T17:19:41.988Z 9 180-199 134 134 132 237 237 235 F F F <---
145172024-04-17T17:19:41.988Z 10 200-219 128 128 128 227 227 227 F F F
145182024-04-17T17:19:41.988Z 11 220-239 126 126 126 223 223 223 F F F
145192024-04-17T17:19:41.988Z 12 240-259 133 133 133 236 236 236 F F F
145202024-04-17T17:19:41.988Z 13 260-279 132 132 132 235 235 235 F F F
145212024-04-17T17:19:41.988Z 14 280-299 132 132 132 233 233 233 F F F
145222024-04-17T17:19:41.988Z 15 300-319 127 127 127 226 226 226 T T F <---
145232024-04-17T17:19:41.988Z 16 320-339 132 132 132 235 235 235 F F F
145242024-04-17T17:19:41.988Z 17 340-359 134 134 132 238 238 235 T T F <---
145252024-04-17T17:19:41.988Z 18 360-379 134 134 130 238 238 231 F F F <---
145262024-04-17T17:19:41.988Z 19 380-399 134 134 126 240 240 225 F F F <---
145272024-04-17T17:19:41.988Z 20 400-419 131 131 131 232 232 232 F F F
145282024-04-17T17:19:41.988Z 21 420-439 134 134 132 237 237 235 F F F <---
145292024-04-17T17:19:41.988Z 22 440-459 134 134 132 241 241 235 F F F <---
145302024-04-17T17:19:41.988Z 23 460-479 134 134 130 241 241 231 F F F <---
145312024-04-17T17:19:41.988Z 24 480-499 130 130 130 231 231 231 F F F
145322024-04-17T17:19:41.988Z 25 500-519 130 130 130 230 230 230 F F F
145332024-04-17T17:19:41.988Z 26 520-539 130 130 130 231 231 231 F F F
145342024-04-17T17:19:41.988Z 27 540-559 128 128 128 227 227 227 F F F
145352024-04-17T17:19:41.988Z 28 560-579 134 134 125 241 241 221 F F F <---
145362024-04-17T17:19:41.988Z 29 580-599 134 134 130 241 241 231 F F F <---
145372024-04-17T17:19:41.988ZMax gen: 134, Max flush: 241
145382024-04-17T17:19:41.992ZError: Difference in extent metadata found!
145392024-04-17T17:19:41.992ZA Difference in extent metadata is expected here
145402024-04-17T17:19:41.992ZOn loop 67
145412024-04-17T17:19:41.992Z
145422024-04-17T17:19:41.996ZVerifying data now
145432024-04-17T17:19:46.679ZNo change after verify
145442024-04-17T17:19:46.679ZLoop: 67 Downstairs dump after verify (and repair):
145452024-04-17T17:19:46.693ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
145462024-04-17T17:19:46.693Z 0 000-019 134 134 134 241 241 241 F F F
145472024-04-17T17:19:46.698Z 1 020-039 134 134 134 237 237 237 F F F
145482024-04-17T17:19:46.698Z 2 040-059 134 134 134 237 237 237 F F F
145492024-04-17T17:19:46.698Z 3 060-079 134 134 134 241 241 241 F F F
145502024-04-17T17:19:46.698Z 4 080-099 134 134 134 241 241 241 F F F
145512024-04-17T17:19:46.698Z 5 100-119 133 133 133 236 236 236 F F F
145522024-04-17T17:19:46.698Z 6 120-139 128 128 128 227 227 227 F F F
145532024-04-17T17:19:46.698Z 7 140-159 133 133 133 236 236 236 F F F
145542024-04-17T17:19:46.698Z 8 160-179 133 133 133 236 236 236 F F F
145552024-04-17T17:19:46.698Z 9 180-199 134 134 134 237 237 237 F F F
145562024-04-17T17:19:46.698Z 10 200-219 128 128 128 227 227 227 F F F
145572024-04-17T17:19:46.698Z 11 220-239 126 126 126 223 223 223 F F F
145582024-04-17T17:19:46.698Z 12 240-259 133 133 133 236 236 236 F F F
145592024-04-17T17:19:46.698Z 13 260-279 132 132 132 235 235 235 F F F
145602024-04-17T17:19:46.698Z 14 280-299 132 132 132 233 233 233 F F F
145612024-04-17T17:19:46.698Z 15 300-319 135 135 135 242 242 242 F F F
145622024-04-17T17:19:46.698Z 16 320-339 132 132 132 235 235 235 F F F
145632024-04-17T17:19:46.698Z 17 340-359 135 135 135 242 242 242 F F F
145642024-04-17T17:19:46.698Z 18 360-379 134 134 134 238 238 238 F F F
145652024-04-17T17:19:46.698Z 19 380-399 134 134 134 240 240 240 F F F
145662024-04-17T17:19:46.698Z 20 400-419 131 131 131 232 232 232 F F F
145672024-04-17T17:19:46.699Z 21 420-439 134 134 134 237 237 237 F F F
145682024-04-17T17:19:46.699Z 22 440-459 134 134 134 241 241 241 F F F
145692024-04-17T17:19:46.699Z 23 460-479 134 134 134 241 241 241 F F F
145702024-04-17T17:19:46.699Z 24 480-499 130 130 130 231 231 231 F F F
145712024-04-17T17:19:46.699Z 25 500-519 130 130 130 230 230 230 F F F
145722024-04-17T17:19:46.699Z 26 520-539 130 130 130 231 231 231 F F F
145732024-04-17T17:19:46.699Z 27 540-559 128 128 128 227 227 227 F F F
145742024-04-17T17:19:46.699Z 28 560-579 134 134 134 241 241 241 F F F
145752024-04-17T17:19:46.699Z 29 580-599 134 134 134 241 241 241 F F F
145762024-04-17T17:19:46.699ZMax gen: 135, Max flush: 242
145772024-04-17T17:19:46.699Z
145782024-04-17T17:19:46.699ZBegin loop 68
145792024-04-17T17:19:46.699ZDownstairs to restart: 0
145802024-04-17T17:19:46.717Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.326087509Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715}
145812024-04-17T17:19:46.717ZCrucible runtime is spawned
145822024-04-17T17:19:46.724Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.326464092Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715}
145832024-04-17T17:19:46.724Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.326513504Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715}
145842024-04-17T17:19:46.724Z{"msg":"Crucible 491264ec-3b6e-4125-b53e-9ea101685f70 has session id: f0df1577-b01d-4269-ae0e-87cb8f33b0d3","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.326536776Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
145852024-04-17T17:19:46.724Z{"msg":"Upstairs opts: Upstairs UUID: 491264ec-3b6e-4125-b53e-9ea101685f70, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.326595609Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
145862024-04-17T17:19:46.724Z{"msg":"Crucible stats registered with UUID: 491264ec-3b6e-4125-b53e-9ea101685f70","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.326627511Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
145872024-04-17T17:19:46.724Z{"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.326652163Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715}
145882024-04-17T17:19:46.724Z{"msg":"491264ec-3b6e-4125-b53e-9ea101685f70 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.326679185Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
145892024-04-17T17:19:46.724Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.326724667Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"client":"0","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
145902024-04-17T17:19:46.724Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.326792741Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"client":"0","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
145912024-04-17T17:19:46.724Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.326846355Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"client":"1","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
145922024-04-17T17:19:46.724Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.326877057Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"client":"1","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
145932024-04-17T17:19:46.724Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.326914019Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"client":"2","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
145942024-04-17T17:19:46.724Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.32693651Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"client":"2","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
145952024-04-17T17:19:46.724Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.326989884Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
145962024-04-17T17:19:46.724Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.327044457Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
145972024-04-17T17:19:46.724Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.32710001Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
145982024-04-17T17:19:46.724Z{"msg":"ds_connection connected from Ok(127.0.0.1:60935)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.327130632Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
145992024-04-17T17:19:46.725Z{"msg":"ds_connection connected from Ok(127.0.0.1:48470)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.327160114Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146002024-04-17T17:19:46.725Z{"msg":"ds_connection connected from Ok(127.0.0.1:57555)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.327198896Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146012024-04-17T17:19:46.725Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.327770961Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"client":"2","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146022024-04-17T17:19:46.725Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.327809443Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"client":"2","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146032024-04-17T17:19:46.725Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.327835055Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"client":"1","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146042024-04-17T17:19:46.725Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.327857366Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"client":"1","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146052024-04-17T17:19:46.725Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.32825164Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"client":"2","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146062024-04-17T17:19:46.725Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.328409039Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"client":"1","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146072024-04-17T17:19:46.725Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.328548668Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"client":"0","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146082024-04-17T17:19:46.725Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.32858929Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"client":"0","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146092024-04-17T17:19:46.725Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.328704657Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"client":"2","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146102024-04-17T17:19:46.725Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.32874192Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"client":"2","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146112024-04-17T17:19:46.725Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.328766901Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146122024-04-17T17:19:46.725Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.328900779Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"client":"1","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146132024-04-17T17:19:46.725Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.328934741Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"client":"1","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146142024-04-17T17:19:46.725Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.328967163Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146152024-04-17T17:19:46.725Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.329116882Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"client":"0","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146162024-04-17T17:19:46.725Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.329545028Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"client":"0","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146172024-04-17T17:19:46.725Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.329594331Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"client":"0","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146182024-04-17T17:19:46.725Z{"msg":"[0]R flush_numbers[0..12]: [241, 237, 237, 241, 241, 236, 227, 236, 236, 237, 227, 223]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.329621993Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"":"downstairs","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146192024-04-17T17:19:46.725Z{"msg":"[0]R generation[0..12]: [134, 134, 134, 134, 134, 133, 128, 133, 133, 134, 128, 126]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.329652735Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"":"downstairs","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146202024-04-17T17:19:46.725Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.329677746Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"":"downstairs","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146212024-04-17T17:19:46.726Z{"msg":"[1]R flush_numbers[0..12]: [241, 237, 237, 241, 241, 236, 227, 236, 236, 237, 227, 223]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.329703658Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"":"downstairs","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146222024-04-17T17:19:46.726Z{"msg":"[1]R generation[0..12]: [134, 134, 134, 134, 134, 133, 128, 133, 133, 134, 128, 126]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
146232024-04-17T17:19:46.726Z,"time":"2024-04-17T17:19:45.329728269Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"":"downstairs","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146242024-04-17T17:19:46.726Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.329752671Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"":"downstairs","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146252024-04-17T17:19:46.726Z{"msg":"[2]R flush_numbers[0..12]: [241, 237, 237, 241, 241, 236, 227, 236, 236, 237, 227, 223]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.329785023Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"":"downstairs","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146262024-04-17T17:19:46.726Z{"msg":"[2]R generation[0..12]: [134, 134, 134, 134, 134, 133, 128, 133, 133, 134, 128, 126]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.329809524Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"":"downstairs","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146272024-04-17T17:19:46.726Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.329837496Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"":"downstairs","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146282024-04-17T17:19:46.726Z{"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.329859677Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"":"downstairs","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146292024-04-17T17:19:46.726Z{"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.329885969Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"":"downstairs","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146302024-04-17T17:19:46.726Z{"msg":"Next flush: 243","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.3299085Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"":"downstairs","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146312024-04-17T17:19:46.726Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.329931302Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"":"downstairs","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146322024-04-17T17:19:46.726Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.329958643Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
146332024-04-17T17:19:46.726Z"session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146342024-04-17T17:19:46.726Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.329985385Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"client":"0","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146352024-04-17T17:19:46.726Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.330012697Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"client":"1","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146362024-04-17T17:19:46.726Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.330035348Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"client":"2","":"downstairs","session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146372024-04-17T17:19:46.726Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.33006281Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146382024-04-17T17:19:46.726Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.330084751Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146392024-04-17T17:19:46.726Z{"msg":"491264ec-3b6e-4125-b53e-9ea101685f70 is now active with session: f0df1577-b01d-4269-ae0e-87cb8f33b0d3","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.330111563Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146402024-04-17T17:19:46.726Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.330132284Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715,"session_id":"f0df1577-b01d-4269-ae0e-87cb8f33b0d3"}
146412024-04-17T17:19:46.726Z{"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:45.330158476Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715}
146422024-04-17T17:19:46.730ZLoading write count information from file "/tmp/test_repair_verify.data"
146432024-04-17T17:19:46.730ZRun Repair workload
146442024-04-17T17:19:46.730Z01/30 Write block 307 len 16384 data: 10 12 11 11
146452024-04-17T17:19:46.730Z02/30 Write block 165 len 36864 data: 10 12 13 11 9 8 9 8 9
146462024-04-17T17:19:46.730Z03/30 Write block 528 len 36864 data: 11 12 11 10 7 6 4 5 6
146472024-04-17T17:19:46.730Z04/30 Read block 513 len 24576
146482024-04-17T17:19:46.731Z05/30 Write block 365 len 40960 data: 7 7 7 11 11 10 12 9 10 8
146492024-04-17T17:19:46.731Z06/30 Flush
146502024-04-17T17:19:46.734Z07/30 Write block 344 len 12288 data: 13 14 13
146512024-04-17T17:19:46.734Z08/30 Flush
146522024-04-17T17:19:46.738Z09/30 Write block 84 len 36864 data: 11 13 10 14 12 14 15 15 16
146532024-04-17T17:19:46.738Z10/30 Read block 575 len 4096
146542024-04-17T17:19:46.738Z11/30 Read block 85 len 4096
146552024-04-17T17:19:46.738Z12/30 Read block 5 len 4096
146562024-04-17T17:19:46.738Z13/30 Write block 570 len 28672 data: 8 8 10 11 14 14 13
146572024-04-17T17:19:46.738Z14/30 Flush
146582024-04-17T17:19:46.744Z15/30 Write block 352 len 8192 data: 7 8
146592024-04-17T17:19:46.744Z16/30 Read block 455 len 36864
146602024-04-17T17:19:46.744Z17/30 Write block 465 len 24576 data: 14 13 12 12 13 14
146612024-04-17T17:19:46.752Z18/30 Read block 204 len 4096
146622024-04-17T17:19:46.752Z19/30 Read block 426 len 20480
146632024-04-17T17:19:46.752Z20/30 Read block 417 len 40960
146642024-04-17T17:19:46.752Z21/30 Flush
146652024-04-17T17:19:46.752Z22/30 Write block 55 len 40960 data: 14 14 14 15 13 14 12 11 14 15
146662024-04-17T17:19:46.752Z23/30 Read block 231 len 40960
146672024-04-17T17:19:46.756Z24/30 Read block 418 len 4096
146682024-04-17T17:19:46.756Z25/30 Read block 372 len 4096
146692024-04-17T17:19:46.756Z26/30 Flush
146702024-04-17T17:19:46.756Z27/30 Write block 412 len 8192 data: 12 13
146712024-04-17T17:19:46.757Z28/30 Write block 424 len 8192 data: 10 9
146722024-04-17T17:19:46.757Z29/30 Read block 554 len 28672
146732024-04-17T17:19:46.757Z30/30 Read block 53 len 32768
146742024-04-17T17:19:46.757Z----------------------------------------------------------------
146752024-04-17T17:19:46.757Z Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:22
146762024-04-17T17:19:46.757ZStates: Active Active Active
146772024-04-17T17:19:46.757ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
146782024-04-17T17:19:46.757Z 9 Acked 1008 Write 9 Done Done Done false
146792024-04-17T17:19:46.757Z 10 Acked 1009 Read 1 Done Done Done false
146802024-04-17T17:19:46.757Z 11 Acked 1010 Read 1 Done Done Done false
146812024-04-17T17:19:46.757Z 12 Acked 1011 Read 1 Sent Done Done false
146822024-04-17T17:19:46.757Z 13 Acked 1012 Write 7 Sent Done Done false
146832024-04-17T17:19:46.757Z 14 Acked 1013 Flush 0 Sent Done Done false
146842024-04-17T17:19:46.757Z 15 Acked 1014 Write 2 Sent Done Done false
146852024-04-17T17:19:46.757Z 16 Acked 1015 Read 9 Sent Done Done false
146862024-04-17T17:19:46.757Z 17 Acked 1016 Write 6 Sent Done Done false
146872024-04-17T17:19:46.757Z 18 Acked 1017 Read 1 Sent Done Done false
146882024-04-17T17:19:46.757Z 19 Acked 1018 Read 5 Sent Done Done false
146892024-04-17T17:19:46.757Z 20 Acked 1019 Read 10 Sent Done Done false
146902024-04-17T17:19:46.757Z 21 Acked 1020 Flush 0 Sent Done Done false
146912024-04-17T17:19:46.757Z 22 Acked 1021 Write 10 Sent Done Done false
146922024-04-17T17:19:46.757Z 23 Acked 1022 Read 10 Sent Done Done false
146932024-04-17T17:19:46.757Z 24 Acked 1023 Read 1 Sent Done Done false
146942024-04-17T17:19:46.758Z 25 Acked 1024 Read 1 Sent Done Done false
146952024-04-17T17:19:46.758Z 26 Acked 1025 Flush 0 Sent Done Done false
146962024-04-17T17:19:46.758Z 27 Acked 1026 Write 2 Sent Done Done false
146972024-04-17T17:19:46.758Z 28 Acked 1027 Write 2 Sent Done Done false
146982024-04-17T17:19:46.758Z 29 Acked 1028 Read 7 Sent Done Done false
146992024-04-17T17:19:46.758Z 30 Acked 1029 Read 8 Sent Done Done false
147002024-04-17T17:19:46.758Z STATES DS:0 DS:1 DS:2 TOTAL
147012024-04-17T17:19:46.758Z New 0 0 0 0
147022024-04-17T17:19:46.758Z Sent 19 0 0 19
147032024-04-17T17:19:46.758Z Done 3 22 22 47
147042024-04-17T17:19:46.758Z Skipped 0 0 0 0
147052024-04-17T17:19:46.758Z Error 0 0 0 0
147062024-04-17T17:19:46.758ZLast Flush: 1007 1025 1025
147072024-04-17T17:19:46.758ZDownstairs last five completed: 1007 1006 1005 1004 1003
147082024-04-17T17:19:46.758ZUpstairs last five completed: 30 29 28 27 26
147092024-04-17T17:19:46.758Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:19:45.365085852Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1715}
147102024-04-17T17:19:46.758ZWrote out file "/tmp/test_repair_verify.data"
147112024-04-17T17:19:46.763Z
147122024-04-17T17:19:46.769ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
147132024-04-17T17:19:46.779ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
147142024-04-17T17:19:46.779Z 0 000-019 134 134 134 241 241 241 F F F
147152024-04-17T17:19:46.779Z 1 020-039 134 134 134 237 237 237 F F F
147162024-04-17T17:19:46.779Z 2 040-059 134 136 136 237 247 247 F F F <---
147172024-04-17T17:19:46.779Z 3 060-079 134 136 136 241 247 247 F F F <---
147182024-04-17T17:19:46.779Z 4 080-099 134 136 136 241 245 245 T F F <---
147192024-04-17T17:19:46.783Z 5 100-119 133 133 133 236 236 236 F F F
147202024-04-17T17:19:46.783Z 6 120-139 128 128 128 227 227 227 F F F
147212024-04-17T17:19:46.783Z 7 140-159 133 133 133 236 236 236 F F F
147222024-04-17T17:19:46.783Z 8 160-179 136 136 136 243 243 243 F F F
147232024-04-17T17:19:46.783Z 9 180-199 134 134 134 237 237 237 F F F
147242024-04-17T17:19:46.783Z 10 200-219 128 128 128 227 227 227 F F F
147252024-04-17T17:19:46.783Z 11 220-239 126 126 126 223 223 223 F F F
147262024-04-17T17:19:46.783Z 12 240-259 133 133 133 236 236 236 F F F
147272024-04-17T17:19:46.783Z 13 260-279 132 132 132 235 235 235 F F F
147282024-04-17T17:19:46.784Z 14 280-299 132 132 132 233 233 233 F F F
147292024-04-17T17:19:46.784Z 15 300-319 136 136 136 243 243 243 F F F
147302024-04-17T17:19:46.784Z 16 320-339 132 132 132 235 235 235 F F F
147312024-04-17T17:19:46.784Z 17 340-359 136 136 136 244 246 246 F F F <---
147322024-04-17T17:19:46.784Z 18 360-379 136 136 136 243 243 243 F F F
147332024-04-17T17:19:46.784Z 19 380-399 134 134 134 240 240 240 F F F
147342024-04-17T17:19:46.784Z 20 400-419 131 131 131 232 232 232 F T T <---
147352024-04-17T17:19:46.784Z 21 420-439 134 134 134 237 237 237 F T T <---
147362024-04-17T17:19:46.784Z 22 440-459 134 134 134 241 241 241 F F F
147372024-04-17T17:19:46.784Z 23 460-479 134 136 136 241 246 246 F F F <---
147382024-04-17T17:19:46.784Z 24 480-499 130 130 130 231 231 231 F F F
147392024-04-17T17:19:46.784Z 25 500-519 130 130 130 230 230 230 F F F
147402024-04-17T17:19:46.784Z 26 520-539 136 136 136 243 243 243 F F F
147412024-04-17T17:19:46.784Z 27 540-559 128 128 128 227 227 227 F F F
147422024-04-17T17:19:46.784Z 28 560-579 134 136 136 241 245 245 F F F <---
147432024-04-17T17:19:46.784Z 29 580-599 134 134 134 241 241 241 F F F
147442024-04-17T17:19:46.784ZMax gen: 136, Max flush: 247
147452024-04-17T17:19:46.784ZError: Difference in extent metadata found!
147462024-04-17T17:19:46.784ZA Difference in extent metadata is expected here
147472024-04-17T17:19:46.784ZOn loop 68
147482024-04-17T17:19:46.784Z
147492024-04-17T17:19:46.788ZVerifying data now
147502024-04-17T17:19:51.179ZNo change after verify
147512024-04-17T17:19:51.179ZLoop: 68 Downstairs dump after verify (and repair):
147522024-04-17T17:19:51.193ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
147532024-04-17T17:19:51.193Z 0 000-019 134 134 134 241 241 241 F F F
147542024-04-17T17:19:51.193Z 1 020-039 134 134 134 237 237 237 F F F
147552024-04-17T17:19:51.194Z 2 040-059 136 136 136 247 247 247 F F F
147562024-04-17T17:19:51.194Z 3 060-079 136 136 136 247 247 247 F F F
147572024-04-17T17:19:51.194Z 4 080-099 136 136 136 245 245 245 F F F
147582024-04-17T17:19:51.194Z 5 100-119 133 133 133 236 236 236 F F F
147592024-04-17T17:19:51.194Z 6 120-139 128 128 128 227 227 227 F F F
147602024-04-17T17:19:51.194Z 7 140-159 133 133 133 236 236 236 F F F
147612024-04-17T17:19:51.198Z 8 160-179 136 136 136 243 243 243 F F F
147622024-04-17T17:19:51.198Z 9 180-199 134 134 134 237 237 237 F F F
147632024-04-17T17:19:51.198Z 10 200-219 128 128 128 227 227 227 F F F
147642024-04-17T17:19:51.198Z 11 220-239 126 126 126 223 223 223 F F F
147652024-04-17T17:19:51.198Z 12 240-259 133 133 133 236 236 236 F F F
147662024-04-17T17:19:51.198Z 13 260-279 132 132 132 235 235 235 F F F
147672024-04-17T17:19:51.198Z 14 280-299 132 132 132 233 233 233 F F F
147682024-04-17T17:19:51.198Z 15 300-319 136 136 136 243 243 243 F F F
147692024-04-17T17:19:51.198Z 16 320-339 132 132 132 235 235 235 F F F
147702024-04-17T17:19:51.198Z 17 340-359 136 136 136 246 246 246 F F F
147712024-04-17T17:19:51.198Z 18 360-379 136 136 136 243 243 243 F F F
147722024-04-17T17:19:51.198Z 19 380-399 134 134 134 240 240 240 F F F
147732024-04-17T17:19:51.198Z 20 400-419 137 137 137 248 248 248 F F F
147742024-04-17T17:19:51.198Z 21 420-439 137 137 137 248 248 248 F F F
147752024-04-17T17:19:51.198Z 22 440-459 134 134 134 241 241 241 F F F
147762024-04-17T17:19:51.198Z 23 460-479 136 136 136 246 246 246 F F F
147772024-04-17T17:19:51.198Z 24 480-499 130 130 130 231 231 231 F F F
147782024-04-17T17:19:51.199Z 25 500-519 130 130 130 230 230 230 F F F
147792024-04-17T17:19:51.199Z 26 520-539 136 136 136 243 243 243 F F F
147802024-04-17T17:19:51.199Z 27 540-559 128 128 128 227 227 227 F F F
147812024-04-17T17:19:51.199Z 28 560-579 136 136 136 245 245 245 F F F
147822024-04-17T17:19:51.199Z 29 580-599 134 134 134 241 241 241 F F F
147832024-04-17T17:19:51.199ZMax gen: 137, Max flush: 248
147842024-04-17T17:19:51.199Z
147852024-04-17T17:19:51.199ZBegin loop 69
147862024-04-17T17:19:51.199ZDownstairs to restart: 0
147872024-04-17T17:19:51.215Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.824397529Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723}
147882024-04-17T17:19:51.215Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.824689147Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723}
147892024-04-17T17:19:51.222Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.824733019Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723}
147902024-04-17T17:19:51.222ZCrucible runtime is spawned
147912024-04-17T17:19:51.222Z{"msg":"Crucible 4fbdffd7-ef22-47ed-b987-e784cb2007f4 has session id: e88abcb6-70de-4b75-b161-d38e0e89a980","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.824784473Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
147922024-04-17T17:19:51.222Z{"msg":"Upstairs opts: Upstairs UUID: 4fbdffd7-ef22-47ed-b987-e784cb2007f4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.824846856Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
147932024-04-17T17:19:51.222Z{"msg":"Crucible stats registered with UUID: 4fbdffd7-ef22-47ed-b987-e784cb2007f4","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.824883328Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
147942024-04-17T17:19:51.222Z{"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.82489882Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723}
147952024-04-17T17:19:51.222Z{"msg":"4fbdffd7-ef22-47ed-b987-e784cb2007f4 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.824961733Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
147962024-04-17T17:19:51.222Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.825025837Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"client":"0","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
147972024-04-17T17:19:51.222Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.825063319Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"client":"0","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
147982024-04-17T17:19:51.222Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.825088561Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"client":"1","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
147992024-04-17T17:19:51.222Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.825112993Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"client":"1","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148002024-04-17T17:19:51.222Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.825136124Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"client":"2","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148012024-04-17T17:19:51.222Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.825159795Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"client":"2","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148022024-04-17T17:19:51.223Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.825181737Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148032024-04-17T17:19:51.223Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.82523086Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148042024-04-17T17:19:51.223Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.825266032Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148052024-04-17T17:19:51.223Z{"msg":"ds_connection connected from Ok(127.0.0.1:46630)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.825358127Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148062024-04-17T17:19:51.223Z{"msg":"ds_connection connected from Ok(127.0.0.1:36289)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.825587961Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148072024-04-17T17:19:51.223Z{"msg":"ds_connection connected from Ok(127.0.0.1:56530)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.825631704Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148082024-04-17T17:19:51.223Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.82623077Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"client":"1","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148092024-04-17T17:19:51.223Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.826267172Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"client":"1","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148102024-04-17T17:19:51.223Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.826379139Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"client":"2","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148112024-04-17T17:19:51.223Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.826425782Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"client":"2","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148122024-04-17T17:19:51.223Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.826485305Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"client":"0","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148132024-04-17T17:19:51.223Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.826517067Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"client":"0","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148142024-04-17T17:19:51.223Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.826751572Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"client":"1","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148152024-04-17T17:19:51.223Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.82689331Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"client":"2","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148162024-04-17T17:19:51.223Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.827226461Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"client":"1","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148172024-04-17T17:19:51.223Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.827266253Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"client":"1","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148182024-04-17T17:19:51.223Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.827364689Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148192024-04-17T17:19:51.223Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.827428933Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"client":"0","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148202024-04-17T17:19:51.223Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.827466295Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"client":"2","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148212024-04-17T17:19:51.224Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.827483756Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"client":"2","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148222024-04-17T17:19:51.224Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.827498277Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148232024-04-17T17:19:51.224Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.82771719Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"client":"0","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148242024-04-17T17:19:51.224Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.827748672Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"client":"0","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148252024-04-17T17:19:51.224Z{"msg":"[0]R flush_numbers[0..12]: [241, 237, 247, 247, 245, 236, 227, 236, 243, 237, 227, 223]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.827764573Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"":"downstairs","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148262024-04-17T17:19:51.224Z{"msg":"[0]R generation[0..12]: [134, 134, 136, 136, 136, 133, 128, 133, 136, 134, 128, 126]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.827779274Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"":"downstairs","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148272024-04-17T17:19:51.224Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.827801005Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"":"downstairs","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148282024-04-17T17:19:51.224Z{"msg":"[1]R flush_numbers[0..12]: [241, 237, 247, 247, 245, 236, 227, 236, 243, 237, 227, 223]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.827815356Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"":"downstairs","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148292024-04-17T17:19:51.224Z{"msg":"[1]R generation[0..12]: [134, 134, 136, 136, 136, 133, 128, 133, 136, 134, 128, 126]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.827835767Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"":"downstairs","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148302024-04-17T17:19:51.224Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.827850188Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"":"downstairs","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148312024-04-17T17:19:51.224Z{"msg":"[2]R flush_numbers[0..12]: [241, 237, 247, 247, 245, 236, 227, 236, 243, 237, 227, 223]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.827870669Z","hostname":"Wait for a query_work_queue command to finish before sending IO
148322024-04-17T17:19:51.224Zw-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"":"downstairs","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148332024-04-17T17:19:51.224Z{"msg":"[2]R generation[0..12]: [134, 134, 136, 136, 136, 133, 128, 133, 136, 134, 128, 126]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.827887741Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"":"downstairs","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148342024-04-17T17:19:51.224Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.827908782Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"":"downstairs","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148352024-04-17T17:19:51.224Z{"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.827921963Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"":"downstairs","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148362024-04-17T17:19:51.224Z{"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.827941754Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"":"downstairs","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148372024-04-17T17:19:51.224Z{"msg":"Next flush: 249","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.827954975Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"":"downstairs","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148382024-04-17T17:19:51.224Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.827974156Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"":"downstairs","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148392024-04-17T17:19:51.224Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.827988167Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148402024-04-17T17:19:51.224Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.828007008Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"client":"0","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148412024-04-17T17:19:51.224Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.828020449Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"client":"1","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148422024-04-17T17:19:51.225Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.82804649Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"client":"2","":"downstairs","session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148432024-04-17T17:19:51.225Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.828060401Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148442024-04-17T17:19:51.225Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.828079012Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148452024-04-17T17:19:51.225Z{"msg":"4fbdffd7-ef22-47ed-b987-e784cb2007f4 is now active with session: e88abcb6-70de-4b75-b161-d38e0e89a980","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.828092253Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148462024-04-17T17:19:51.225Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.828110714Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723,"session_id":"e88abcb6-70de-4b75-b161-d38e0e89a980"}
148472024-04-17T17:19:51.225Z{"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:49.828123635Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723}
148482024-04-17T17:19:51.225ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
148492024-04-17T17:19:51.229ZLoading write count information from file "/tmp/test_repair_verify.data"
148502024-04-17T17:19:51.229ZRun Repair workload
148512024-04-17T17:19:51.229Z01/30 Write block 207 len 4096 data: 19
148522024-04-17T17:19:51.229Z02/30 Read block 252 len 28672
148532024-04-17T17:19:51.229Z03/30 Flush
148542024-04-17T17:19:51.229Z04/30 Write block 338 len 36864 data: 14 14 13 11 13 14 14 15 14
148552024-04-17T17:19:51.229Z05/30 Write block 280 len 12288 data: 8 7 8
148562024-04-17T17:19:51.229Z06/30 Read block 560 len 40960
148572024-04-17T17:19:51.229Z07/30 Read block 32 len 8192
148582024-04-17T17:19:51.230Z08/30 Flush
148592024-04-17T17:19:51.234Z09/30 Write block 131 len 4096 data: 11
148602024-04-17T17:19:51.234Z10/30 Read block 269 len 28672
148612024-04-17T17:19:51.234Z11/30 Read block 51 len 4096
148622024-04-17T17:19:51.234Z12/30 Read block 140 len 24576
148632024-04-17T17:19:51.234Z13/30 Read block 100 len 12288
148642024-04-17T17:19:51.234Z14/30 Read block 541 len 12288
148652024-04-17T17:19:51.234Z15/30 Read block 283 len 20480
148662024-04-17T17:19:51.238Z16/30 Write block 331 len 24576 data: 9 8 7 9 10 13
148672024-04-17T17:19:51.238Z17/30 Read block 538 len 16384
148682024-04-17T17:19:51.238Z18/30 Write block 181 len 36864 data: 9 9 7 7 6 7 11 14 13
148692024-04-17T17:19:51.242Z19/30 Write block 92 len 40960 data: 17 15 13 10 9 8 10 15 15 13
148702024-04-17T17:19:51.242Z20/30 Write block 59 len 28672 data: 14 15 13 12 15 16 13
148712024-04-17T17:19:51.242Z21/30 Flush
148722024-04-17T17:19:51.245Z22/30 Write block 65 len 8192 data: 14 12
148732024-04-17T17:19:51.245Z23/30 Read block 324 len 36864
148742024-04-17T17:19:51.245Z24/30 Write block 252 len 4096 data: 12
148752024-04-17T17:19:51.246Z25/30 Write block 40 len 12288 data: 11 13 12
148762024-04-17T17:19:51.249Z26/30 Write block 258 len 20480 data: 9 10 9 11 11
148772024-04-17T17:19:51.249Z27/30 Write block 286 len 24576 data: 7 7 8 12 13 11
148782024-04-17T17:19:51.249Z28/30 Read block 556 len 36864
148792024-04-17T17:19:51.249Z29/30 Write block 558 len 32768 data: 9 10 11 11 12 12 14 9
148802024-04-17T17:19:51.254Z30/30 Read block 58 len 32768
148812024-04-17T17:19:51.254Z----------------------------------------------------------------
148822024-04-17T17:19:51.255Z Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:30
148832024-04-17T17:19:51.255ZStates: Active Active Active
148842024-04-17T17:19:51.255ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
148852024-04-17T17:19:51.255Z 1 Acked 1000 Write 1 Sent Done Done false
148862024-04-17T17:19:51.255Z 2 Acked 1001 Read 7 Sent Done Done false
148872024-04-17T17:19:51.255Z 3 Acked 1002 Flush 0 Sent Done Done false
148882024-04-17T17:19:51.255Z 4 Acked 1003 Write 9 Sent Done Done false
148892024-04-17T17:19:51.255Z 5 Acked 1004 Write 3 Sent Done Done false
148902024-04-17T17:19:51.255Z 6 Acked 1005 Read 10 Sent Done Done false
148912024-04-17T17:19:51.255Z 7 Acked 1006 Read 2 Sent Done Done false
148922024-04-17T17:19:51.255Z 8 Acked 1007 Flush 0 Sent Done Done false
148932024-04-17T17:19:51.255Z 9 Acked 1008 Write 1 Sent Done Done false
148942024-04-17T17:19:51.255Z 10 Acked 1009 Read 7 Sent Done Done false
148952024-04-17T17:19:51.255Z 11 Acked 1010 Read 1 Sent Done Done false
148962024-04-17T17:19:51.255Z 12 Acked 1011 Read 6 Sent Done Done false
148972024-04-17T17:19:51.255Z 13 Acked 1012 Read 3 Sent Done Done false
148982024-04-17T17:19:51.255Z 14 Acked 1013 Read 3 Sent Done Done false
148992024-04-17T17:19:51.255Z 15 Acked 1014 Read 5 Sent Done Done false
149002024-04-17T17:19:51.255Z 16 Acked 1015 Write 6 Sent Done Done false
149012024-04-17T17:19:51.255Z 17 Acked 1016 Read 4 Sent Done Done false
149022024-04-17T17:19:51.255Z 18 Acked 1017 Write 9 Sent Done Done false
149032024-04-17T17:19:51.255Z 19 Acked 1018 Write 10 Sent Done Done false
149042024-04-17T17:19:51.256Z 20 Acked 1019 Write 7 Sent Done Done false
149052024-04-17T17:19:51.256Z 21 Acked 1020 Flush 0 Sent Done Done false
149062024-04-17T17:19:51.256Z 22 Acked 1021 Write 2 Sent Done Done false
149072024-04-17T17:19:51.256Z 23 Acked 1022 Read 9 Sent Done Done false
149082024-04-17T17:19:51.256Z 24 Acked 1023 Write 1 Sent Done Done false
149092024-04-17T17:19:51.256Z 25 Acked 1024 Write 3 Sent Done Done false
149102024-04-17T17:19:51.256Z 26 Acked 1025 Write 5 Sent Done Done false
149112024-04-17T17:19:51.256Z 27 Acked 1026 Write 6 Sent Done Done false
149122024-04-17T17:19:51.256Z 28 Acked 1027 Read 9 Sent Done Done false
149132024-04-17T17:19:51.256Z 29 Acked 1028 Write 8 Sent Done Done false
149142024-04-17T17:19:51.256Z 30 Acked 1029 Read 8 Sent Done Sent false
149152024-04-17T17:19:51.256Z STATES DS:0 DS:1 DS:2 TOTAL
149162024-04-17T17:19:51.256Z New 0 0 0 0
149172024-04-17T17:19:51.256Z Sent 30 0 1 31
149182024-04-17T17:19:51.256Z Done 0 30 29 59
149192024-04-17T17:19:51.256Z Skipped 0 0 0 0
149202024-04-17T17:19:51.256Z Error 0 0 0 0
149212024-04-17T17:19:51.256ZLast Flush: 0 1020 1020
149222024-04-17T17:19:51.256ZDownstairs last five completed:
149232024-04-17T17:19:51.256ZUpstairs last five completed: 30 29 28 27 26
149242024-04-17T17:19:51.256Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:19:49.860198919Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1723}
149252024-04-17T17:19:51.257ZWrote out file "/tmp/test_repair_verify.data"
149262024-04-17T17:19:51.261Z
149272024-04-17T17:19:51.261ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
149282024-04-17T17:19:51.275ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
149292024-04-17T17:19:51.275Z 0 000-019 134 134 134 241 241 241 F F F
149302024-04-17T17:19:51.280Z 1 020-039 134 134 134 237 237 237 F F F
149312024-04-17T17:19:51.280Z 2 040-059 136 138 138 247 251 251 F T T <---
149322024-04-17T17:19:51.280Z 3 060-079 136 138 138 247 251 251 F T T <---
149332024-04-17T17:19:51.280Z 4 080-099 136 138 138 245 251 251 F F F <---
149342024-04-17T17:19:51.280Z 5 100-119 133 138 138 236 251 251 F F F <---
149352024-04-17T17:19:51.281Z 6 120-139 128 138 138 227 251 251 F F F <---
149362024-04-17T17:19:51.281Z 7 140-159 133 133 133 236 236 236 F F F
149372024-04-17T17:19:51.281Z 8 160-179 136 136 136 243 243 243 F F F
149382024-04-17T17:19:51.281Z 9 180-199 134 138 138 237 251 251 F F F <---
149392024-04-17T17:19:51.281Z 10 200-219 128 138 138 227 249 249 F F F <---
149402024-04-17T17:19:51.281Z 11 220-239 126 126 126 223 223 223 F F F
149412024-04-17T17:19:51.281Z 12 240-259 133 133 133 236 236 236 F T T <---
149422024-04-17T17:19:51.281Z 13 260-279 132 132 132 235 235 235 F T T <---
149432024-04-17T17:19:51.281Z 14 280-299 132 138 138 233 250 250 F T T <---
149442024-04-17T17:19:51.281Z 15 300-319 136 136 136 243 243 243 F F F
149452024-04-17T17:19:51.281Z 16 320-339 132 138 138 235 251 251 F F F <---
149462024-04-17T17:19:51.281Z 17 340-359 136 138 138 246 250 250 F F F <---
149472024-04-17T17:19:51.281Z 18 360-379 136 136 136 243 243 243 F F F
149482024-04-17T17:19:51.281Z 19 380-399 134 134 134 240 240 240 F F F
149492024-04-17T17:19:51.281Z 20 400-419 137 137 137 248 248 248 F F F
149502024-04-17T17:19:51.281Z 21 420-439 137 137 137 248 248 248 F F F
149512024-04-17T17:19:51.281Z 22 440-459 134 134 134 241 241 241 F F F
149522024-04-17T17:19:51.281Z 23 460-479 136 136 136 246 246 246 F F F
149532024-04-17T17:19:51.281Z 24 480-499 130 130 130 231 231 231 F F F
149542024-04-17T17:19:51.281Z 25 500-519 130 130 130 230 230 230 F F F
149552024-04-17T17:19:51.282Z 26 520-539 136 136 136 243 243 243 F F F
149562024-04-17T17:19:51.282Z 27 540-559 128 128 128 227 227 227 F T T <---
149572024-04-17T17:19:51.282Z 28 560-579 136 136 136 245 245 245 F T T <---
149582024-04-17T17:19:51.282Z 29 580-599 134 134 134 241 241 241 F F F
149592024-04-17T17:19:51.282ZMax gen: 138, Max flush: 251
149602024-04-17T17:19:51.282ZError: Difference in extent metadata found!
149612024-04-17T17:19:51.282ZA Difference in extent metadata is expected here
149622024-04-17T17:19:51.282ZOn loop 69
149632024-04-17T17:19:51.282Z
149642024-04-17T17:19:51.286ZVerifying data now
149652024-04-17T17:19:55.919ZNo change after verify
149662024-04-17T17:19:55.919ZLoop: 69 Downstairs dump after verify (and repair):
149672024-04-17T17:19:55.933ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
149682024-04-17T17:19:55.933Z 0 000-019 134 134 134 241 241 241 F F F
149692024-04-17T17:19:55.937Z 1 020-039 134 134 134 237 237 237 F F F
149702024-04-17T17:19:55.937Z 2 040-059 139 139 139 252 252 252 F F F
149712024-04-17T17:19:55.937Z 3 060-079 139 139 139 252 252 252 F F F
149722024-04-17T17:19:55.938Z 4 080-099 138 138 138 251 251 251 F F F
149732024-04-17T17:19:55.938Z 5 100-119 138 138 138 251 251 251 F F F
149742024-04-17T17:19:55.938Z 6 120-139 138 138 138 251 251 251 F F F
149752024-04-17T17:19:55.938Z 7 140-159 133 133 133 236 236 236 F F F
149762024-04-17T17:19:55.938Z 8 160-179 136 136 136 243 243 243 F F F
149772024-04-17T17:19:55.938Z 9 180-199 138 138 138 251 251 251 F F F
149782024-04-17T17:19:55.938Z 10 200-219 138 138 138 249 249 249 F F F
149792024-04-17T17:19:55.938Z 11 220-239 126 126 126 223 223 223 F F F
149802024-04-17T17:19:55.938Z 12 240-259 139 139 139 252 252 252 F F F
149812024-04-17T17:19:55.938Z 13 260-279 139 139 139 252 252 252 F F F
149822024-04-17T17:19:55.938Z 14 280-299 139 139 139 252 252 252 F F F
149832024-04-17T17:19:55.938Z 15 300-319 136 136 136 243 243 243 F F F
149842024-04-17T17:19:55.938Z 16 320-339 138 138 138 251 251 251 F F F
149852024-04-17T17:19:55.938Z 17 340-359 138 138 138 250 250 250 F F F
149862024-04-17T17:19:55.938Z 18 360-379 136 136 136 243 243 243 F F F
149872024-04-17T17:19:55.938Z 19 380-399 134 134 134 240 240 240 F F F
149882024-04-17T17:19:55.938Z 20 400-419 137 137 137 248 248 248 F F F
149892024-04-17T17:19:55.938Z 21 420-439 137 137 137 248 248 248 F F F
149902024-04-17T17:19:55.938Z 22 440-459 134 134 134 241 241 241 F F F
149912024-04-17T17:19:55.938Z 23 460-479 136 136 136 246 246 246 F F F
149922024-04-17T17:19:55.938Z 24 480-499 130 130 130 231 231 231 F F F
149932024-04-17T17:19:55.938Z 25 500-519 130 130 130 230 230 230 F F F
149942024-04-17T17:19:55.938Z 26 520-539 136 136 136 243 243 243 F F F
149952024-04-17T17:19:55.939Z 27 540-559 139 139 139 252 252 252 F F F
149962024-04-17T17:19:55.939Z 28 560-579 139 139 139 252 252 252 F F F
149972024-04-17T17:19:55.939Z 29 580-599 134 134 134 241 241 241 F F F
149982024-04-17T17:19:55.939ZMax gen: 139, Max flush: 252
149992024-04-17T17:19:55.939Z
150002024-04-17T17:19:55.939ZBegin loop 70
150012024-04-17T17:19:55.939ZDownstairs to restart: 0
150022024-04-17T17:19:55.955Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.564122971Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731}
150032024-04-17T17:19:55.961Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.564452921Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731}
150042024-04-17T17:19:55.961ZCrucible runtime is spawned
150052024-04-17T17:19:55.961Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.564500873Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731}
150062024-04-17T17:19:55.961Z{"msg":"Crucible 45e02943-b980-4fce-a4d9-d02e3e7cf545 has session id: 6d6afb32-54a0-46b3-aa45-a31b5929bc30","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.564542976Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150072024-04-17T17:19:55.961Z{"msg":"Upstairs opts: Upstairs UUID: 45e02943-b980-4fce-a4d9-d02e3e7cf545, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.564616191Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150082024-04-17T17:19:55.961Z{"msg":"Crucible stats registered with UUID: 45e02943-b980-4fce-a4d9-d02e3e7cf545","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.564668914Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150092024-04-17T17:19:55.961Z{"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.564712826Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731}
150102024-04-17T17:19:55.961Z{"msg":"45e02943-b980-4fce-a4d9-d02e3e7cf545 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.564748819Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150112024-04-17T17:19:55.961Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.564789351Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"client":"0","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150122024-04-17T17:19:55.961Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.564815543Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"client":"0","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150132024-04-17T17:19:55.961Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.564852665Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"client":"1","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150142024-04-17T17:19:55.961Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.564882317Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"client":"1","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150152024-04-17T17:19:55.961Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.564921059Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"client":"2","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150162024-04-17T17:19:55.961Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.564961881Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"client":"2","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150172024-04-17T17:19:55.961Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.564990783Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"":"io task","client":"0","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150182024-04-17T17:19:55.961Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.565052157Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"":"io task","client":"2","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150192024-04-17T17:19:55.961Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.56509436Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"":"io task","client":"1","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150202024-04-17T17:19:55.962Z{"msg":"ds_connection connected from Ok(127.0.0.1:55687)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.565125782Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"":"io task","client":"2","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150212024-04-17T17:19:55.962Z{"msg":"ds_connection connected from Ok(127.0.0.1:39824)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.565161333Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"":"io task","client":"1","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150222024-04-17T17:19:55.962Z{"msg":"ds_connection connected from Ok(127.0.0.1:56440)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.565187735Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"":"io task","client":"0","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150232024-04-17T17:19:55.962Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.56576014Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"client":"2","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150242024-04-17T17:19:55.962Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.565814833Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"client":"2","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150252024-04-17T17:19:55.962Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.565843425Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"client":"1","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150262024-04-17T17:19:55.962Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.565867756Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"client":"1","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150272024-04-17T17:19:55.962Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.566109991Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"client":"0","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150282024-04-17T17:19:55.962Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.566148904Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"client":"0","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150292024-04-17T17:19:55.962Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.566338545Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"client":"1","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150302024-04-17T17:19:55.962Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.566467573Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"client":"2","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150312024-04-17T17:19:55.962Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.566779212Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"client":"1","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150322024-04-17T17:19:55.962Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.566822154Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"client":"1","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150332024-04-17T17:19:55.962Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.566864707Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150342024-04-17T17:19:55.962Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.566898099Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"client":"0","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150352024-04-17T17:19:55.962Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.5669176Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"client":"2","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150362024-04-17T17:19:55.962Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.566948552Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"client":"2","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150372024-04-17T17:19:55.962Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.566967393Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150382024-04-17T17:19:55.962Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.56725628Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"client":"0","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150392024-04-17T17:19:55.962Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.567289862Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"client":"0","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150402024-04-17T17:19:55.962Z{"msg":"[0]R flush_numbers[0..12]: [241, 237, 252, 252, 251, 251, 251, 236, 243, 251, 249, 223]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.567306503Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"":"downstairs","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150412024-04-17T17:19:55.962Z{"msg":"[0]R generation[0..12]: [134, 134, 139, 139, 138, 138, 138, 133, 136, 138, 138, 126]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.567339786Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"":"downstairs","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150422024-04-17T17:19:55.963Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.567371118Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"":"downstairs","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150432024-04-17T17:19:55.963Z{"msg":"[1]R flush_numbers[0..12]: [241, 237, 252, 252, 251, 251, 251, 236, 243, 251, 249, 223]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
150442024-04-17T17:19:55.963Z,"time":"2024-04-17T17:19:54.567394539Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"":"downstairs","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150452024-04-17T17:19:55.963Z{"msg":"[1]R generation[0..12]: [134, 134, 139, 139, 138, 138, 138, 133, 136, 138, 138, 126]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.56741525Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"":"downstairs","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150462024-04-17T17:19:55.963Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.567446842Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"":"downstairs","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150472024-04-17T17:19:55.963Z{"msg":"[2]R flush_numbers[0..12]: [241, 237, 252, 252, 251, 251, 251, 236, 243, 251, 249, 223]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.567463383Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"":"downstairs","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150482024-04-17T17:19:55.963Z{"msg":"[2]R generation[0..12]: [134, 134, 139, 139, 138, 138, 138, 133, 136, 138, 138, 126]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.567493115Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"":"downstairs","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150492024-04-17T17:19:55.963Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.567510696Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"":"downstairs","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150502024-04-17T17:19:55.963Z{"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.567540278Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"":"downstairs","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150512024-04-17T17:19:55.963Z{"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.567555399Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"":"downstairs","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150522024-04-17T17:19:55.963Z{"msg":"Next flush: 253","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.567585131Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"":"downstairs","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150532024-04-17T17:19:55.963Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.567600031Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"":"downstairs","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150542024-04-17T17:19:55.963Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.567630213Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150552024-04-17T17:19:55.963Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.567643964Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"client":"0","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150562024-04-17T17:19:55.963Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
150572024-04-17T17:19:55.963Z,"time":"2024-04-17T17:19:54.567674446Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"client":"1","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150582024-04-17T17:19:55.963Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.567695717Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"client":"2","":"downstairs","session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150592024-04-17T17:19:55.963Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.567725399Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150602024-04-17T17:19:55.963Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.56773913Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150612024-04-17T17:19:55.963Z{"msg":"45e02943-b980-4fce-a4d9-d02e3e7cf545 is now active with session: 6d6afb32-54a0-46b3-aa45-a31b5929bc30","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.567757181Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150622024-04-17T17:19:55.963Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.567783772Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731,"session_id":"6d6afb32-54a0-46b3-aa45-a31b5929bc30"}
150632024-04-17T17:19:55.963Z{"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:19:54.567796983Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731}
150642024-04-17T17:19:55.967ZLoading write count information from file "/tmp/test_repair_verify.data"
150652024-04-17T17:19:55.967ZRun Repair workload
150662024-04-17T17:19:55.967Z01/30 Write block 299 len 28672 data: 8 8 10 10 13 14 14
150672024-04-17T17:19:55.967Z02/30 Read block 273 len 28672
150682024-04-17T17:19:55.967Z03/30 Write block 595 len 20480 data: 6 5 6 5 5
150692024-04-17T17:19:55.967Z04/30 Write block 326 len 16384 data: 9 8 8 8
150702024-04-17T17:19:55.967Z05/30 Flush
150712024-04-17T17:19:55.967Z06/30 Write block 79 len 40960 data: 9 12 10 12 15 12 14 11 15 13
150722024-04-17T17:19:55.971Z07/30 Read block 292 len 16384
150732024-04-17T17:19:55.971Z08/30 Read block 500 len 12288
150742024-04-17T17:19:55.971Z09/30 Flush
150752024-04-17T17:19:55.974Z10/30 Write block 271 len 16384 data: 13 12 14 12
150762024-04-17T17:19:55.975Z11/30 Write block 276 len 36864 data: 12 6 8 8 9 8 9 8 9
150772024-04-17T17:19:55.975Z12/30 Read block 562 len 32768
150782024-04-17T17:19:55.975Z13/30 Read block 421 len 36864
150792024-04-17T17:19:55.978Z14/30 Read block 273 len 4096
150802024-04-17T17:19:55.978Z15/30 Write block 143 len 24576 data: 8 9 12 12 10 9
150812024-04-17T17:19:55.978Z16/30 Read block 235 len 32768
150822024-04-17T17:19:55.978Z17/30 Read block 518 len 16384
150832024-04-17T17:19:55.978Z18/30 Flush
150842024-04-17T17:19:55.982Z19/30 Write block 382 len 4096 data: 10
150852024-04-17T17:19:55.985Z20/30 Write block 32 len 36864 data: 18 18 21 18 15 13 11 11 12
150862024-04-17T17:19:55.985Z21/30 Write block 187 len 24576 data: 12 15 14 11 9 9
150872024-04-17T17:19:55.985Z22/30 Read block 254 len 36864
150882024-04-17T17:19:55.988Z23/30 Write block 301 len 40960 data: 11 11 14 15 15 12 11 13 12 12
150892024-04-17T17:19:55.988Z24/30 Read block 354 len 16384
150902024-04-17T17:19:55.989Z25/30 Read block 511 len 40960
150912024-04-17T17:19:55.992Z26/30 Write block 386 len 36864 data: 11 13 11 12 11 12 13 11 12
150922024-04-17T17:19:55.992Z27/30 Read block 233 len 32768
150932024-04-17T17:19:55.992Z28/30 Write block 592 len 20480 data: 9 7 6 7 6
150942024-04-17T17:19:55.992Z29/30 Write block 520 len 20480 data: 7 8 9 9 7
150952024-04-17T17:19:55.997Z30/30 Write block 370 len 4096 data: 11
150962024-04-17T17:19:55.997Z----------------------------------------------------------------
150972024-04-17T17:19:55.997Z Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30
150982024-04-17T17:19:55.997ZStates: Active Active Active
150992024-04-17T17:19:55.997ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
151002024-04-17T17:19:55.997Z 1 Acked 1000 Write 7 Done Done Done false
151012024-04-17T17:19:55.997Z 2 Acked 1001 Read 7 Done Done Done false
151022024-04-17T17:19:55.997Z 3 Acked 1002 Write 5 Done Done Done false
151032024-04-17T17:19:55.997Z 4 Acked 1003 Write 4 Done Done Done false
151042024-04-17T17:19:55.997Z 5 Acked 1004 Flush 0 Sent Done Done false
151052024-04-17T17:19:55.997Z 6 Acked 1005 Write 10 Sent Done Done false
151062024-04-17T17:19:55.997Z 7 Acked 1006 Read 4 Sent Done Done false
151072024-04-17T17:19:55.997Z 8 Acked 1007 Read 3 Sent Done Done false
151082024-04-17T17:19:55.997Z 9 Acked 1008 Flush 0 Sent Done Done false
151092024-04-17T17:19:55.997Z 10 Acked 1009 Write 4 Sent Done Done false
151102024-04-17T17:19:55.997Z 11 Acked 1010 Write 9 Sent Done Done false
151112024-04-17T17:19:55.997Z 12 Acked 1011 Read 8 Sent Done Done false
151122024-04-17T17:19:55.997Z 13 Acked 1012 Read 9 Sent Done Done false
151132024-04-17T17:19:55.997Z 14 Acked 1013 Read 1 Sent Done Done false
151142024-04-17T17:19:55.997Z 15 Acked 1014 Write 6 Sent Done Done false
151152024-04-17T17:19:55.998Z 16 Acked 1015 Read 8 Sent Done Done false
151162024-04-17T17:19:55.998Z 17 Acked 1016 Read 4 Sent Done Done false
151172024-04-17T17:19:55.998Z 18 Acked 1017 Flush 0 Sent Done Done false
151182024-04-17T17:19:55.998Z 19 Acked 1018 Write 1 Sent Done Done false
151192024-04-17T17:19:55.998Z 20 Acked 1019 Write 9 Sent Done Done false
151202024-04-17T17:19:55.998Z 21 Acked 1020 Write 6 Sent Done Done false
151212024-04-17T17:19:55.998Z 22 Acked 1021 Read 9 Sent Done Done false
151222024-04-17T17:19:55.998Z 23 Acked 1022 Write 10 Sent Done Done false
151232024-04-17T17:19:55.998Z 24 Acked 1023 Read 4 Sent Done Done false
151242024-04-17T17:19:55.998Z 25 Acked 1024 Read 10 Sent Done Done false
151252024-04-17T17:19:55.998Z 26 Acked 1025 Write 9 Sent Done Done false
151262024-04-17T17:19:55.998Z 27 Acked 1026 Read 8 Sent Done Done false
151272024-04-17T17:19:55.998Z 28 Acked 1027 Write 5 Sent Done Done false
151282024-04-17T17:19:55.998Z 29 Acked 1028 Write 5 Sent Done Done false
151292024-04-17T17:19:55.998Z 30 Acked 1029 Write 1 Sent Sent Sent false
151302024-04-17T17:19:55.998Z STATES DS:0 DS:1 DS:2 TOTAL
151312024-04-17T17:19:55.998Z New 0 0 0 0
151322024-04-17T17:19:55.998Z Sent 26 1 1 28
151332024-04-17T17:19:55.998Z Done 4 29 29 62
151342024-04-17T17:19:55.998Z Skipped 0 0 0 0
151352024-04-17T17:19:55.998Z Error 0 0 0 0
151362024-04-17T17:19:55.998ZLast Flush: 0 1017 1017
151372024-04-17T17:19:55.999ZDownstairs last five completed:
151382024-04-17T17:19:55.999ZUpstairs last five completed: 30 29 28 27 26
151392024-04-17T17:19:55.999Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:19:54.602545799Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1731}
151402024-04-17T17:19:55.999ZWrote out file "/tmp/test_repair_verify.data"
151412024-04-17T17:19:56.002Z
151422024-04-17T17:19:56.002ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
151432024-04-17T17:19:56.019ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
151442024-04-17T17:19:56.019Z 0 000-019 134 134 134 241 241 241 F F F
151452024-04-17T17:19:56.019Z 1 020-039 134 134 134 237 237 237 F T T <---
151462024-04-17T17:19:56.019Z 2 040-059 139 139 139 252 252 252 F T T <---
151472024-04-17T17:19:56.019Z 3 060-079 139 140 140 252 254 254 F F F <---
151482024-04-17T17:19:56.020Z 4 080-099 138 140 140 251 254 254 F F F <---
151492024-04-17T17:19:56.024Z 5 100-119 138 138 138 251 251 251 F F F
151502024-04-17T17:19:56.025Z 6 120-139 138 138 138 251 251 251 F F F
151512024-04-17T17:19:56.025Z 7 140-159 133 140 140 236 255 255 F F F <---
151522024-04-17T17:19:56.025Z 8 160-179 136 136 136 243 243 243 F F F
151532024-04-17T17:19:56.025Z 9 180-199 138 138 138 251 251 251 F T T <---
151542024-04-17T17:19:56.025Z 10 200-219 138 138 138 249 249 249 F F F
151552024-04-17T17:19:56.025Z 11 220-239 126 126 126 223 223 223 F F F
151562024-04-17T17:19:56.025Z 12 240-259 139 139 139 252 252 252 F F F
151572024-04-17T17:19:56.025Z 13 260-279 139 140 140 252 255 255 F F F <---
151582024-04-17T17:19:56.025Z 14 280-299 139 140 140 252 255 255 T F F <---
151592024-04-17T17:19:56.025Z 15 300-319 136 140 140 243 253 253 T T T <---
151602024-04-17T17:19:56.025Z 16 320-339 138 140 140 251 253 253 T F F <---
151612024-04-17T17:19:56.025Z 17 340-359 138 138 138 250 250 250 F F F
151622024-04-17T17:19:56.025Z 18 360-379 136 136 136 243 243 243 F T T <---
151632024-04-17T17:19:56.025Z 19 380-399 134 134 134 240 240 240 F T T <---
151642024-04-17T17:19:56.026Z 20 400-419 137 137 137 248 248 248 F F F
151652024-04-17T17:19:56.026Z 21 420-439 137 137 137 248 248 248 F F F
151662024-04-17T17:19:56.026Z 22 440-459 134 134 134 241 241 241 F F F
151672024-04-17T17:19:56.026Z 23 460-479 136 136 136 246 246 246 F F F
151682024-04-17T17:19:56.026Z 24 480-499 130 130 130 231 231 231 F F F
151692024-04-17T17:19:56.026Z 25 500-519 130 130 130 230 230 230 F F F
151702024-04-17T17:19:56.026Z 26 520-539 136 136 136 243 243 243 F T T <---
151712024-04-17T17:19:56.026Z 27 540-559 139 139 139 252 252 252 F F F
151722024-04-17T17:19:56.026Z 28 560-579 139 139 139 252 252 252 F F F
151732024-04-17T17:19:56.026Z 29 580-599 134 140 140 241 253 253 T T T <---
151742024-04-17T17:19:56.026ZMax gen: 140, Max flush: 255
151752024-04-17T17:19:56.026ZError: Difference in extent metadata found!
151762024-04-17T17:19:56.026ZA Difference in extent metadata is expected here
151772024-04-17T17:19:56.026ZOn loop 70
151782024-04-17T17:19:56.027Z
151792024-04-17T17:19:56.032ZVerifying data now
151802024-04-17T17:20:10.696ZNo change after verify
151812024-04-17T17:20:10.696ZLoop: 70 Downstairs dump after verify (and repair):
151822024-04-17T17:20:10.710ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
151832024-04-17T17:20:10.711Z 0 000-019 134 134 134 241 241 241 F F F
151842024-04-17T17:20:10.715Z 1 020-039 141 141 141 256 256 256 F F F
151852024-04-17T17:20:10.715Z 2 040-059 141 141 141 256 256 256 F F F
151862024-04-17T17:20:10.715Z 3 060-079 140 140 140 254 254 254 F F F
151872024-04-17T17:20:10.715Z 4 080-099 140 140 140 254 254 254 F F F
151882024-04-17T17:20:10.715Z 5 100-119 138 138 138 251 251 251 F F F
151892024-04-17T17:20:10.715Z 6 120-139 138 138 138 251 251 251 F F F
151902024-04-17T17:20:10.715Z 7 140-159 140 140 140 255 255 255 F F F
151912024-04-17T17:20:10.715Z 8 160-179 136 136 136 243 243 243 F F F
151922024-04-17T17:20:10.715Z 9 180-199 141 141 141 256 256 256 F F F
151932024-04-17T17:20:10.715Z 10 200-219 138 138 138 249 249 249 F F F
151942024-04-17T17:20:10.715Z 11 220-239 126 126 126 223 223 223 F F F
151952024-04-17T17:20:10.715Z 12 240-259 139 139 139 252 252 252 F F F
151962024-04-17T17:20:10.716Z 13 260-279 140 140 140 255 255 255 F F F
151972024-04-17T17:20:10.716Z 14 280-299 140 140 140 255 255 255 F F F
151982024-04-17T17:20:10.716Z 15 300-319 141 141 141 256 256 256 F F F
151992024-04-17T17:20:10.716Z 16 320-339 140 140 140 253 253 253 F F F
152002024-04-17T17:20:10.716Z 17 340-359 138 138 138 250 250 250 F F F
152012024-04-17T17:20:10.716Z 18 360-379 141 141 141 256 256 256 F F F
152022024-04-17T17:20:10.716Z 19 380-399 141 141 141 256 256 256 F F F
152032024-04-17T17:20:10.716Z 20 400-419 137 137 137 248 248 248 F F F
152042024-04-17T17:20:10.716Z 21 420-439 137 137 137 248 248 248 F F F
152052024-04-17T17:20:10.716Z 22 440-459 134 134 134 241 241 241 F F F
152062024-04-17T17:20:10.716Z 23 460-479 136 136 136 246 246 246 F F F
152072024-04-17T17:20:10.716Z 24 480-499 130 130 130 231 231 231 F F F
152082024-04-17T17:20:10.716Z 25 500-519 130 130 130 230 230 230 F F F
152092024-04-17T17:20:10.716Z 26 520-539 141 141 141 256 256 256 F F F
152102024-04-17T17:20:10.716Z 27 540-559 139 139 139 252 252 252 F F F
152112024-04-17T17:20:10.716Z 28 560-579 139 139 139 252 252 252 F F F
152122024-04-17T17:20:10.716Z 29 580-599 141 141 141 256 256 256 F F F
152132024-04-17T17:20:10.716ZMax gen: 141, Max flush: 256
152142024-04-17T17:20:10.716Z
152152024-04-17T17:20:10.716ZBegin loop 71
152162024-04-17T17:20:10.716ZDownstairs to restart: 0
152172024-04-17T17:20:10.733Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.342372599Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739}
152182024-04-17T17:20:10.739Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.342658536Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739}
152192024-04-17T17:20:10.739ZCrucible runtime is spawned
152202024-04-17T17:20:10.739Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.34271286Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739}
152212024-04-17T17:20:10.740Z{"msg":"Crucible 462a978a-3e56-4d2d-ad64-dfb59516192b has session id: 979097fc-81e9-4106-b3c0-99de72f6e3f4","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.342741571Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152222024-04-17T17:20:10.740Z{"msg":"Upstairs opts: Upstairs UUID: 462a978a-3e56-4d2d-ad64-dfb59516192b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.342815956Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152232024-04-17T17:20:10.740Z{"msg":"Crucible stats registered with UUID: 462a978a-3e56-4d2d-ad64-dfb59516192b","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.342862949Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152242024-04-17T17:20:10.740Z{"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.34288125Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739}
152252024-04-17T17:20:10.740Z{"msg":"462a978a-3e56-4d2d-ad64-dfb59516192b active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.342897081Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152262024-04-17T17:20:10.740Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.342912482Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"client":"0","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152272024-04-17T17:20:10.740Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.342940253Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"client":"0","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152282024-04-17T17:20:10.740Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.342957664Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"client":"1","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152292024-04-17T17:20:10.740Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.342974415Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"client":"1","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152302024-04-17T17:20:10.740Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.342991406Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"client":"2","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152312024-04-17T17:20:10.740Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.343019438Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"client":"2","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152322024-04-17T17:20:10.740Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.34304834Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152332024-04-17T17:20:10.740Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.343102573Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152342024-04-17T17:20:10.740Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.343144626Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152352024-04-17T17:20:10.740Z{"msg":"ds_connection connected from Ok(127.0.0.1:62025)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.34339035Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152362024-04-17T17:20:10.740Z{"msg":"ds_connection connected from Ok(127.0.0.1:57425)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.343449344Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152372024-04-17T17:20:10.740Z{"msg":"ds_connection connected from Ok(127.0.0.1:56702)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.343479926Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152382024-04-17T17:20:10.740Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.34420645Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"client":"1","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152392024-04-17T17:20:10.740Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.344242722Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"client":"1","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152402024-04-17T17:20:10.740Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.344260013Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"client":"0","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152412024-04-17T17:20:10.740Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.344275324Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"client":"0","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152422024-04-17T17:20:10.740Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.344307856Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"client":"2","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152432024-04-17T17:20:10.741Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.344329787Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"client":"2","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152442024-04-17T17:20:10.741Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.344806296Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"client":"2","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152452024-04-17T17:20:10.741Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.344842969Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"client":"1","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152462024-04-17T17:20:10.741Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.344907822Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"client":"0","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152472024-04-17T17:20:10.741Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.345329158Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"client":"2","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152482024-04-17T17:20:10.741Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.345383131Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"client":"2","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152492024-04-17T17:20:10.741Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.345435794Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152502024-04-17T17:20:10.741Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.345478227Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"client":"1","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152512024-04-17T17:20:10.741Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.345511639Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"client":"1","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152522024-04-17T17:20:10.741Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.345532871Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152532024-04-17T17:20:10.741Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.345562302Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"client":"0","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152542024-04-17T17:20:10.741Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.345587374Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"client":"0","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152552024-04-17T17:20:10.741Z{"msg":"[0]R flush_numbers[0..12]: [241, 256, 256, 254, 254, 251, 251, 255, 243, 256, 249, 223]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.345619876Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"":"downstairs","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152562024-04-17T17:20:10.741Z{"msg":"[0]R generation[0..12]: [134, 141, 141, 140, 140, 138, 138, 140, 136, 141, 138, 126]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.345645457Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"":"downstairs","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152572024-04-17T17:20:10.741ZWait for a query_work_queue command to finish before sending IO
152582024-04-17T17:20:10.741Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.345679629Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"":"downstairs","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152592024-04-17T17:20:10.741Z{"msg":"[1]R flush_numbers[0..12]: [241, 256, 256, 254, 254, 251, 251, 255, 243, 256, 249, 223]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.345705571Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"":"downstairs","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152602024-04-17T17:20:10.741Z{"msg":"[1]R generation[0..12]: [134, 141, 141, 140, 140, 138, 138, 140, 136, 141, 138, 126]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.345737993Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"":"downstairs","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152612024-04-17T17:20:10.741Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.345763905Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"":"downstairs","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152622024-04-17T17:20:10.741Z{"msg":"[2]R flush_numbers[0..12]: [241, 256, 256, 254, 254, 251, 251, 255, 243, 256, 249, 223]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.345792486Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"":"downstairs","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152632024-04-17T17:20:10.741Z{"msg":"[2]R generation[0..12]: [134, 141, 141, 140, 140, 138, 138, 140, 136, 141, 138, 126]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.345816438Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"":"downstairs","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152642024-04-17T17:20:10.741Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.345844169Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"":"downstairs","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152652024-04-17T17:20:10.741Z{"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.345868121Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"":"downstairs","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152662024-04-17T17:20:10.742Z{"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.345897682Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"":"downstairs","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152672024-04-17T17:20:10.742Z{"msg":"Next flush: 257","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.345920814Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"":"downstairs","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152682024-04-17T17:20:10.742Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.345951336Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"":"downstairs","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152692024-04-17T17:20:10.742Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.345974817Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152702024-04-17T17:20:10.742Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.345999269Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"client":"0","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152712024-04-17T17:20:10.742Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.34602271Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"client":"1","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152722024-04-17T17:20:10.742ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
152732024-04-17T17:20:10.742Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.346056812Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"client":"2","":"downstairs","session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152742024-04-17T17:20:10.742Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.346087804Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152752024-04-17T17:20:10.742Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.346108895Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152762024-04-17T17:20:10.742Z{"msg":"462a978a-3e56-4d2d-ad64-dfb59516192b is now active with session: 979097fc-81e9-4106-b3c0-99de72f6e3f4","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.346137397Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152772024-04-17T17:20:10.742Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.346158678Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739,"session_id":"979097fc-81e9-4106-b3c0-99de72f6e3f4"}
152782024-04-17T17:20:10.742Z{"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:09.3461839Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739}
152792024-04-17T17:20:10.742ZLoading write count information from file "/tmp/test_repair_verify.data"
152802024-04-17T17:20:10.742ZRun Repair workload
152812024-04-17T17:20:10.746Z01/30 Write block 267 len 28672 data: 8 9 12 11 14 13 15
152822024-04-17T17:20:10.746Z02/30 Write block 216 len 40960 data: 12 9 10 9 11 10 9 11 12 11
152832024-04-17T17:20:10.746Z03/30 Read block 497 len 36864
152842024-04-17T17:20:10.746Z04/30 Write block 265 len 20480 data: 8 8 9 10 13
152852024-04-17T17:20:10.746Z05/30 Read block 359 len 4096
152862024-04-17T17:20:10.746Z06/30 Write block 162 len 12288 data: 12 14 12
152872024-04-17T17:20:10.746Z07/30 Read block 385 len 36864
152882024-04-17T17:20:10.746Z08/30 Flush
152892024-04-17T17:20:10.752Z09/30 Write block 96 len 12288 data: 10 9 11
152902024-04-17T17:20:10.752Z10/30 Read block 92 len 40960
152912024-04-17T17:20:10.752Z11/30 Read block 13 len 8192
152922024-04-17T17:20:10.752Z12/30 Read block 546 len 8192
152932024-04-17T17:20:11.167Z13/30 Flush
152942024-04-17T17:20:11.167Z14/30 Write block 559 len 4096 data: 11
152952024-04-17T17:20:11.167Z15/30 Write block 137 len 32768 data: 10 9 9 8 7 7 9 10
152962024-04-17T17:20:11.167Z16/30 Write block 166 len 40960 data: 13 14 12 10 9 10 9 10 7 10
152972024-04-17T17:20:11.167Z17/30 Write block 182 len 40960 data: 10 8 8 7 8 13 16 15 12 10
152982024-04-17T17:20:11.167Z18/30 Read block 549 len 4096
152992024-04-17T17:20:11.176Z19/30 Write block 264 len 24576 data: 10 9 9 10 11 14
153002024-04-17T17:20:11.176Z20/30 Flush
153012024-04-17T17:20:11.176Z21/30 Write block 65 len 24576 data: 15 13 13 12 10 10
153022024-04-17T17:20:11.176Z22/30 Read block 543 len 20480
153032024-04-17T17:20:11.176Z23/30 Flush
153042024-04-17T17:20:11.176Z24/30 Write block 181 len 20480 data: 10 11 9 9 8
153052024-04-17T17:20:11.176Z25/30 Write block 461 len 20480 data: 10 9 11 12 15
153062024-04-17T17:20:11.177Z26/30 Write block 201 len 24576 data: 13 16 17 18 17 16
153072024-04-17T17:20:11.177Z27/30 Read block 445 len 12288
153082024-04-17T17:20:11.177Z28/30 Write block 509 len 36864 data: 9 10 9 11 7 11 9 7 7
153092024-04-17T17:20:11.177Z29/30 Read block 276 len 12288
153102024-04-17T17:20:11.177Z30/30 Read block 505 len 24576
153112024-04-17T17:20:11.177Z----------------------------------------------------------------
153122024-04-17T17:20:11.177Z Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:22
153132024-04-17T17:20:11.177ZStates: Active Active Active
153142024-04-17T17:20:11.177ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
153152024-04-17T17:20:11.177Z 9 Acked 1008 Write 3 Sent Done Done false
153162024-04-17T17:20:11.177Z 10 Acked 1009 Read 10 Sent Done Done false
153172024-04-17T17:20:11.177Z 11 Acked 1010 Read 2 Sent Done Done false
153182024-04-17T17:20:11.177Z 12 Acked 1011 Read 2 Sent Done Done false
153192024-04-17T17:20:11.177Z 13 Acked 1012 Flush 0 Sent Done Done false
153202024-04-17T17:20:11.177Z 14 Acked 1013 Write 1 Sent Done Done false
153212024-04-17T17:20:11.177Z 15 Acked 1014 Write 8 Sent Done Done false
153222024-04-17T17:20:11.177Z 16 Acked 1015 Write 10 Sent Done Done false
153232024-04-17T17:20:11.177Z 17 Acked 1016 Write 10 Sent Done Done false
153242024-04-17T17:20:11.177Z 18 Acked 1017 Read 1 Sent Done Done false
153252024-04-17T17:20:11.177Z 19 Acked 1018 Write 6 Sent Done Done false
153262024-04-17T17:20:11.177Z 20 Acked 1019 Flush 0 Sent Done Done false
153272024-04-17T17:20:11.177Z 21 Acked 1020 Write 6 Sent Done Done false
153282024-04-17T17:20:11.178Z 22 Acked 1021 Read 5 Sent Done Done false
153292024-04-17T17:20:11.178Z 23 Acked 1022 Flush 0 Sent Done Done false
153302024-04-17T17:20:11.178Z 24 Acked 1023 Write 5 Sent Done Done false
153312024-04-17T17:20:11.178Z 25 Acked 1024 Write 5 Sent Done Done false
153322024-04-17T17:20:11.178Z 26 Acked 1025 Write 6 Sent Done Done false
153332024-04-17T17:20:11.178Z 27 Acked 1026 Read 3 Sent Done Done false
153342024-04-17T17:20:11.178Z 28 Acked 1027 Write 9 Sent Done Done false
153352024-04-17T17:20:11.178Z 29 Acked 1028 Read 3 Sent Done Done false
153362024-04-17T17:20:11.178Z 30 Acked 1029 Read 6 Sent Sent Done false
153372024-04-17T17:20:11.178Z STATES DS:0 DS:1 DS:2 TOTAL
153382024-04-17T17:20:11.178Z New 0 0 0 0
153392024-04-17T17:20:11.178Z Sent 22 1 0 23
153402024-04-17T17:20:11.178Z Done 0 21 22 43
153412024-04-17T17:20:11.178Z Skipped 0 0 0 0
153422024-04-17T17:20:11.178Z Error 0 0 0 0
153432024-04-17T17:20:11.178ZLast Flush: 1007 1022 1022
153442024-04-17T17:20:11.178ZDownstairs last five completed: 1007 1006 1005 1004 1003
153452024-04-17T17:20:11.178ZUpstairs last five completed: 30 29 28 27 26
153462024-04-17T17:20:11.178Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:20:09.382347932Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1739}
153472024-04-17T17:20:11.178ZWrote out file "/tmp/test_repair_verify.data"
153482024-04-17T17:20:11.178Z
153492024-04-17T17:20:11.179ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
153502024-04-17T17:20:11.179ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
153512024-04-17T17:20:11.179Z 0 000-019 134 134 134 241 241 241 F F F
153522024-04-17T17:20:11.179Z 1 020-039 141 141 141 256 256 256 F F F
153532024-04-17T17:20:11.179Z 2 040-059 141 141 141 256 256 256 F F F
153542024-04-17T17:20:11.179Z 3 060-079 140 142 142 254 260 260 F F F <---
153552024-04-17T17:20:11.179Z 4 080-099 140 142 142 254 258 258 F F F <---
153562024-04-17T17:20:11.179Z 5 100-119 138 138 138 251 251 251 F F F
153572024-04-17T17:20:11.179Z 6 120-139 138 142 142 251 259 259 F F F <---
153582024-04-17T17:20:11.179Z 7 140-159 140 142 142 255 259 259 F F F <---
153592024-04-17T17:20:11.179Z 8 160-179 142 142 142 257 259 259 F F F <---
153602024-04-17T17:20:11.179Z 9 180-199 141 142 142 256 259 259 F T T <---
153612024-04-17T17:20:11.179Z 10 200-219 142 142 142 257 257 257 F T T <---
153622024-04-17T17:20:11.179Z 11 220-239 142 142 142 257 257 257 F F F
153632024-04-17T17:20:11.179Z 12 240-259 139 139 139 252 252 252 F F F
153642024-04-17T17:20:11.179Z 13 260-279 142 142 142 257 259 259 F F F <---
153652024-04-17T17:20:11.179Z 14 280-299 140 140 140 255 255 255 F F F
153662024-04-17T17:20:11.179Z 15 300-319 141 141 141 256 256 256 F F F
153672024-04-17T17:20:11.179Z 16 320-339 140 140 140 253 253 253 F F F
153682024-04-17T17:20:11.179Z 17 340-359 138 138 138 250 250 250 F F F
153692024-04-17T17:20:11.180Z 18 360-379 141 141 141 256 256 256 F F F
153702024-04-17T17:20:11.180Z 19 380-399 141 141 141 256 256 256 F F F
153712024-04-17T17:20:11.180Z 20 400-419 137 137 137 248 248 248 F F F
153722024-04-17T17:20:11.180Z 21 420-439 137 137 137 248 248 248 F F F
153732024-04-17T17:20:11.180Z 22 440-459 134 134 134 241 241 241 F F F
153742024-04-17T17:20:11.180Z 23 460-479 136 136 136 246 246 246 F T T <---
153752024-04-17T17:20:11.180Z 24 480-499 130 130 130 231 231 231 F F F
153762024-04-17T17:20:11.180Z 25 500-519 130 130 130 230 230 230 F T T <---
153772024-04-17T17:20:11.180Z 26 520-539 141 141 141 256 256 256 F F F
153782024-04-17T17:20:11.180Z 27 540-559 139 142 142 252 259 259 F F F <---
153792024-04-17T17:20:11.180Z 28 560-579 139 139 139 252 252 252 F F F
153802024-04-17T17:20:11.180Z 29 580-599 141 141 141 256 256 256 F F F
153812024-04-17T17:20:11.180ZMax gen: 142, Max flush: 260
153822024-04-17T17:20:11.180ZError: Difference in extent metadata found!
153832024-04-17T17:20:11.180ZA Difference in extent metadata is expected here
153842024-04-17T17:20:11.180ZOn loop 71
153852024-04-17T17:20:11.180Z
153862024-04-17T17:20:11.180ZVerifying data now
153872024-04-17T17:20:15.319ZNo change after verify
153882024-04-17T17:20:15.319ZLoop: 71 Downstairs dump after verify (and repair):
153892024-04-17T17:20:15.333ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
153902024-04-17T17:20:15.333Z 0 000-019 134 134 134 241 241 241 F F F
153912024-04-17T17:20:15.333Z 1 020-039 141 141 141 256 256 256 F F F
153922024-04-17T17:20:15.333Z 2 040-059 141 141 141 256 256 256 F F F
153932024-04-17T17:20:15.333Z 3 060-079 142 142 142 260 260 260 F F F
153942024-04-17T17:20:15.333Z 4 080-099 142 142 142 258 258 258 F F F
153952024-04-17T17:20:15.337Z 5 100-119 138 138 138 251 251 251 F F F
153962024-04-17T17:20:15.337Z 6 120-139 142 142 142 259 259 259 F F F
153972024-04-17T17:20:15.337Z 7 140-159 142 142 142 259 259 259 F F F
153982024-04-17T17:20:15.337Z 8 160-179 142 142 142 259 259 259 F F F
153992024-04-17T17:20:15.337Z 9 180-199 143 143 143 261 261 261 F F F
154002024-04-17T17:20:15.338Z 10 200-219 143 143 143 261 261 261 F F F
154012024-04-17T17:20:15.338Z 11 220-239 142 142 142 257 257 257 F F F
154022024-04-17T17:20:15.338Z 12 240-259 139 139 139 252 252 252 F F F
154032024-04-17T17:20:15.338Z 13 260-279 142 142 142 259 259 259 F F F
154042024-04-17T17:20:15.338Z 14 280-299 140 140 140 255 255 255 F F F
154052024-04-17T17:20:15.338Z 15 300-319 141 141 141 256 256 256 F F F
154062024-04-17T17:20:15.338Z 16 320-339 140 140 140 253 253 253 F F F
154072024-04-17T17:20:15.338Z 17 340-359 138 138 138 250 250 250 F F F
154082024-04-17T17:20:15.338Z 18 360-379 141 141 141 256 256 256 F F F
154092024-04-17T17:20:15.338Z 19 380-399 141 141 141 256 256 256 F F F
154102024-04-17T17:20:15.338Z 20 400-419 137 137 137 248 248 248 F F F
154112024-04-17T17:20:15.338Z 21 420-439 137 137 137 248 248 248 F F F
154122024-04-17T17:20:15.338Z 22 440-459 134 134 134 241 241 241 F F F
154132024-04-17T17:20:15.338Z 23 460-479 143 143 143 261 261 261 F F F
154142024-04-17T17:20:15.338Z 24 480-499 130 130 130 231 231 231 F F F
154152024-04-17T17:20:15.338Z 25 500-519 143 143 143 261 261 261 F F F
154162024-04-17T17:20:15.338Z 26 520-539 141 141 141 256 256 256 F F F
154172024-04-17T17:20:15.338Z 27 540-559 142 142 142 259 259 259 F F F
154182024-04-17T17:20:15.338Z 28 560-579 139 139 139 252 252 252 F F F
154192024-04-17T17:20:15.338Z 29 580-599 141 141 141 256 256 256 F F F
154202024-04-17T17:20:15.338ZMax gen: 143, Max flush: 261
154212024-04-17T17:20:15.338Z
154222024-04-17T17:20:15.338ZBegin loop 72
154232024-04-17T17:20:15.339ZDownstairs to restart: 1
154242024-04-17T17:20:15.356Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.965388538Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747}
154252024-04-17T17:20:15.356Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.965665114Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747}
154262024-04-17T17:20:15.356Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.965681965Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747}
154272024-04-17T17:20:15.356Z{"msg":"Crucible aa4a2cd7-26ac-4acd-aeec-ca59eb6360d7 has session id: 11049b4d-b4de-4f84-9d54-1880588f6ed6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.965696836Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154282024-04-17T17:20:15.362Z{Crucible runtime is spawned
154292024-04-17T17:20:15.362Z"msg":"Upstairs opts: Upstairs UUID: aa4a2cd7-26ac-4acd-aeec-ca59eb6360d7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.965808643Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154302024-04-17T17:20:15.362Z{"msg":"Crucible stats registered with UUID: aa4a2cd7-26ac-4acd-aeec-ca59eb6360d7","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.965901089Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154312024-04-17T17:20:15.362Z{"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.965934131Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747}
154322024-04-17T17:20:15.362Z{"msg":"aa4a2cd7-26ac-4acd-aeec-ca59eb6360d7 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.965956482Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154332024-04-17T17:20:15.362Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.965989174Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"client":"0","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154342024-04-17T17:20:15.362Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.966070919Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"client":"0","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154352024-04-17T17:20:15.362Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.966101291Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"client":"1","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154362024-04-17T17:20:15.362Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.966142623Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"client":"1","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154372024-04-17T17:20:15.362Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.966169825Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"client":"2","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154382024-04-17T17:20:15.362Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.966278821Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"client":"2","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154392024-04-17T17:20:15.362Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.966329315Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154402024-04-17T17:20:15.362Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.966359466Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154412024-04-17T17:20:15.362Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.966396588Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154422024-04-17T17:20:15.362Z{"msg":"ds_connection connected from Ok(127.0.0.1:54504)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.9664269Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154432024-04-17T17:20:15.362Z{"msg":"ds_connection connected from Ok(127.0.0.1:33735)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.966467723Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154442024-04-17T17:20:15.362Z{"msg":"ds_connection connected from Ok(127.0.0.1:39477)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.966498645Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154452024-04-17T17:20:15.362Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.967009946Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"client":"2","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154462024-04-17T17:20:15.363Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.967094301Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"client":"2","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154472024-04-17T17:20:15.363Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.967147904Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"client":"0","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154482024-04-17T17:20:15.363Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.967178646Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"client":"0","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154492024-04-17T17:20:15.363Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.967205998Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"client":"1","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154502024-04-17T17:20:15.363Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.967232729Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"client":"1","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154512024-04-17T17:20:15.363Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.96757218Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"client":"2","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154522024-04-17T17:20:15.363Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.967608012Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"client":"0","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154532024-04-17T17:20:15.363Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.967894169Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"client":"1","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154542024-04-17T17:20:15.363Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.968188167Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"client":"2","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154552024-04-17T17:20:15.363Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.9682379Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"client":"2","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154562024-04-17T17:20:15.363Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.968263562Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154572024-04-17T17:20:15.363Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.968296414Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"client":"0","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154582024-04-17T17:20:15.363Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.968324896Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"client":"0","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154592024-04-17T17:20:15.363Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.968362808Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154602024-04-17T17:20:15.363Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.96839447Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"client":"1","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154612024-04-17T17:20:15.363Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.968427062Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"client":"1","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154622024-04-17T17:20:15.363Z{"msg":"[0]R flush_numbers[0..12]: [241, 256, 256, 260, 258, 251, 259, 259, 259, 261, 261, 257]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.968463214Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"":"downstairs","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154632024-04-17T17:20:15.363Z{"msg":"[0]R generation[0..12]: [134, 141, 141, 142, 142, 138, 142, 142, 142, 143, 143, 142]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.968499156Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"":"downstairs","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154642024-04-17T17:20:15.363Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.968535118Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"":"downstairs","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154652024-04-17T17:20:15.363Z{"msg":"[1]R flush_numbers[0..12]: [241, 256, 256, 260, 258, 251, 259, 259, 259, 261, 261, 257]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.968578481Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"":"downstairs","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154662024-04-17T17:20:15.363Z{"msg":"[1]R generation[0..12]: [134, 141, 141, 142, 142, 138, 142, 142, 142, 143, 143, 142]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.968623514Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"":"downstairs","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154672024-04-17T17:20:15.363Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.968656276Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
154682024-04-17T17:20:15.363Z11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154692024-04-17T17:20:15.364Z{"msg":"[2]R flush_numbers[0..12]: [241, 256, 256, 260, 258, 251, 259, 259, 259, 261, 261, 257]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.968674527Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"":"downstairs","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154702024-04-17T17:20:15.364Z{"msg":"[2]R generation[0..12]: [134, 141, 141, 142, 142, 138, 142, 142, 142, 143, 143, 142]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.968700708Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"":"downstairs","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154712024-04-17T17:20:15.364Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.96872817Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"":"downstairs","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154722024-04-17T17:20:15.364Z{"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.968764482Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"":"downstairs","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154732024-04-17T17:20:15.364Z{"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.968801104Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"":"downstairs","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154742024-04-17T17:20:15.364Z{"msg":"Next flush: 262","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.968838536Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"":"downstairs","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154752024-04-17T17:20:15.364Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.968873099Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"":"downstairs","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154762024-04-17T17:20:15.364Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.96889986Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154772024-04-17T17:20:15.364Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.968933172Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"client":"0","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154782024-04-17T17:20:15.364Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.968961434Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"client":"1","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154792024-04-17T17:20:15.364Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.968997026Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"client":"2","":"downstairs","session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154802024-04-17T17:20:15.364Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.969023488Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154812024-04-17T17:20:15.364Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
154822024-04-17T17:20:15.364Z,"time":"2024-04-17T17:20:13.96905936Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154832024-04-17T17:20:15.364Z{"msg":"aa4a2cd7-26ac-4acd-aeec-ca59eb6360d7 is now active with session: 11049b4d-b4de-4f84-9d54-1880588f6ed6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.969090002Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154842024-04-17T17:20:15.364Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.969115113Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747,"session_id":"11049b4d-b4de-4f84-9d54-1880588f6ed6"}
154852024-04-17T17:20:15.364Z{"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:13.969127784Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747}
154862024-04-17T17:20:15.368ZLoading write count information from file "/tmp/test_repair_verify.data"
154872024-04-17T17:20:15.368ZRun Repair workload
154882024-04-17T17:20:15.368Z01/30 Write block 51 len 36864 data: 11 11 12 17 15 15 15 16 15
154892024-04-17T17:20:15.368Z02/30 Write block 392 len 16384 data: 14 12 13 8
154902024-04-17T17:20:15.368Z03/30 Flush
154912024-04-17T17:20:15.368Z04/30 Write block 382 len 16384 data: 11 12 11 13
154922024-04-17T17:20:15.368Z05/30 Read block 405 len 40960
154932024-04-17T17:20:15.368Z06/30 Write block 16 len 28672 data: 7 10 9 10 11 14 14
154942024-04-17T17:20:15.368Z07/30 Flush
154952024-04-17T17:20:15.372Z08/30 Write block 578 len 40960 data: 11 14 12 13 12 13 14 13 14 12
154962024-04-17T17:20:15.375Z09/30 Read block 484 len 12288
154972024-04-17T17:20:15.375Z10/30 Write block 198 len 8192 data: 11 9
154982024-04-17T17:20:15.375Z11/30 Read block 402 len 24576
154992024-04-17T17:20:15.375Z12/30 Write block 318 len 16384 data: 15 13 11 10
155002024-04-17T17:20:15.375Z13/30 Read block 22 len 36864
155012024-04-17T17:20:15.379Z14/30 Read block 486 len 4096
155022024-04-17T17:20:15.379Z15/30 Read block 383 len 8192
155032024-04-17T17:20:15.379Z16/30 Flush
155042024-04-17T17:20:15.382Z17/30 Write block 594 len 24576 data: 7 8 7 7 6 6
155052024-04-17T17:20:15.383Z18/30 Read block 573 len 12288
155062024-04-17T17:20:15.383Z19/30 Write block 364 len 40960 data: 8 8 8 8 12 12 12 13 10 11
155072024-04-17T17:20:15.386Z20/30 Write block 235 len 20480 data: 14 12 10 8 10
155082024-04-17T17:20:15.386Z21/30 Read block 267 len 20480
155092024-04-17T17:20:15.386Z22/30 Read block 228 len 24576
155102024-04-17T17:20:15.386Z23/30 Write block 243 len 32768 data: 13 12 11 13 14 14 11 10
155112024-04-17T17:20:15.390Z24/30 Read block 241 len 40960
155122024-04-17T17:20:15.390Z25/30 Write block 343 len 32768 data: 15 15 16 15 11 10 10 11
155132024-04-17T17:20:15.390Z26/30 Read block 78 len 16384
155142024-04-17T17:20:15.390Z27/30 Read block 8 len 16384
155152024-04-17T17:20:15.394Z28/30 Read block 141 len 40960
155162024-04-17T17:20:15.394Z29/30 Write block 1 len 40960 data: 5 4 9 11 11 9 8 8 9 9
155172024-04-17T17:20:15.394Z30/30 Write block 575 len 12288 data: 15 14 12
155182024-04-17T17:20:15.394Z----------------------------------------------------------------
155192024-04-17T17:20:15.395Z Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:30
155202024-04-17T17:20:15.395ZStates: Active Active Active
155212024-04-17T17:20:15.395ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
155222024-04-17T17:20:15.395Z 1 Acked 1000 Write 9 Done Sent Done false
155232024-04-17T17:20:15.395Z 2 Acked 1001 Write 4 Done Sent Done false
155242024-04-17T17:20:15.395Z 3 Acked 1002 Flush 0 Done Sent Done false
155252024-04-17T17:20:15.395Z 4 Acked 1003 Write 4 Done Sent Done false
155262024-04-17T17:20:15.395Z 5 Acked 1004 Read 10 Done Sent Done false
155272024-04-17T17:20:15.395Z 6 Acked 1005 Write 7 Done Sent Done false
155282024-04-17T17:20:15.395Z 7 Acked 1006 Flush 0 Done Sent Done false
155292024-04-17T17:20:15.395Z 8 Acked 1007 Write 10 Done Sent Done false
155302024-04-17T17:20:15.395Z 9 Acked 1008 Read 3 Done Sent Done false
155312024-04-17T17:20:15.395Z 10 Acked 1009 Write 2 Done Sent Done false
155322024-04-17T17:20:15.395Z 11 Acked 1010 Read 6 Done Sent Done false
155332024-04-17T17:20:15.395Z 12 Acked 1011 Write 4 Done Sent Done false
155342024-04-17T17:20:15.395Z 13 Acked 1012 Read 9 Done Sent Done false
155352024-04-17T17:20:15.395Z 14 Acked 1013 Read 1 Done Sent Done false
155362024-04-17T17:20:15.395Z 15 Acked 1014 Read 2 Done Sent Done false
155372024-04-17T17:20:15.395Z 16 Acked 1015 Flush 0 Done Sent Done false
155382024-04-17T17:20:15.395Z 17 Acked 1016 Write 6 Done Sent Done false
155392024-04-17T17:20:15.395Z 18 Acked 1017 Read 3 Done Sent Done false
155402024-04-17T17:20:15.395Z 19 Acked 1018 Write 10 Done Sent Done false
155412024-04-17T17:20:15.395Z 20 Acked 1019 Write 5 Done Sent Done false
155422024-04-17T17:20:15.395Z 21 Acked 1020 Read 5 Done Sent Done false
155432024-04-17T17:20:15.396Z 22 Acked 1021 Read 6 Done Sent Done false
155442024-04-17T17:20:15.396Z 23 Acked 1022 Write 8 Done Sent Done false
155452024-04-17T17:20:15.396Z 24 Acked 1023 Read 10 Done Sent Done false
155462024-04-17T17:20:15.396Z 25 Acked 1024 Write 8 Done Sent Done false
155472024-04-17T17:20:15.396Z 26 Acked 1025 Read 4 Done Sent Done false
155482024-04-17T17:20:15.396Z 27 Acked 1026 Read 4 Done Sent Done false
155492024-04-17T17:20:15.396Z 28 Acked 1027 Read 10 Done Sent Done false
155502024-04-17T17:20:15.396Z 29 Acked 1028 Write 10 Done Sent Done false
155512024-04-17T17:20:15.396Z 30 Acked 1029 Write 3 Sent Sent Sent false
155522024-04-17T17:20:15.396Z STATES DS:0 DS:1 DS:2 TOTAL
155532024-04-17T17:20:15.396Z New 0 0 0 0
155542024-04-17T17:20:15.396Z Sent 1 30 1 32
155552024-04-17T17:20:15.396Z Done 29 0 29 58
155562024-04-17T17:20:15.396Z Skipped 0 0 0 0
155572024-04-17T17:20:15.396Z Error 0 0 0 0
155582024-04-17T17:20:15.396ZLast Flush: 1015 0 1015
155592024-04-17T17:20:15.396ZDownstairs last five completed:
155602024-04-17T17:20:15.396ZUpstairs last five completed: 30 29 28 27 26
155612024-04-17T17:20:15.396Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:20:14.003181338Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1747}
155622024-04-17T17:20:15.400ZWrote out file "/tmp/test_repair_verify.data"
155632024-04-17T17:20:15.400Z
155642024-04-17T17:20:15.403ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
155652024-04-17T17:20:15.418ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
155662024-04-17T17:20:15.418Z 0 000-019 144 134 144 263 241 263 T F T <---
155672024-04-17T17:20:15.418Z 1 020-039 144 141 144 263 256 263 F F F <---
155682024-04-17T17:20:15.418Z 2 040-059 144 141 144 262 256 262 F F F <---
155692024-04-17T17:20:15.418Z 3 060-079 142 142 142 260 260 260 F F F
155702024-04-17T17:20:15.418Z 4 080-099 142 142 142 258 258 258 F F F
155712024-04-17T17:20:15.418Z 5 100-119 138 138 138 251 251 251 F F F
155722024-04-17T17:20:15.418Z 6 120-139 142 142 142 259 259 259 F F F
155732024-04-17T17:20:15.418Z 7 140-159 142 142 142 259 259 259 F F F
155742024-04-17T17:20:15.422Z 8 160-179 142 142 142 259 259 259 F F F
155752024-04-17T17:20:15.422Z 9 180-199 144 143 144 264 261 264 F F F <---
155762024-04-17T17:20:15.423Z 10 200-219 143 143 143 261 261 261 F F F
155772024-04-17T17:20:15.423Z 11 220-239 142 142 142 257 257 257 T F T <---
155782024-04-17T17:20:15.423Z 12 240-259 139 139 139 252 252 252 T F T <---
155792024-04-17T17:20:15.423Z 13 260-279 142 142 142 259 259 259 F F F
155802024-04-17T17:20:15.423Z 14 280-299 140 140 140 255 255 255 F F F
155812024-04-17T17:20:15.423Z 15 300-319 144 141 144 264 256 264 F F F <---
155822024-04-17T17:20:15.423Z 16 320-339 144 140 144 264 253 264 F F F <---
155832024-04-17T17:20:15.423Z 17 340-359 138 138 138 250 250 250 T F T <---
155842024-04-17T17:20:15.423Z 18 360-379 141 141 141 256 256 256 T F T <---
155852024-04-17T17:20:15.423Z 19 380-399 144 141 144 263 256 263 F F F <---
155862024-04-17T17:20:15.423Z 20 400-419 137 137 137 248 248 248 F F F
155872024-04-17T17:20:15.423Z 21 420-439 137 137 137 248 248 248 F F F
155882024-04-17T17:20:15.423Z 22 440-459 134 134 134 241 241 241 F F F
155892024-04-17T17:20:15.423Z 23 460-479 143 143 143 261 261 261 F F F
155902024-04-17T17:20:15.423Z 24 480-499 130 130 130 231 231 231 F F F
155912024-04-17T17:20:15.423Z 25 500-519 143 143 143 261 261 261 F F F
155922024-04-17T17:20:15.423Z 26 520-539 141 141 141 256 256 256 F F F
155932024-04-17T17:20:15.423Z 27 540-559 142 142 142 259 259 259 F F F
155942024-04-17T17:20:15.423Z 28 560-579 144 139 144 264 252 264 T F T <---
155952024-04-17T17:20:15.423Z 29 580-599 144 141 144 264 256 264 T F T <---
155962024-04-17T17:20:15.423ZMax gen: 144, Max flush: 264
155972024-04-17T17:20:15.423ZError: Difference in extent metadata found!
155982024-04-17T17:20:15.423ZA Difference in extent metadata is expected here
155992024-04-17T17:20:15.424ZOn loop 72
156002024-04-17T17:20:15.424Z
156012024-04-17T17:20:15.427ZVerifying data now
156022024-04-17T17:20:20.031ZNo change after verify
156032024-04-17T17:20:20.031ZLoop: 72 Downstairs dump after verify (and repair):
156042024-04-17T17:20:20.045ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
156052024-04-17T17:20:20.045Z 0 000-019 145 145 145 265 265 265 F F F
156062024-04-17T17:20:20.049Z 1 020-039 144 144 144 263 263 263 F F F
156072024-04-17T17:20:20.049Z 2 040-059 144 144 144 262 262 262 F F F
156082024-04-17T17:20:20.049Z 3 060-079 142 142 142 260 260 260 F F F
156092024-04-17T17:20:20.050Z 4 080-099 142 142 142 258 258 258 F F F
156102024-04-17T17:20:20.050Z 5 100-119 138 138 138 251 251 251 F F F
156112024-04-17T17:20:20.050Z 6 120-139 142 142 142 259 259 259 F F F
156122024-04-17T17:20:20.050Z 7 140-159 142 142 142 259 259 259 F F F
156132024-04-17T17:20:20.050Z 8 160-179 142 142 142 259 259 259 F F F
156142024-04-17T17:20:20.050Z 9 180-199 144 144 144 264 264 264 F F F
156152024-04-17T17:20:20.050Z 10 200-219 143 143 143 261 261 261 F F F
156162024-04-17T17:20:20.050Z 11 220-239 145 145 145 265 265 265 F F F
156172024-04-17T17:20:20.050Z 12 240-259 145 145 145 265 265 265 F F F
156182024-04-17T17:20:20.050Z 13 260-279 142 142 142 259 259 259 F F F
156192024-04-17T17:20:20.050Z 14 280-299 140 140 140 255 255 255 F F F
156202024-04-17T17:20:20.050Z 15 300-319 144 144 144 264 264 264 F F F
156212024-04-17T17:20:20.050Z 16 320-339 144 144 144 264 264 264 F F F
156222024-04-17T17:20:20.050Z 17 340-359 145 145 145 265 265 265 F F F
156232024-04-17T17:20:20.050Z 18 360-379 145 145 145 265 265 265 F F F
156242024-04-17T17:20:20.050Z 19 380-399 144 144 144 263 263 263 F F F
156252024-04-17T17:20:20.050Z 20 400-419 137 137 137 248 248 248 F F F
156262024-04-17T17:20:20.050Z 21 420-439 137 137 137 248 248 248 F F F
156272024-04-17T17:20:20.050Z 22 440-459 134 134 134 241 241 241 F F F
156282024-04-17T17:20:20.050Z 23 460-479 143 143 143 261 261 261 F F F
156292024-04-17T17:20:20.050Z 24 480-499 130 130 130 231 231 231 F F F
156302024-04-17T17:20:20.050Z 25 500-519 143 143 143 261 261 261 F F F
156312024-04-17T17:20:20.050Z 26 520-539 141 141 141 256 256 256 F F F
156322024-04-17T17:20:20.051Z 27 540-559 142 142 142 259 259 259 F F F
156332024-04-17T17:20:20.051Z 28 560-579 145 145 145 265 265 265 F F F
156342024-04-17T17:20:20.051Z 29 580-599 145 145 145 265 265 265 F F F
156352024-04-17T17:20:20.051ZMax gen: 145, Max flush: 265
156362024-04-17T17:20:20.051Z
156372024-04-17T17:20:20.051ZBegin loop 73
156382024-04-17T17:20:20.051ZDownstairs to restart: 2
156392024-04-17T17:20:20.069Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.678132714Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755}
156402024-04-17T17:20:20.069Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.678365538Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755}
156412024-04-17T17:20:20.074Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.67839906Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755}
156422024-04-17T17:20:20.075Z{"msg":"Crucible ae963c16-3179-433c-a116-02501bc71841 has session id: 76424cf8-0628-4e4e-a248-f1ef46531e5f","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.678438652Z"Crucible runtime is spawned
156432024-04-17T17:20:20.075Z,"hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156442024-04-17T17:20:20.075Z{"msg":"Upstairs opts: Upstairs UUID: ae963c16-3179-433c-a116-02501bc71841, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.678494856Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156452024-04-17T17:20:20.075Z{"msg":"Crucible stats registered with UUID: ae963c16-3179-433c-a116-02501bc71841","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.678534788Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156462024-04-17T17:20:20.075Z{"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.678559Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755}
156472024-04-17T17:20:20.075Z{"msg":"ae963c16-3179-433c-a116-02501bc71841 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.678676647Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156482024-04-17T17:20:20.075Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.678707949Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"client":"0","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156492024-04-17T17:20:20.075Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.678765942Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"client":"0","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156502024-04-17T17:20:20.075Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.678817915Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"client":"1","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156512024-04-17T17:20:20.075Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.678842437Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"client":"1","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156522024-04-17T17:20:20.075Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.678874779Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"client":"2","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156532024-04-17T17:20:20.075Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.678911911Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"client":"2","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156542024-04-17T17:20:20.075Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.678937022Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"":"io task","client":"2","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156552024-04-17T17:20:20.075Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.678967644Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"":"io task","client":"0","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156562024-04-17T17:20:20.075Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.679029338Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"":"io task","client":"1","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156572024-04-17T17:20:20.075Z{"msg":"ds_connection connected from Ok(127.0.0.1:56539)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.679069291Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"":"io task","client":"1","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156582024-04-17T17:20:20.075Z{"msg":"ds_connection connected from Ok(127.0.0.1:42171)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.679094562Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"":"io task","client":"0","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156592024-04-17T17:20:20.075Z{"msg":"ds_connection connected from Ok(127.0.0.1:35494)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.679125624Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"":"io task","client":"2","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156602024-04-17T17:20:20.075Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.679871809Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"client":"0","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156612024-04-17T17:20:20.075Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.679910041Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"client":"0","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156622024-04-17T17:20:20.075Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.679926992Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"client":"1","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156632024-04-17T17:20:20.075Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.679962245Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"client":"1","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156642024-04-17T17:20:20.076Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.680006117Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"client":"2","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156652024-04-17T17:20:20.076Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.680100283Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"client":"2","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156662024-04-17T17:20:20.076Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.680523499Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"client":"1","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156672024-04-17T17:20:20.076Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.680566511Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"client":"0","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156682024-04-17T17:20:20.076Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.680584742Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"client":"2","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156692024-04-17T17:20:20.076Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.68104144Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"client":"1","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156702024-04-17T17:20:20.076Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.681080482Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"client":"1","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156712024-04-17T17:20:20.076Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.681106294Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156722024-04-17T17:20:20.076Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.681164027Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"client":"2","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156732024-04-17T17:20:20.076Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.681194809Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"client":"2","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156742024-04-17T17:20:20.076Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.681218311Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156752024-04-17T17:20:20.076Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.681244872Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"client":"0","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156762024-04-17T17:20:20.076Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.681268404Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"client":"0","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156772024-04-17T17:20:20.076Z{"msg":"[0]R flush_numbers[0..12]: [265, 263, 262, 260, 258, 251, 259, 259, 259, 264, 261, 265]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.681296806Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"":"downstairs","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156782024-04-17T17:20:20.076Z{"msg":"[0]R generation[0..12]: [145, 144, 144, 142, 142, 138, 142, 142, 142, 144, 143, 145]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.681321557Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"":"downstairs","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156792024-04-17T17:20:20.076Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.681350289Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"":"downstairs","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156802024-04-17T17:20:20.076Z{"msg":"[1]R flush_numbers[0..12]: [265, 263, 262, 260, 258, 251, 259, 259, 259, 264, 261, 265]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.68137523Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"":"downstairs","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156812024-04-17T17:20:20.076ZWait for a query_work_queue command to finish before sending IO
156822024-04-17T17:20:20.076Z{"msg":"[1]R generation[0..12]: [145, 144, 144, 142, 142, 138, 142, 142, 142, 144, 143, 145]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.681406452Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"":"downstairs","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156832024-04-17T17:20:20.076Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.681432784Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"":"downstairs","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156842024-04-17T17:20:20.076Z{"msg":"[2]R flush_numbers[0..12]: [265, 263, 262, 260, 258, 251, 259, 259, 259, 264, 261, 265]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.681465616Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"":"downstairs","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156852024-04-17T17:20:20.076Z{"msg":"[2]R generation[0..12]: [145, 144, 144, 142, 142, 138, 142, 142, 142, 144, 143, 145]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.681507528Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"":"downstairs","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156862024-04-17T17:20:20.076Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.681547901Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"":"downstairs","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156872024-04-17T17:20:20.077Z{"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.681570562Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"":"downstairs","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156882024-04-17T17:20:20.077Z{"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.681598274Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"":"downstairs","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156892024-04-17T17:20:20.077Z{"msg":"Next flush: 266","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.681620755Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"":"downstairs","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156902024-04-17T17:20:20.077Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.681647577Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"":"downstairs","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156912024-04-17T17:20:20.077Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.681684599Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156922024-04-17T17:20:20.077Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.68170679Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"client":"0","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156932024-04-17T17:20:20.077Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.681733012Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"client":"1","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156942024-04-17T17:20:20.077Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.681768744Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"client":"2","":"downstairs","session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156952024-04-17T17:20:20.077Z{"msg":"All required reconciliation work is completed","vRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
156962024-04-17T17:20:20.077Z":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.681800646Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156972024-04-17T17:20:20.077Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.681833738Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156982024-04-17T17:20:20.077Z{"msg":"ae963c16-3179-433c-a116-02501bc71841 is now active with session: 76424cf8-0628-4e4e-a248-f1ef46531e5f","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.681849149Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
156992024-04-17T17:20:20.077Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.68186304Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755,"session_id":"76424cf8-0628-4e4e-a248-f1ef46531e5f"}
157002024-04-17T17:20:20.077Z{"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:18.681882661Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755}
157012024-04-17T17:20:20.081ZLoading write count information from file "/tmp/test_repair_verify.data"
157022024-04-17T17:20:20.081ZRun Repair workload
157032024-04-17T17:20:20.081Z01/30 Write block 112 len 40960 data: 10 9 7 8 8 7 5 9 10 11
157042024-04-17T17:20:20.081Z02/30 Read block 468 len 4096
157052024-04-17T17:20:20.081Z03/30 Read block 133 len 40960
157062024-04-17T17:20:20.081Z04/30 Flush
157072024-04-17T17:20:20.081Z05/30 Write block 171 len 32768 data: 11 10 11 8 11 11 10 13
157082024-04-17T17:20:20.081Z06/30 Write block 430 len 20480 data: 7 8 10 14 13
157092024-04-17T17:20:20.085Z07/30 Read block 120 len 36864
157102024-04-17T17:20:20.085Z08/30 Write block 92 len 8192 data: 18 16
157112024-04-17T17:20:20.085Z09/30 Read block 568 len 36864
157122024-04-17T17:20:20.085Z10/30 Read block 268 len 32768
157132024-04-17T17:20:20.085Z11/30 Read block 115 len 20480
157142024-04-17T17:20:20.088Z12/30 Write block 574 len 8192 data: 15 16
157152024-04-17T17:20:20.088Z13/30 Write block 336 len 12288 data: 14 15 15
157162024-04-17T17:20:20.088Z14/30 Read block 332 len 36864
157172024-04-17T17:20:20.088Z15/30 Read block 592 len 16384
157182024-04-17T17:20:20.088Z16/30 Write block 445 len 4096 data: 8
157192024-04-17T17:20:20.092Z17/30 Write block 499 len 40960 data: 12 13 14 13 13 12 9 8 10 10
157202024-04-17T17:20:20.092Z18/30 Read block 139 len 40960
157212024-04-17T17:20:20.092Z19/30 Flush
157222024-04-17T17:20:20.095Z20/30 Write block 108 len 40960 data: 11 10 11 12 11 10 8 9 9 8
157232024-04-17T17:20:20.098Z21/30 Write block 166 len 36864 data: 14 15 13 11 10 12 11 12 9
157242024-04-17T17:20:20.098Z22/30 Read block 379 len 8192
157252024-04-17T17:20:20.098Z23/30 Read block 535 len 28672
157262024-04-17T17:20:20.098Z24/30 Read block 72 len 8192
157272024-04-17T17:20:20.102Z25/30 Write block 335 len 28672 data: 11 15 16 16 15 14 12
157282024-04-17T17:20:20.102Z26/30 Write block 34 len 4096 data: 22
157292024-04-17T17:20:20.102Z27/30 Write block 185 len 20480 data: 9 9 14 17 16
157302024-04-17T17:20:20.102Z28/30 Write block 520 len 16384 data: 8 9 10 10
157312024-04-17T17:20:20.102Z29/30 Read block 119 len 36864
157322024-04-17T17:20:20.106Z30/30 Write block 291 len 4096 data: 12
157332024-04-17T17:20:20.106Z----------------------------------------------------------------
157342024-04-17T17:20:20.106Z Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:30
157352024-04-17T17:20:20.106ZStates: Active Active Active
157362024-04-17T17:20:20.106ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
157372024-04-17T17:20:20.107Z 1 Acked 1000 Write 10 Done Done Sent false
157382024-04-17T17:20:20.107Z 2 Acked 1001 Read 1 Done Done Sent false
157392024-04-17T17:20:20.107Z 3 Acked 1002 Read 10 Done Done Sent false
157402024-04-17T17:20:20.107Z 4 Acked 1003 Flush 0 Done Done Sent false
157412024-04-17T17:20:20.107Z 5 Acked 1004 Write 8 Done Done Sent false
157422024-04-17T17:20:20.107Z 6 Acked 1005 Write 5 Done Done Sent false
157432024-04-17T17:20:20.107Z 7 Acked 1006 Read 9 Done Done Sent false
157442024-04-17T17:20:20.107Z 8 Acked 1007 Write 2 Done Done Sent false
157452024-04-17T17:20:20.107Z 9 Acked 1008 Read 9 Done Done Sent false
157462024-04-17T17:20:20.107Z 10 Acked 1009 Read 8 Done Done Sent false
157472024-04-17T17:20:20.107Z 11 Acked 1010 Read 5 Done Done Sent false
157482024-04-17T17:20:20.107Z 12 Acked 1011 Write 2 Done Done Sent false
157492024-04-17T17:20:20.107Z 13 Acked 1012 Write 3 Done Done Sent false
157502024-04-17T17:20:20.107Z 14 Acked 1013 Read 9 Done Done Sent false
157512024-04-17T17:20:20.107Z 15 Acked 1014 Read 4 Done Done Sent false
157522024-04-17T17:20:20.107Z 16 Acked 1015 Write 1 Done Done Sent false
157532024-04-17T17:20:20.107Z 17 Acked 1016 Write 10 Done Done Sent false
157542024-04-17T17:20:20.107Z 18 Acked 1017 Read 10 Done Done Sent false
157552024-04-17T17:20:20.107Z 19 Acked 1018 Flush 0 Done Done Sent false
157562024-04-17T17:20:20.107Z 20 Acked 1019 Write 10 Done Done Sent false
157572024-04-17T17:20:20.107Z 21 Acked 1020 Write 9 Done Done Sent false
157582024-04-17T17:20:20.107Z 22 Acked 1021 Read 2 Done Done Sent false
157592024-04-17T17:20:20.107Z 23 Acked 1022 Read 7 Done Done Sent false
157602024-04-17T17:20:20.107Z 24 Acked 1023 Read 2 Done Done Sent false
157612024-04-17T17:20:20.108Z 25 Acked 1024 Write 7 Done Done Sent false
157622024-04-17T17:20:20.108Z 26 Acked 1025 Write 1 Done Done Sent false
157632024-04-17T17:20:20.108Z 27 Acked 1026 Write 5 Done Done Sent false
157642024-04-17T17:20:20.108Z 28 Acked 1027 Write 4 Done Done Sent false
157652024-04-17T17:20:20.108Z 29 Acked 1028 Read 9 Done Done Sent false
157662024-04-17T17:20:20.108Z 30 Acked 1029 Write 1 Sent Sent Sent false
157672024-04-17T17:20:20.108Z STATES DS:0 DS:1 DS:2 TOTAL
157682024-04-17T17:20:20.108Z New 0 0 0 0
157692024-04-17T17:20:20.108Z Sent 1 1 30 32
157702024-04-17T17:20:20.108Z Done 29 29 0 58
157712024-04-17T17:20:20.108Z Skipped 0 0 0 0
157722024-04-17T17:20:20.108Z Error 0 0 0 0
157732024-04-17T17:20:20.108ZLast Flush: 1018 1018 0
157742024-04-17T17:20:20.108ZDownstairs last five completed:
157752024-04-17T17:20:20.108ZUpstairs last five completed: 30 29 28 27 26
157762024-04-17T17:20:20.108Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:20:18.712823486Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1755}
157772024-04-17T17:20:20.108ZWrote out file "/tmp/test_repair_verify.data"
157782024-04-17T17:20:20.112Z
157792024-04-17T17:20:20.112ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
157802024-04-17T17:20:20.128ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
157812024-04-17T17:20:20.128Z 0 000-019 145 145 145 265 265 265 F F F
157822024-04-17T17:20:20.128Z 1 020-039 144 144 144 263 263 263 T T F <---
157832024-04-17T17:20:20.128Z 2 040-059 144 144 144 262 262 262 F F F
157842024-04-17T17:20:20.128Z 3 060-079 142 142 142 260 260 260 F F F
157852024-04-17T17:20:20.128Z 4 080-099 146 146 142 267 267 258 F F F <---
157862024-04-17T17:20:20.128Z 5 100-119 146 146 138 266 266 251 T T F <---
157872024-04-17T17:20:20.131Z 6 120-139 146 146 142 266 266 259 F F F <---
157882024-04-17T17:20:20.131Z 7 140-159 142 142 142 259 259 259 F F F
157892024-04-17T17:20:20.132Z 8 160-179 146 146 142 267 267 259 T T F <---
157902024-04-17T17:20:20.132Z 9 180-199 144 144 144 264 264 264 T T F <---
157912024-04-17T17:20:20.132Z 10 200-219 143 143 143 261 261 261 F F F
157922024-04-17T17:20:20.132Z 11 220-239 145 145 145 265 265 265 F F F
157932024-04-17T17:20:20.132Z 12 240-259 145 145 145 265 265 265 F F F
157942024-04-17T17:20:20.132Z 13 260-279 142 142 142 259 259 259 F F F
157952024-04-17T17:20:20.132Z 14 280-299 140 140 140 255 255 255 T T F <---
157962024-04-17T17:20:20.132Z 15 300-319 144 144 144 264 264 264 F F F
157972024-04-17T17:20:20.132Z 16 320-339 146 146 144 267 267 264 T T F <---
157982024-04-17T17:20:20.132Z 17 340-359 145 145 145 265 265 265 T T F <---
157992024-04-17T17:20:20.132Z 18 360-379 145 145 145 265 265 265 F F F
158002024-04-17T17:20:20.132Z 19 380-399 144 144 144 263 263 263 F F F
158012024-04-17T17:20:20.132Z 20 400-419 137 137 137 248 248 248 F F F
158022024-04-17T17:20:20.132Z 21 420-439 146 146 137 267 267 248 F F F <---
158032024-04-17T17:20:20.132Z 22 440-459 146 146 134 267 267 241 F F F <---
158042024-04-17T17:20:20.132Z 23 460-479 143 143 143 261 261 261 F F F
158052024-04-17T17:20:20.132Z 24 480-499 146 146 130 267 267 231 F F F <---
158062024-04-17T17:20:20.132Z 25 500-519 146 146 143 267 267 261 F F F <---
158072024-04-17T17:20:20.132Z 26 520-539 141 141 141 256 256 256 T T F <---
158082024-04-17T17:20:20.132Z 27 540-559 142 142 142 259 259 259 F F F
158092024-04-17T17:20:20.132Z 28 560-579 146 146 145 267 267 265 F F F <---
158102024-04-17T17:20:20.132Z 29 580-599 145 145 145 265 265 265 F F F
158112024-04-17T17:20:20.132ZMax gen: 146, Max flush: 267
158122024-04-17T17:20:20.132ZError: Difference in extent metadata found!
158132024-04-17T17:20:20.133ZA Difference in extent metadata is expected here
158142024-04-17T17:20:20.133ZOn loop 73
158152024-04-17T17:20:20.133Z
158162024-04-17T17:20:20.136ZVerifying data now
158172024-04-17T17:20:24.825ZNo change after verify
158182024-04-17T17:20:24.825ZLoop: 73 Downstairs dump after verify (and repair):
158192024-04-17T17:20:24.839ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
158202024-04-17T17:20:24.839Z 0 000-019 145 145 145 265 265 265 F F F
158212024-04-17T17:20:24.839Z 1 020-039 147 147 147 268 268 268 F F F
158222024-04-17T17:20:24.839Z 2 040-059 144 144 144 262 262 262 F F F
158232024-04-17T17:20:24.839Z 3 060-079 142 142 142 260 260 260 F F F
158242024-04-17T17:20:24.839Z 4 080-099 146 146 146 267 267 267 F F F
158252024-04-17T17:20:24.839Z 5 100-119 147 147 147 268 268 268 F F F
158262024-04-17T17:20:24.839Z 6 120-139 146 146 146 266 266 266 F F F
158272024-04-17T17:20:24.840Z 7 140-159 142 142 142 259 259 259 F F F
158282024-04-17T17:20:24.843Z 8 160-179 147 147 147 268 268 268 F F F
158292024-04-17T17:20:24.843Z 9 180-199 147 147 147 268 268 268 F F F
158302024-04-17T17:20:24.843Z 10 200-219 143 143 143 261 261 261 F F F
158312024-04-17T17:20:24.843Z 11 220-239 145 145 145 265 265 265 F F F
158322024-04-17T17:20:24.843Z 12 240-259 145 145 145 265 265 265 F F F
158332024-04-17T17:20:24.843Z 13 260-279 142 142 142 259 259 259 F F F
158342024-04-17T17:20:24.843Z 14 280-299 147 147 147 268 268 268 F F F
158352024-04-17T17:20:24.844Z 15 300-319 144 144 144 264 264 264 F F F
158362024-04-17T17:20:24.844Z 16 320-339 147 147 147 268 268 268 F F F
158372024-04-17T17:20:24.844Z 17 340-359 147 147 147 268 268 268 F F F
158382024-04-17T17:20:24.844Z 18 360-379 145 145 145 265 265 265 F F F
158392024-04-17T17:20:24.844Z 19 380-399 144 144 144 263 263 263 F F F
158402024-04-17T17:20:24.844Z 20 400-419 137 137 137 248 248 248 F F F
158412024-04-17T17:20:24.844Z 21 420-439 146 146 146 267 267 267 F F F
158422024-04-17T17:20:24.844Z 22 440-459 146 146 146 267 267 267 F F F
158432024-04-17T17:20:24.844Z 23 460-479 143 143 143 261 261 261 F F F
158442024-04-17T17:20:24.844Z 24 480-499 146 146 146 267 267 267 F F F
158452024-04-17T17:20:24.844Z 25 500-519 146 146 146 267 267 267 F F F
158462024-04-17T17:20:24.844Z 26 520-539 147 147 147 268 268 268 F F F
158472024-04-17T17:20:24.844Z 27 540-559 142 142 142 259 259 259 F F F
158482024-04-17T17:20:24.844Z 28 560-579 146 146 146 267 267 267 F F F
158492024-04-17T17:20:24.844Z 29 580-599 145 145 145 265 265 265 F F F
158502024-04-17T17:20:24.844ZMax gen: 147, Max flush: 268
158512024-04-17T17:20:24.848Z
158522024-04-17T17:20:24.848ZBegin loop 74
158532024-04-17T17:20:24.848ZDownstairs to restart: 0
158542024-04-17T17:20:24.865Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.474498739Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763}
158552024-04-17T17:20:24.865Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.474763844Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763}
158562024-04-17T17:20:24.865Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.474781706Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763}
158572024-04-17T17:20:24.865Z{"msg":"Crucible aa3d34a4-104d-4980-bada-c37d8b22b472 has session id: f563a4c8-7a5f-4814-9d99-6f01a80ef4cc","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.474805537Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158582024-04-17T17:20:24.871Z{"msg":"Upstairs opts: Upstairs UUID: aa3d34a4-104d-4980-bada-c37d8b22b472, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
158592024-04-17T17:20:24.871Z,"time":"2024-04-17T17:20:23.47486339Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158602024-04-17T17:20:24.872Z{"msg":"Crucible stats registered with UUID: aa3d34a4-104d-4980-bada-c37d8b22b472","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.474903513Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158612024-04-17T17:20:24.872Z{"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.474945755Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763}
158622024-04-17T17:20:24.872Z{"msg":"aa3d34a4-104d-4980-bada-c37d8b22b472 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.475075954Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158632024-04-17T17:20:24.872Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.475098955Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"client":"0","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158642024-04-17T17:20:24.872Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.475117586Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"client":"0","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158652024-04-17T17:20:24.872Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.475151558Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"client":"1","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158662024-04-17T17:20:24.872Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.475173429Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"client":"1","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158672024-04-17T17:20:24.872Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.475201241Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"client":"2","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158682024-04-17T17:20:24.872Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.475224543Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"client":"2","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158692024-04-17T17:20:24.872Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.475251654Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"":"io task","client":"0","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158702024-04-17T17:20:24.872Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.475318678Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"":"io task","client":"2","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158712024-04-17T17:20:24.872Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.4753521Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"":"io task","client":"1","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158722024-04-17T17:20:24.872Z{"msg":"ds_connection connected from Ok(127.0.0.1:58623)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.475497499Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"":"io task","client":"1","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158732024-04-17T17:20:24.872Z{"msg":"ds_connection connected from Ok(127.0.0.1:38695)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.475545672Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"":"io task","client":"0","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158742024-04-17T17:20:24.872Z{"msg":"ds_connection connected from Ok(127.0.0.1:56047)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.475574114Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"":"io task","client":"2","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158752024-04-17T17:20:24.872Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.476388903Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"client":"1","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158762024-04-17T17:20:24.872Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.476432716Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"client":"1","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158772024-04-17T17:20:24.872Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.476450907Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"client":"2","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158782024-04-17T17:20:24.872Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.476475458Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"client":"2","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158792024-04-17T17:20:24.872Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.47650641Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"client":"0","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158802024-04-17T17:20:24.872Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.476525271Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"client":"0","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158812024-04-17T17:20:24.873Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.477053863Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"client":"1","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158822024-04-17T17:20:24.873Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.477090336Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"client":"2","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158832024-04-17T17:20:24.873Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.477106586Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"client":"0","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158842024-04-17T17:20:24.873Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.477577195Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"client":"1","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158852024-04-17T17:20:24.873Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.477631168Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"client":"1","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158862024-04-17T17:20:24.873Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.477688952Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158872024-04-17T17:20:24.873Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.477717134Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"client":"2","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158882024-04-17T17:20:24.873Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.477741835Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"client":"2","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158892024-04-17T17:20:24.873Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.477756326Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158902024-04-17T17:20:24.873Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.477775687Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"client":"0","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158912024-04-17T17:20:24.873Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.477789308Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"client":"0","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158922024-04-17T17:20:24.873Z{"msg":"[0]R flush_numbers[0..12]: [265, 268, 262, 260, 267, 268, 266, 259, 268, 268, 261, 265]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.477809649Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"":"downstairs","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158932024-04-17T17:20:24.873Z{"msg":"[0]R generation[0..12]: [145, 147, 144, 142, 146, 147, 146, 142, 147, 147, 143, 145]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.47782417Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"":"downstairs","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158942024-04-17T17:20:24.873Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.477844771Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"":"downstairs","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158952024-04-17T17:20:24.873Z{"msg":"[1]R flush_numbers[0..12]: [265, 268, 262, 260, 267, 268, 266, 259, 268, 268, 261, 265]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.477859142Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"":"downstairs","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158962024-04-17T17:20:24.873Z{"msg":"[1]R generation[0..12]: [145, 147, 144, 142, 146, 147, 146, 142, 147, 147, 143, 145]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.477879763Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"":"downstairs","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158972024-04-17T17:20:24.873Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.477894324Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"":"downstairs","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158982024-04-17T17:20:24.873Z{"msg":"[2]R flush_numbers[0..12]: [265, 268, 262, 260, 267, 268, 266, 259, 268, 268, 261, 265]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.477916286Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"":"downstairs","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
158992024-04-17T17:20:24.873Z{"msg":"[2]R generation[0..12]: [145, 147, 144, 142, 146, 147, 146, 142, 147, 147, 143, 145]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.477930136Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"":"downstairs","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
159002024-04-17T17:20:24.873Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
159012024-04-17T17:20:24.873Z,"time":"2024-04-17T17:20:23.477943927Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"":"downstairs","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
159022024-04-17T17:20:24.873Z{"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.477956688Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"":"downstairs","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
159032024-04-17T17:20:24.873Z{"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.477969529Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"":"downstairs","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
159042024-04-17T17:20:24.874Z{"msg":"Next flush: 269","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.47798891Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"":"downstairs","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
159052024-04-17T17:20:24.874Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.478001941Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"":"downstairs","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
159062024-04-17T17:20:24.874Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.478014701Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
159072024-04-17T17:20:24.874Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.478033433Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"client":"0","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
159082024-04-17T17:20:24.874Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.478047014Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"client":"1","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
159092024-04-17T17:20:24.874Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.478060224Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"client":"2","":"downstairs","session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
159102024-04-17T17:20:24.874Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.478073185Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
159112024-04-17T17:20:24.874Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.478085686Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
159122024-04-17T17:20:24.874Z{"msg":"aa3d34a4-104d-4980-bada-c37d8b22b472 is now active with session: f563a4c8-7a5f-4814-9d99-6f01a80ef4cc","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.478098217Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
159132024-04-17T17:20:24.874Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.478109777Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763,"session_id":"f563a4c8-7a5f-4814-9d99-6f01a80ef4cc"}
159142024-04-17T17:20:24.874Z{"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:23.478122588Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763}
159152024-04-17T17:20:24.874ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
159162024-04-17T17:20:24.874ZLoading write count information from file "/tmp/test_repair_verify.data"
159172024-04-17T17:20:24.874ZRun Repair workload
159182024-04-17T17:20:24.878Z01/30 Write block 593 len 4096 data: 8
159192024-04-17T17:20:24.878Z02/30 Read block 395 len 8192
159202024-04-17T17:20:24.878Z03/30 Read block 205 len 40960
159212024-04-17T17:20:24.878Z04/30 Read block 406 len 40960
159222024-04-17T17:20:24.878Z05/30 Read block 573 len 28672
159232024-04-17T17:20:24.878Z06/30 Read block 95 len 40960
159242024-04-17T17:20:24.878Z07/30 Read block 490 len 36864
159252024-04-17T17:20:24.878Z08/30 Read block 327 len 24576
159262024-04-17T17:20:24.878Z09/30 Read block 107 len 28672
159272024-04-17T17:20:24.878Z10/30 Write block 71 len 8192 data: 10 10
159282024-04-17T17:20:24.882Z11/30 Write block 539 len 16384 data: 11 11 12 15
159292024-04-17T17:20:24.882Z12/30 Flush
159302024-04-17T17:20:24.882Z13/30 Write block 74 len 32768 data: 10 12 11 11 10 10 13 11
159312024-04-17T17:20:24.882Z14/30 Read block 388 len 20480
159322024-04-17T17:20:24.885Z15/30 Read block 406 len 20480
159332024-04-17T17:20:24.885Z16/30 Read block 78 len 32768
159342024-04-17T17:20:24.885Z17/30 Read block 83 len 32768
159352024-04-17T17:20:24.888Z18/30 Read block 577 len 20480
159362024-04-17T17:20:24.888Z19/30 Write block 372 len 16384 data: 11 12 9 9
159372024-04-17T17:20:24.888Z20/30 Read block 275 len 32768
159382024-04-17T17:20:24.889Z21/30 Write block 171 len 32768 data: 13 12 13 10 12 12 11 14
159392024-04-17T17:20:24.892Z22/30 Write block 267 len 32768 data: 11 12 15 12 15 14 16 13
159402024-04-17T17:20:24.892Z23/30 Read block 184 len 24576
159412024-04-17T17:20:24.892Z24/30 Write block 204 len 40960 data: 19 18 17 20 11 12 13 13 13 13
159422024-04-17T17:20:24.896Z25/30 Write block 526 len 24576 data: 11 11 12 13 12 11
159432024-04-17T17:20:24.896Z26/30 Write block 413 len 16384 data: 14 13 12 13
159442024-04-17T17:20:24.896Z27/30 Read block 28 len 20480
159452024-04-17T17:20:24.896Z28/30 Read block 47 len 12288
159462024-04-17T17:20:24.899Z29/30 Read block 320 len 32768
159472024-04-17T17:20:24.899Z30/30 Read block 575 len 36864
159482024-04-17T17:20:24.899Z----------------------------------------------------------------
159492024-04-17T17:20:24.900Z Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30
159502024-04-17T17:20:24.900ZStates: Active Active Active
159512024-04-17T17:20:24.900ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
159522024-04-17T17:20:24.900Z 1 Acked 1000 Write 1 Sent Done Done false
159532024-04-17T17:20:24.900Z 2 Acked 1001 Read 2 Sent Done Done false
159542024-04-17T17:20:24.900Z 3 Acked 1002 Read 10 Sent Done Done false
159552024-04-17T17:20:24.900Z 4 Acked 1003 Read 10 Sent Done Done false
159562024-04-17T17:20:24.900Z 5 Acked 1004 Read 7 Sent Done Done false
159572024-04-17T17:20:24.900Z 6 Acked 1005 Read 10 Sent Done Done false
159582024-04-17T17:20:24.900Z 7 Acked 1006 Read 9 Sent Done Done false
159592024-04-17T17:20:24.900Z 8 Acked 1007 Read 6 Sent Done Done false
159602024-04-17T17:20:24.900Z 9 Acked 1008 Read 7 Sent Done Done false
159612024-04-17T17:20:24.900Z 10 Acked 1009 Write 2 Sent Done Done false
159622024-04-17T17:20:24.900Z 11 Acked 1010 Write 4 Sent Done Done false
159632024-04-17T17:20:24.900Z 12 Acked 1011 Flush 0 Sent Done Done false
159642024-04-17T17:20:24.900Z 13 Acked 1012 Write 8 Sent Done Done false
159652024-04-17T17:20:24.900Z 14 Acked 1013 Read 5 Sent Done Done false
159662024-04-17T17:20:24.900Z 15 Acked 1014 Read 5 Sent Done Done false
159672024-04-17T17:20:24.900Z 16 Acked 1015 Read 8 Sent Done Done false
159682024-04-17T17:20:24.900Z 17 Acked 1016 Read 8 Sent Done Done false
159692024-04-17T17:20:24.900Z 18 Acked 1017 Read 5 Sent Done Done false
159702024-04-17T17:20:24.900Z 19 Acked 1018 Write 4 Sent Done Done false
159712024-04-17T17:20:24.901Z 20 Acked 1019 Read 8 Sent Done Done false
159722024-04-17T17:20:24.901Z 21 Acked 1020 Write 8 Sent Done Done false
159732024-04-17T17:20:24.901Z 22 Acked 1021 Write 8 Sent Done Done false
159742024-04-17T17:20:24.901Z 23 Acked 1022 Read 6 Sent Done Done false
159752024-04-17T17:20:24.901Z 24 Acked 1023 Write 10 Sent Done Done false
159762024-04-17T17:20:24.901Z 25 Acked 1024 Write 6 Sent Done Done false
159772024-04-17T17:20:24.901Z 26 Acked 1025 Write 4 Sent Done Done false
159782024-04-17T17:20:24.901Z 27 Acked 1026 Read 5 Sent Done Done false
159792024-04-17T17:20:24.901Z 28 Acked 1027 Read 3 Sent Done Done false
159802024-04-17T17:20:24.901Z 29 Acked 1028 Read 8 Sent Done Done false
159812024-04-17T17:20:24.901Z 30 Acked 1029 Read 9 Sent Done Sent false
159822024-04-17T17:20:24.901Z STATES DS:0 DS:1 DS:2 TOTAL
159832024-04-17T17:20:24.901Z New 0 0 0 0
159842024-04-17T17:20:24.901Z Sent 30 0 1 31
159852024-04-17T17:20:24.901Z Done 0 30 29 59
159862024-04-17T17:20:24.901Z Skipped 0 0 0 0
159872024-04-17T17:20:24.901Z Error 0 0 0 0
159882024-04-17T17:20:24.901ZLast Flush: 0 1011 1011
159892024-04-17T17:20:24.901ZDownstairs last five completed:
159902024-04-17T17:20:24.901ZUpstairs last five completed: 30 29 28 27 26
159912024-04-17T17:20:24.901Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:20:23.507270455Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1763}
159922024-04-17T17:20:24.901ZWrote out file "/tmp/test_repair_verify.data"
159932024-04-17T17:20:24.905Z
159942024-04-17T17:20:24.909ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
159952024-04-17T17:20:24.925ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
159962024-04-17T17:20:24.925Z 0 000-019 145 145 145 265 265 265 F F F
159972024-04-17T17:20:24.925Z 1 020-039 147 147 147 268 268 268 F F F
159982024-04-17T17:20:24.925Z 2 040-059 144 144 144 262 262 262 F F F
159992024-04-17T17:20:24.925Z 3 060-079 142 148 148 260 269 269 F T T <---
160002024-04-17T17:20:24.925Z 4 080-099 146 146 146 267 267 267 F T T <---
160012024-04-17T17:20:24.925Z 5 100-119 147 147 147 268 268 268 F F F
160022024-04-17T17:20:24.925Z 6 120-139 146 146 146 266 266 266 F F F
160032024-04-17T17:20:24.925Z 7 140-159 142 142 142 259 259 259 F F F
160042024-04-17T17:20:24.925Z 8 160-179 147 147 147 268 268 268 F T T <---
160052024-04-17T17:20:24.929Z 9 180-199 147 147 147 268 268 268 F F F
160062024-04-17T17:20:24.929Z 10 200-219 143 143 143 261 261 261 F T T <---
160072024-04-17T17:20:24.929Z 11 220-239 145 145 145 265 265 265 F F F
160082024-04-17T17:20:24.929Z 12 240-259 145 145 145 265 265 265 F F F
160092024-04-17T17:20:24.929Z 13 260-279 142 142 142 259 259 259 F T T <---
160102024-04-17T17:20:24.929Z 14 280-299 147 147 147 268 268 268 F F F
160112024-04-17T17:20:24.929Z 15 300-319 144 144 144 264 264 264 F F F
160122024-04-17T17:20:24.929Z 16 320-339 147 147 147 268 268 268 F F F
160132024-04-17T17:20:24.929Z 17 340-359 147 147 147 268 268 268 F F F
160142024-04-17T17:20:24.929Z 18 360-379 145 145 145 265 265 265 F T T <---
160152024-04-17T17:20:24.929Z 19 380-399 144 144 144 263 263 263 F F F
160162024-04-17T17:20:24.929Z 20 400-419 137 137 137 248 248 248 F T T <---
160172024-04-17T17:20:24.929Z 21 420-439 146 146 146 267 267 267 F F F
160182024-04-17T17:20:24.929Z 22 440-459 146 146 146 267 267 267 F F F
160192024-04-17T17:20:24.929Z 23 460-479 143 143 143 261 261 261 F F F
160202024-04-17T17:20:24.929Z 24 480-499 146 146 146 267 267 267 F F F
160212024-04-17T17:20:24.929Z 25 500-519 146 146 146 267 267 267 F F F
160222024-04-17T17:20:24.929Z 26 520-539 147 148 148 268 269 269 F T T <---
160232024-04-17T17:20:24.929Z 27 540-559 142 148 148 259 269 269 F F F <---
160242024-04-17T17:20:24.930Z 28 560-579 146 146 146 267 267 267 F F F
160252024-04-17T17:20:24.930Z 29 580-599 145 148 148 265 269 269 F F F <---
160262024-04-17T17:20:24.930ZMax gen: 148, Max flush: 269
160272024-04-17T17:20:24.930ZError: Difference in extent metadata found!
160282024-04-17T17:20:24.930ZA Difference in extent metadata is expected here
160292024-04-17T17:20:24.930ZOn loop 74
160302024-04-17T17:20:24.930Z
160312024-04-17T17:20:24.933ZVerifying data now
160322024-04-17T17:20:29.428ZNo change after verify
160332024-04-17T17:20:29.429ZLoop: 74 Downstairs dump after verify (and repair):
160342024-04-17T17:20:29.443ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
160352024-04-17T17:20:29.443Z 0 000-019 145 145 145 265 265 265 F F F
160362024-04-17T17:20:29.447Z 1 020-039 147 147 147 268 268 268 F F F
160372024-04-17T17:20:29.447Z 2 040-059 144 144 144 262 262 262 F F F
160382024-04-17T17:20:29.447Z 3 060-079 149 149 149 270 270 270 F F F
160392024-04-17T17:20:29.447Z 4 080-099 149 149 149 270 270 270 F F F
160402024-04-17T17:20:29.447Z 5 100-119 147 147 147 268 268 268 F F F
160412024-04-17T17:20:29.447Z 6 120-139 146 146 146 266 266 266 F F F
160422024-04-17T17:20:29.447Z 7 140-159 142 142 142 259 259 259 F F F
160432024-04-17T17:20:29.447Z 8 160-179 149 149 149 270 270 270 F F F
160442024-04-17T17:20:29.447Z 9 180-199 147 147 147 268 268 268 F F F
160452024-04-17T17:20:29.447Z 10 200-219 149 149 149 270 270 270 F F F
160462024-04-17T17:20:29.447Z 11 220-239 145 145 145 265 265 265 F F F
160472024-04-17T17:20:29.447Z 12 240-259 145 145 145 265 265 265 F F F
160482024-04-17T17:20:29.447Z 13 260-279 149 149 149 270 270 270 F F F
160492024-04-17T17:20:29.447Z 14 280-299 147 147 147 268 268 268 F F F
160502024-04-17T17:20:29.447Z 15 300-319 144 144 144 264 264 264 F F F
160512024-04-17T17:20:29.447Z 16 320-339 147 147 147 268 268 268 F F F
160522024-04-17T17:20:29.447Z 17 340-359 147 147 147 268 268 268 F F F
160532024-04-17T17:20:29.448Z 18 360-379 149 149 149 270 270 270 F F F
160542024-04-17T17:20:29.448Z 19 380-399 144 144 144 263 263 263 F F F
160552024-04-17T17:20:29.448Z 20 400-419 149 149 149 270 270 270 F F F
160562024-04-17T17:20:29.448Z 21 420-439 146 146 146 267 267 267 F F F
160572024-04-17T17:20:29.448Z 22 440-459 146 146 146 267 267 267 F F F
160582024-04-17T17:20:29.448Z 23 460-479 143 143 143 261 261 261 F F F
160592024-04-17T17:20:29.448Z 24 480-499 146 146 146 267 267 267 F F F
160602024-04-17T17:20:29.448Z 25 500-519 146 146 146 267 267 267 F F F
160612024-04-17T17:20:29.448Z 26 520-539 149 149 149 270 270 270 F F F
160622024-04-17T17:20:29.448Z 27 540-559 148 148 148 269 269 269 F F F
160632024-04-17T17:20:29.448Z 28 560-579 146 146 146 267 267 267 F F F
160642024-04-17T17:20:29.448Z 29 580-599 148 148 148 269 269 269 F F F
160652024-04-17T17:20:29.448ZMax gen: 149, Max flush: 270
160662024-04-17T17:20:29.448Z
160672024-04-17T17:20:29.448ZBegin loop 75
160682024-04-17T17:20:29.448ZDownstairs to restart: 1
160692024-04-17T17:20:29.465Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.075030675Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771}
160702024-04-17T17:20:29.471Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.075330103Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771}
160712024-04-17T17:20:29.471Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30Crucible runtime is spawned
160722024-04-17T17:20:29.471Z,"time":"2024-04-17T17:20:28.075369936Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771}
160732024-04-17T17:20:29.471Z{"msg":"Crucible c7ad4da8-4b52-4c65-b684-1272eedbe8b9 has session id: 5a18ad2e-9b44-480a-a8b9-5882f800803c","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.075428389Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
160742024-04-17T17:20:29.471Z{"msg":"Upstairs opts: Upstairs UUID: c7ad4da8-4b52-4c65-b684-1272eedbe8b9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.075494853Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
160752024-04-17T17:20:29.471Z{"msg":"Crucible stats registered with UUID: c7ad4da8-4b52-4c65-b684-1272eedbe8b9","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.075536606Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
160762024-04-17T17:20:29.471Z{"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.075560227Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771}
160772024-04-17T17:20:29.471Z{"msg":"c7ad4da8-4b52-4c65-b684-1272eedbe8b9 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.075580398Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
160782024-04-17T17:20:29.471Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.07561035Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"client":"0","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
160792024-04-17T17:20:29.471Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.075635372Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"client":"0","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
160802024-04-17T17:20:29.471Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.075671404Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"client":"1","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
160812024-04-17T17:20:29.471Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.075697295Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"client":"1","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
160822024-04-17T17:20:29.471Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.075740348Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"client":"2","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
160832024-04-17T17:20:29.471Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.07576868Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"client":"2","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
160842024-04-17T17:20:29.472Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.075804042Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"":"io task","client":"0","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
160852024-04-17T17:20:29.472Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.075890247Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"":"io task","client":"2","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
160862024-04-17T17:20:29.472Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.07594854Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
160872024-04-17T17:20:29.472Z{"msg":"ds_connection connected from Ok(127.0.0.1:32860)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.075983343Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"":"io task","client":"0","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
160882024-04-17T17:20:29.472Z{"msg":"ds_connection connected from Ok(127.0.0.1:46331)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.076039676Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"":"io task","client":"2","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
160892024-04-17T17:20:29.472Z{"msg":"ds_connection connected from Ok(127.0.0.1:57025)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.076069638Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
160902024-04-17T17:20:29.472Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.076696976Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"client":"2","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
160912024-04-17T17:20:29.472Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.076733338Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"client":"2","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
160922024-04-17T17:20:29.472Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.076750219Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"client":"0","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
160932024-04-17T17:20:29.472Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.076777661Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"client":"0","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
160942024-04-17T17:20:29.472Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.076798172Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"client":"1","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
160952024-04-17T17:20:29.472Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.076837144Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"client":"1","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
160962024-04-17T17:20:29.472Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.077271121Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"client":"2","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
160972024-04-17T17:20:29.472Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.077412789Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"client":"0","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
160982024-04-17T17:20:29.472Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.077450531Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"client":"1","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
160992024-04-17T17:20:29.472Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.077744239Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"client":"2","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
161002024-04-17T17:20:29.472Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.077788712Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"client":"2","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
161012024-04-17T17:20:29.472Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.077814664Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
161022024-04-17T17:20:29.472Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.077943721Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"client":"0","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
161032024-04-17T17:20:29.472Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.078011316Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"client":"0","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
161042024-04-17T17:20:29.472Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.078044858Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
161052024-04-17T17:20:29.472Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.078058258Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"client":"1","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
161062024-04-17T17:20:29.472Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.07807854Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"client":"1","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
161072024-04-17T17:20:29.473Z{"msg":"[0]R flush_numbers[0..12]: [265, 268, 262, 270, 270, 268, 266, 259, 270, 268, 270, 265]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.078092921Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"":"downstairs","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
161082024-04-17T17:20:29.473Z{"msg":"[0]R generation[0..12]: [145, 147, 144, 149, 149, 147, 146, 142, 149, 147, 149, 145]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.078114102Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"":"downstairs","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
161092024-04-17T17:20:29.473Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.078128643Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"":"downstairs","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
161102024-04-17T17:20:29.473Z{"msg":"[1]R flush_numbers[0..12]: [265, 268, 262, 270, 270, 268, 266, 259, 270, 268, 270, 265]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.078149094Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"":"downstairs","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
161112024-04-17T17:20:29.473Z{"msg":"[1]R generation[0..12]: [145, 147, 144, 149, 149, 147, 146, 142, 149, 147, 149, 145]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.078163195Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"":"downstairs","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
161122024-04-17T17:20:29.473Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.078183776Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"":"downstairs","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
161132024-04-17T17:20:29.473Z{"msg":"Wait for a query_work_queue command to finish before sending IO
161142024-04-17T17:20:29.473Z[2]R flush_numbers[0..12]: [265, 268, 262, 270, 270, 268, 266, 259, 270, 268, 270, 265]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.07824744Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"":"downstairs","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
161152024-04-17T17:20:29.473Z{"msg":"[2]R generation[0..12]: [145, 147, 144, 149, 149, 147, 146, 142, 149, 147, 149, 145]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.078285272Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"":"downstairs","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
161162024-04-17T17:20:29.473Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.078348346Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"":"downstairs","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
161172024-04-17T17:20:29.473Z{"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.078382598Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"":"downstairs","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
161182024-04-17T17:20:29.473Z{"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.07841806Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"":"downstairs","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
161192024-04-17T17:20:29.473Z{"msg":"Next flush: 271","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.078457483Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"":"downstairs","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
161202024-04-17T17:20:29.473Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.078496165Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"":"downstairs","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
161212024-04-17T17:20:29.473Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
161222024-04-17T17:20:29.473Z,"time":"2024-04-17T17:20:28.078534247Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
161232024-04-17T17:20:29.473Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.078573039Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"client":"0","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
161242024-04-17T17:20:29.473Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.078597441Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"client":"1","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
161252024-04-17T17:20:29.473Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.078611082Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"client":"2","":"downstairs","session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
161262024-04-17T17:20:29.473Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.078638063Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
161272024-04-17T17:20:29.473Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.078666935Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
161282024-04-17T17:20:29.480Z{"msg":"c7ad4da8-4b52-4c65-b684-1272eedbe8b9 is now active with session: 5a18ad2e-9b44-480a-a8b9-5882f800803c","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.078689887Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
161292024-04-17T17:20:29.480Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.078702067Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771,"session_id":"5a18ad2e-9b44-480a-a8b9-5882f800803c"}
161302024-04-17T17:20:29.480Z{"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:28.078714038Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771}
161312024-04-17T17:20:29.480ZLoading write count information from file "/tmp/test_repair_verify.data"
161322024-04-17T17:20:29.480ZRun Repair workload
161332024-04-17T17:20:29.480Z01/30 Write block 187 len 28672 data: 15 18 17 13 11 10 9
161342024-04-17T17:20:29.480Z02/30 Write block 19 len 32768 data: 11 12 15 15 13 12 11 10
161352024-04-17T17:20:29.480Z03/30 Write block 11 len 16384 data: 8 10 11 12
161362024-04-17T17:20:29.480Z04/30 Write block 344 len 40960 data: 16 17 16 12 11 11 12 7 8 9
161372024-04-17T17:20:29.481Z05/30 Read block 164 len 28672
161382024-04-17T17:20:29.481Z06/30 Read block 367 len 28672
161392024-04-17T17:20:29.481Z07/30 Read block 540 len 16384
161402024-04-17T17:20:29.484Z08/30 Write block 314 len 16384 data: 14 13 12 13
161412024-04-17T17:20:29.484Z09/30 Read block 553 len 32768
161422024-04-17T17:20:29.484Z10/30 Write block 267 len 16384 data: 12 13 16 13
161432024-04-17T17:20:29.484Z11/30 Write block 244 len 4096 data: 13
161442024-04-17T17:20:29.484Z12/30 Read block 211 len 40960
161452024-04-17T17:20:29.484Z13/30 Write block 46 len 8192 data: 10 9
161462024-04-17T17:20:29.484Z14/30 Read block 370 len 8192
161472024-04-17T17:20:29.484Z15/30 Write block 273 len 20480 data: 17 14 12 13 7
161482024-04-17T17:20:29.485Z16/30 Read block 541 len 40960
161492024-04-17T17:20:29.488Z17/30 Write block 80 len 16384 data: 14 12 13 16
161502024-04-17T17:20:29.488Z18/30 Write block 235 len 8192 data: 15 13
161512024-04-17T17:20:29.488Z19/30 Write block 213 len 16384 data: 14 12 14 13
161522024-04-17T17:20:29.488Z20/30 Write block 136 len 20480 data: 11 11 10 10 9
161532024-04-17T17:20:29.488Z21/30 Read block 575 len 8192
161542024-04-17T17:20:29.491Z22/30 Read block 466 len 36864
161552024-04-17T17:20:29.492Z23/30 Read block 274 len 32768
161562024-04-17T17:20:29.492Z24/30 Flush
161572024-04-17T17:20:29.505Z25/30 Write block 587 len 32768 data: 13 8 7 8 6 10 9 8
161582024-04-17T17:20:29.505Z26/30 Flush
161592024-04-17T17:20:29.505Z27/30 Write block 80 len 20480 data: 15 13 14 17 13
161602024-04-17T17:20:29.505Z28/30 Write block 124 len 36864 data: 9 11 11 11 8 9 11 12 12
161612024-04-17T17:20:29.505Z29/30 Read block 84 len 28672
161622024-04-17T17:20:29.505Z30/30 Write block 370 len 24576 data: 13 14 12 13 10 10
161632024-04-17T17:20:29.506Z----------------------------------------------------------------
161642024-04-17T17:20:29.506Z Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:30
161652024-04-17T17:20:29.506ZStates: Active Active Active
161662024-04-17T17:20:29.506ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
161672024-04-17T17:20:29.506Z 1 Acked 1000 Write 7 Done Done Done false
161682024-04-17T17:20:29.506Z 2 Acked 1001 Write 8 Done Done Done false
161692024-04-17T17:20:29.506Z 3 Acked 1002 Write 4 Done Done Done false
161702024-04-17T17:20:29.506Z 4 Acked 1003 Write 10 Done Sent Done false
161712024-04-17T17:20:29.506Z 5 Acked 1004 Read 7 Done Sent Done false
161722024-04-17T17:20:29.506Z 6 Acked 1005 Read 7 Done Sent Done false
161732024-04-17T17:20:29.506Z 7 Acked 1006 Read 4 Done Sent Done false
161742024-04-17T17:20:29.506Z 8 Acked 1007 Write 4 Done Sent Done false
161752024-04-17T17:20:29.506Z 9 Acked 1008 Read 8 Done Sent Done false
161762024-04-17T17:20:29.506Z 10 Acked 1009 Write 4 Done Sent Done false
161772024-04-17T17:20:29.506Z 11 Acked 1010 Write 1 Done Sent Done false
161782024-04-17T17:20:29.506Z 12 Acked 1011 Read 10 Done Sent Done false
161792024-04-17T17:20:29.506Z 13 Acked 1012 Write 2 Done Sent Done false
161802024-04-17T17:20:29.506Z 14 Acked 1013 Read 2 Done Sent Done false
161812024-04-17T17:20:29.506Z 15 Acked 1014 Write 5 Done Sent Done false
161822024-04-17T17:20:29.506Z 16 Acked 1015 Read 10 Done Sent Done false
161832024-04-17T17:20:29.506Z 17 Acked 1016 Write 4 Done Sent Done false
161842024-04-17T17:20:29.506Z 18 Acked 1017 Write 2 Done Sent Done false
161852024-04-17T17:20:29.506Z 19 Acked 1018 Write 4 Done Sent Done false
161862024-04-17T17:20:29.506Z 20 Acked 1019 Write 5 Done Sent Done false
161872024-04-17T17:20:29.507Z 21 Acked 1020 Read 2 Done Sent Done false
161882024-04-17T17:20:29.507Z 22 Acked 1021 Read 9 Done Sent Done false
161892024-04-17T17:20:29.507Z 23 Acked 1022 Read 8 Done Sent Done false
161902024-04-17T17:20:29.507Z 24 Acked 1023 Flush 0 Done Sent Done false
161912024-04-17T17:20:29.507Z 25 Acked 1024 Write 8 Done Sent Done false
161922024-04-17T17:20:29.507Z 26 Acked 1025 Flush 0 Done Sent Done false
161932024-04-17T17:20:29.507Z 27 Acked 1026 Write 5 Done Sent Done false
161942024-04-17T17:20:29.507Z 28 Acked 1027 Write 9 Done Sent Done false
161952024-04-17T17:20:29.507Z 29 Acked 1028 Read 7 Done Sent Done false
161962024-04-17T17:20:29.507Z 30 Acked 1029 Write 6 Sent Sent Sent false
161972024-04-17T17:20:29.507Z STATES DS:0 DS:1 DS:2 TOTAL
161982024-04-17T17:20:29.507Z New 0 0 0 0
161992024-04-17T17:20:29.507Z Sent 1 27 1 29
162002024-04-17T17:20:29.507Z Done 29 3 29 61
162012024-04-17T17:20:29.507Z Skipped 0 0 0 0
162022024-04-17T17:20:29.507Z Error 0 0 0 0
162032024-04-17T17:20:29.507ZLast Flush: 1025 0 1025
162042024-04-17T17:20:29.507ZDownstairs last five completed:
162052024-04-17T17:20:29.507ZUpstairs last five completed: 30 29 28 27 26
162062024-04-17T17:20:29.507Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:20:28.112264561Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1771}
162072024-04-17T17:20:29.507ZWrote out file "/tmp/test_repair_verify.data"
162082024-04-17T17:20:29.512Z
162092024-04-17T17:20:29.513ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
162102024-04-17T17:20:29.526ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
162112024-04-17T17:20:29.526Z 0 000-019 150 145 150 271 265 271 F T F <---
162122024-04-17T17:20:29.526Z 1 020-039 150 147 150 271 268 271 F T F <---
162132024-04-17T17:20:29.526Z 2 040-059 150 144 150 271 262 271 F F F <---
162142024-04-17T17:20:29.526Z 3 060-079 149 149 149 270 270 270 F F F
162152024-04-17T17:20:29.526Z 4 080-099 150 149 150 271 270 271 T F T <---
162162024-04-17T17:20:29.526Z 5 100-119 147 147 147 268 268 268 F F F
162172024-04-17T17:20:29.526Z 6 120-139 150 146 150 271 266 271 T F T <---
162182024-04-17T17:20:29.526Z 7 140-159 150 142 150 271 259 271 F F F <---
162192024-04-17T17:20:29.527Z 8 160-179 149 149 149 270 270 270 F F F
162202024-04-17T17:20:29.527Z 9 180-199 150 147 150 271 268 271 F T F <---
162212024-04-17T17:20:29.530Z 10 200-219 150 149 150 271 270 271 F F F <---
162222024-04-17T17:20:29.530Z 11 220-239 150 145 150 271 265 271 F F F <---
162232024-04-17T17:20:29.530Z 12 240-259 150 145 150 271 265 271 F F F <---
162242024-04-17T17:20:29.530Z 13 260-279 150 149 150 271 270 271 F F F <---
162252024-04-17T17:20:29.530Z 14 280-299 147 147 147 268 268 268 F F F
162262024-04-17T17:20:29.530Z 15 300-319 150 144 150 271 264 271 F F F <---
162272024-04-17T17:20:29.530Z 16 320-339 147 147 147 268 268 268 F F F
162282024-04-17T17:20:29.530Z 17 340-359 150 147 150 271 268 271 F F F <---
162292024-04-17T17:20:29.530Z 18 360-379 149 149 149 270 270 270 T F T <---
162302024-04-17T17:20:29.530Z 19 380-399 144 144 144 263 263 263 F F F
162312024-04-17T17:20:29.530Z 20 400-419 149 149 149 270 270 270 F F F
162322024-04-17T17:20:29.531Z 21 420-439 146 146 146 267 267 267 F F F
162332024-04-17T17:20:29.531Z 22 440-459 146 146 146 267 267 267 F F F
162342024-04-17T17:20:29.531Z 23 460-479 143 143 143 261 261 261 F F F
162352024-04-17T17:20:29.531Z 24 480-499 146 146 146 267 267 267 F F F
162362024-04-17T17:20:29.531Z 25 500-519 146 146 146 267 267 267 F F F
162372024-04-17T17:20:29.531Z 26 520-539 149 149 149 270 270 270 F F F
162382024-04-17T17:20:29.531Z 27 540-559 148 148 148 269 269 269 F F F
162392024-04-17T17:20:29.531Z 28 560-579 146 146 146 267 267 267 F F F
162402024-04-17T17:20:29.531Z 29 580-599 150 148 150 272 269 272 F F F <---
162412024-04-17T17:20:29.531ZMax gen: 150, Max flush: 272
162422024-04-17T17:20:29.531ZError: Difference in extent metadata found!
162432024-04-17T17:20:29.531ZA Difference in extent metadata is expected here
162442024-04-17T17:20:29.531ZOn loop 75
162452024-04-17T17:20:29.531Z
162462024-04-17T17:20:29.535ZVerifying data now
162472024-04-17T17:20:44.303ZNo change after verify
162482024-04-17T17:20:44.304ZLoop: 75 Downstairs dump after verify (and repair):
162492024-04-17T17:20:44.304ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
162502024-04-17T17:20:44.304Z 0 000-019 150 150 150 271 271 271 F F F
162512024-04-17T17:20:44.304Z 1 020-039 150 150 150 271 271 271 F F F
162522024-04-17T17:20:44.304Z 2 040-059 150 150 150 271 271 271 F F F
162532024-04-17T17:20:44.304Z 3 060-079 149 149 149 270 270 270 F F F
162542024-04-17T17:20:44.304Z 4 080-099 151 151 151 273 273 273 F F F
162552024-04-17T17:20:44.304Z 5 100-119 147 147 147 268 268 268 F F F
162562024-04-17T17:20:44.304Z 6 120-139 151 151 151 273 273 273 F F F
162572024-04-17T17:20:44.304Z 7 140-159 150 150 150 271 271 271 F F F
162582024-04-17T17:20:44.304Z 8 160-179 149 149 149 270 270 270 F F F
162592024-04-17T17:20:44.304Z 9 180-199 150 150 150 271 271 271 F F F
162602024-04-17T17:20:44.304Z 10 200-219 150 150 150 271 271 271 F F F
162612024-04-17T17:20:44.305Z 11 220-239 150 150 150 271 271 271 F F F
162622024-04-17T17:20:44.305Z 12 240-259 150 150 150 271 271 271 F F F
162632024-04-17T17:20:44.305Z 13 260-279 150 150 150 271 271 271 F F F
162642024-04-17T17:20:44.305Z 14 280-299 147 147 147 268 268 268 F F F
162652024-04-17T17:20:44.305Z 15 300-319 150 150 150 271 271 271 F F F
162662024-04-17T17:20:44.305Z 16 320-339 147 147 147 268 268 268 F F F
162672024-04-17T17:20:44.305Z 17 340-359 150 150 150 271 271 271 F F F
162682024-04-17T17:20:44.305Z 18 360-379 151 151 151 273 273 273 F F F
162692024-04-17T17:20:44.305Z 19 380-399 144 144 144 263 263 263 F F F
162702024-04-17T17:20:44.305Z 20 400-419 149 149 149 270 270 270 F F F
162712024-04-17T17:20:44.305Z 21 420-439 146 146 146 267 267 267 F F F
162722024-04-17T17:20:44.305Z 22 440-459 146 146 146 267 267 267 F F F
162732024-04-17T17:20:44.305Z 23 460-479 143 143 143 261 261 261 F F F
162742024-04-17T17:20:44.305Z 24 480-499 146 146 146 267 267 267 F F F
162752024-04-17T17:20:44.305Z 25 500-519 146 146 146 267 267 267 F F F
162762024-04-17T17:20:44.305Z 26 520-539 149 149 149 270 270 270 F F F
162772024-04-17T17:20:44.305Z 27 540-559 148 148 148 269 269 269 F F F
162782024-04-17T17:20:44.305Z 28 560-579 146 146 146 267 267 267 F F F
162792024-04-17T17:20:44.305Z 29 580-599 150 150 150 272 272 272 F F F
162802024-04-17T17:20:44.305ZMax gen: 151, Max flush: 273
162812024-04-17T17:20:44.313Z
162822024-04-17T17:20:44.313ZBegin loop 76
162832024-04-17T17:20:44.313ZDownstairs to restart: 1
162842024-04-17T17:20:44.324Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.930327862Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779}
162852024-04-17T17:20:44.335Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.930600518Z","Crucible runtime is spawned
162862024-04-17T17:20:44.335Zhostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779}
162872024-04-17T17:20:44.335Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.930725866Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779}
162882024-04-17T17:20:44.335Z{"msg":"Crucible ae1e2a3a-f710-4087-b4a3-a5a509fb6055 has session id: 208e885e-98af-49fa-8614-604b8f808184","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.930771138Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"session_id":"208e885e-98af-49fa-8614-604b8f808184"}
162892024-04-17T17:20:44.335Z{"msg":"Upstairs opts: Upstairs UUID: ae1e2a3a-f710-4087-b4a3-a5a509fb6055, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.930835072Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"session_id":"208e885e-98af-49fa-8614-604b8f808184"}
162902024-04-17T17:20:44.335Z{"msg":"Crucible stats registered with UUID: ae1e2a3a-f710-4087-b4a3-a5a509fb6055","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.930885155Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"session_id":"208e885e-98af-49fa-8614-604b8f808184"}
162912024-04-17T17:20:44.336Z{"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.930914797Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779}
162922024-04-17T17:20:44.336Z{"msg":"ae1e2a3a-f710-4087-b4a3-a5a509fb6055 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.930939388Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"session_id":"208e885e-98af-49fa-8614-604b8f808184"}
162932024-04-17T17:20:44.336Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.930976631Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"client":"0","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
162942024-04-17T17:20:44.336Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.931016533Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"client":"0","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
162952024-04-17T17:20:44.336Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.931049215Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"client":"1","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
162962024-04-17T17:20:44.336Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.931082337Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"client":"1","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
162972024-04-17T17:20:44.336Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.931111289Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"client":"2","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
162982024-04-17T17:20:44.336Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.931140011Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"client":"2","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
162992024-04-17T17:20:44.336Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.931183603Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"":"io task","client":"2","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163002024-04-17T17:20:44.336Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.931232636Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"":"io task","client":"1","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163012024-04-17T17:20:44.336Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.931268588Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"":"io task","client":"0","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163022024-04-17T17:20:44.336Z{"msg":"ds_connection connected from Ok(127.0.0.1:49878)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.931312591Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"":"io task","client":"2","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163032024-04-17T17:20:44.336Z{"msg":"ds_connection connected from Ok(127.0.0.1:34514)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.931348813Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"":"io task","client":"0","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163042024-04-17T17:20:44.337Z{"msg":"ds_connection connected from Ok(127.0.0.1:62333)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.931381235Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"":"io task","client":"1","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163052024-04-17T17:20:44.337Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.932002043Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"client":"2","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163062024-04-17T17:20:44.337Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.932050096Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"client":"2","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163072024-04-17T17:20:44.337Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.932082458Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"client":"0","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163082024-04-17T17:20:44.337Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.932109199Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"client":"0","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163092024-04-17T17:20:44.337Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.932138251Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"client":"1","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163102024-04-17T17:20:44.337Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.932162593Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"client":"1","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163112024-04-17T17:20:44.337Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.932630221Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"client":"2","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163122024-04-17T17:20:44.337Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.932680664Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"client":"0","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163132024-04-17T17:20:44.337Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.93277372Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"client":"1","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163142024-04-17T17:20:44.337Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.933182844Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"client":"2","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163152024-04-17T17:20:44.337Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.933233547Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"client":"2","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163162024-04-17T17:20:44.337Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.933260289Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163172024-04-17T17:20:44.338Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.933292761Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"client":"0","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163182024-04-17T17:20:44.338Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.933320853Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"client":"0","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163192024-04-17T17:20:44.338Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.933335894Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163202024-04-17T17:20:44.338Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.933362915Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"client":"1","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163212024-04-17T17:20:44.338Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.933389597Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"client":"1","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163222024-04-17T17:20:44.338Z{"msg":"[0]R flush_numbers[0..12]: [271, 271, 271, 270, 273, 268, 273, 271, 270, 271, 271, 271]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.933408688Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"":"downstairs","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163232024-04-17T17:20:44.338Z{"msg":"[0]R generation[0..12]: [150, 150, 150, 149, 151, 147, 151, 150, 149, 150, 150, 150]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.93343238Z","Wait for a query_work_queue command to finish before sending IO
163242024-04-17T17:20:44.338Zhostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"":"downstairs","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163252024-04-17T17:20:44.338Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.933462992Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"":"downstairs","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163262024-04-17T17:20:44.338Z{"msg":"[1]R flush_numbers[0..12]: [271, 271, 271, 270, 273, 268, 273, 271, 270, 271, 271, 271]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.933488753Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"":"downstairs","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163272024-04-17T17:20:44.338Z{"msg":"[1]R generation[0..12]: [150, 150, 150, 149, 151, 147, 151, 150, 149, 150, 150, 150]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.933506864Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"":"downstairs","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163282024-04-17T17:20:44.338Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.933534986Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"":"downstairs","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163292024-04-17T17:20:44.338Z{"msg":"[2]R flush_numbers[0..12]: [271, 271, 271, 270, 273, 268, 273, 271, 270, 271, 271, 271]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.933568828Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"":"downstairs","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163302024-04-17T17:20:44.338Z{"msg":"[2]R generation[0..12]: [150, 150, 150, 149, 151, 147, 151, 150, 149, 150, 150, 150]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.933586649Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"":"downstairs","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163312024-04-17T17:20:44.338Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.93361275Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"":"downstairs","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163322024-04-17T17:20:44.338Z{"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.933640142Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"":"downstairs","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163332024-04-17T17:20:44.338Z{"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.933663974Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"":"downstairs","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163342024-04-17T17:20:44.338Z{"msg":"Next flush: 274","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.933682005Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"":"downstairs","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163352024-04-17T17:20:44.338Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.933711736Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"":"downstairs","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163362024-04-17T17:20:44.338Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.933726877Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163372024-04-17T17:20:44.339Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.933746779Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"client":"0","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163382024-04-17T17:20:44.339ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
163392024-04-17T17:20:44.339Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.933776531Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"client":"1","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163402024-04-17T17:20:44.339Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.933790531Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"client":"2","":"downstairs","session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163412024-04-17T17:20:44.339Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.933804212Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163422024-04-17T17:20:44.339Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.933816153Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163432024-04-17T17:20:44.339Z{"msg":"ae1e2a3a-f710-4087-b4a3-a5a509fb6055 is now active with session: 208e885e-98af-49fa-8614-604b8f808184","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.933835764Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163442024-04-17T17:20:44.339Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.933847905Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779,"session_id":"208e885e-98af-49fa-8614-604b8f808184"}
163452024-04-17T17:20:44.339Z{"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:42.933866376Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779}
163462024-04-17T17:20:44.339ZLoading write count information from file "/tmp/test_repair_verify.data"
163472024-04-17T17:20:44.339ZRun Repair workload
163482024-04-17T17:20:44.339Z01/30 Write block 205 len 40960 data: 19 18 21 12 13 14 14 14 15 13
163492024-04-17T17:20:44.339Z02/30 Write block 551 len 20480 data: 14 15 12 12 13
163502024-04-17T17:20:44.339Z03/30 Read block 599 len 4096
163512024-04-17T17:20:44.357Z04/30 Write block 111 len 32768 data: 13 12 11 9 10 10 9 6
163522024-04-17T17:20:44.357Z05/30 Write block 324 len 24576 data: 5 7 10 9 9 9
163532024-04-17T17:20:44.357Z06/30 Flush
163542024-04-17T17:20:44.357Z07/30 Write block 494 len 32768 data: 11 12 11 11 13 13 14 15
163552024-04-17T17:20:44.357Z08/30 Write block 314 len 40960 data: 15 14 13 14 16 14 12 11 6 4
163562024-04-17T17:20:44.357Z09/30 Write block 327 len 4096 data: 10
163572024-04-17T17:20:44.357Z10/30 Read block 524 len 40960
163582024-04-17T17:20:44.357Z11/30 Write block 7 len 8192 data: 9 9
163592024-04-17T17:20:44.357Z12/30 Read block 254 len 4096
163602024-04-17T17:20:44.357Z13/30 Write block 364 len 32768 data: 9 9 9 9 13 13 14 15
163612024-04-17T17:20:44.358Z14/30 Read block 263 len 36864
163622024-04-17T17:20:44.358Z15/30 Read block 132 len 24576
163632024-04-17T17:20:44.358Z16/30 Write block 521 len 32768 data: 10 11 11 8 9 12 12 13
163642024-04-17T17:20:44.358Z17/30 Flush
163652024-04-17T17:20:44.358Z18/30 Write block 13 len 36864 data: 12 13 9 8 11 10 12 13 16
163662024-04-17T17:20:44.358Z19/30 Read block 274 len 4096
163672024-04-17T17:20:44.358Z20/30 Read block 372 len 28672
163682024-04-17T17:20:44.358Z21/30 Read block 254 len 12288
163692024-04-17T17:20:44.358Z22/30 Read block 25 len 16384
163702024-04-17T17:20:44.358Z23/30 Write block 109 len 36864 data: 11 12 14 13 12 10 11 11 10
163712024-04-17T17:20:44.358Z24/30 Read block 50 len 40960
163722024-04-17T17:20:44.358Z25/30 Read block 393 len 16384
163732024-04-17T17:20:44.366Z26/30 Write block 287 len 40960 data: 8 9 13 14 13 10 13 11 8 6
163742024-04-17T17:20:44.366Z27/30 Read block 251 len 16384
163752024-04-17T17:20:44.366Z28/30 Write block 153 len 36864 data: 6 9 9 11 12 11 13 13 13
163762024-04-17T17:20:44.366Z29/30 Read block 510 len 8192
163772024-04-17T17:20:44.366Z30/30 Read block 419 len 20480
163782024-04-17T17:20:44.366Z----------------------------------------------------------------
163792024-04-17T17:20:44.367Z Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:30
163802024-04-17T17:20:44.367ZStates: Active Active Active
163812024-04-17T17:20:44.367ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
163822024-04-17T17:20:44.367Z 1 Acked 1000 Write 10 Done Done Done false
163832024-04-17T17:20:44.367Z 2 Acked 1001 Write 5 Done Sent Done false
163842024-04-17T17:20:44.367Z 3 Acked 1002 Read 1 Done Sent Done false
163852024-04-17T17:20:44.367Z 4 Acked 1003 Write 8 Done Sent Done false
163862024-04-17T17:20:44.367Z 5 Acked 1004 Write 6 Done Sent Done false
163872024-04-17T17:20:44.367Z 6 Acked 1005 Flush 0 Done Sent Done false
163882024-04-17T17:20:44.367Z 7 Acked 1006 Write 8 Done Sent Done false
163892024-04-17T17:20:44.367Z 8 Acked 1007 Write 10 Done Sent Done false
163902024-04-17T17:20:44.367Z 9 Acked 1008 Write 1 Done Sent Done false
163912024-04-17T17:20:44.367Z 10 Acked 1009 Read 10 Done Sent Done false
163922024-04-17T17:20:44.367Z 11 Acked 1010 Write 2 Done Sent Done false
163932024-04-17T17:20:44.367Z 12 Acked 1011 Read 1 Done Sent Done false
163942024-04-17T17:20:44.367Z 13 Acked 1012 Write 8 Done Sent Done false
163952024-04-17T17:20:44.367Z 14 Acked 1013 Read 9 Done Sent Done false
163962024-04-17T17:20:44.367Z 15 Acked 1014 Read 6 Done Sent Done false
163972024-04-17T17:20:44.367Z 16 Acked 1015 Write 8 Done Sent Done false
163982024-04-17T17:20:44.367Z 17 Acked 1016 Flush 0 Done Sent Done false
163992024-04-17T17:20:44.367Z 18 Acked 1017 Write 9 Done Sent Done false
164002024-04-17T17:20:44.367Z 19 Acked 1018 Read 1 Done Sent Done false
164012024-04-17T17:20:44.367Z 20 Acked 1019 Read 7 Done Sent Done false
164022024-04-17T17:20:44.367Z 21 Acked 1020 Read 3 Done Sent Done false
164032024-04-17T17:20:44.368Z 22 Acked 1021 Read 4 Done Sent Done false
164042024-04-17T17:20:44.368Z 23 Acked 1022 Write 9 Done Sent Done false
164052024-04-17T17:20:44.368Z 24 Acked 1023 Read 10 Done Sent Done false
164062024-04-17T17:20:44.368Z 25 Acked 1024 Read 4 Done Sent Done false
164072024-04-17T17:20:44.368Z 26 Acked 1025 Write 10 Done Sent Done false
164082024-04-17T17:20:44.368Z 27 Acked 1026 Read 4 Done Sent Done false
164092024-04-17T17:20:44.368Z 28 Acked 1027 Write 9 Done Sent Done false
164102024-04-17T17:20:44.368Z 29 Acked 1028 Read 2 Done Sent Done false
164112024-04-17T17:20:44.368Z 30 Acked 1029 Read 5 Done Sent Done false
164122024-04-17T17:20:44.368Z STATES DS:0 DS:1 DS:2 TOTAL
164132024-04-17T17:20:44.368Z New 0 0 0 0
164142024-04-17T17:20:44.368Z Sent 0 29 0 29
164152024-04-17T17:20:44.368Z Done 30 1 30 61
164162024-04-17T17:20:44.368Z Skipped 0 0 0 0
164172024-04-17T17:20:44.368Z Error 0 0 0 0
164182024-04-17T17:20:44.368ZLast Flush: 1016 0 1016
164192024-04-17T17:20:44.368ZDownstairs last five completed:
164202024-04-17T17:20:44.368ZUpstairs last five completed: 30 29 28 27 26
164212024-04-17T17:20:44.369Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:20:42.968136433Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1779}
164222024-04-17T17:20:44.369ZWrote out file "/tmp/test_repair_verify.data"
164232024-04-17T17:20:44.369Z
164242024-04-17T17:20:44.376ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
164252024-04-17T17:20:44.383ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
164262024-04-17T17:20:44.383Z 0 000-019 152 150 152 275 271 275 T F T <---
164272024-04-17T17:20:44.388Z 1 020-039 150 150 150 271 271 271 T F T <---
164282024-04-17T17:20:44.388Z 2 040-059 150 150 150 271 271 271 F F F
164292024-04-17T17:20:44.388Z 3 060-079 149 149 149 270 270 270 F F F
164302024-04-17T17:20:44.389Z 4 080-099 151 151 151 273 273 273 F F F
164312024-04-17T17:20:44.389Z 5 100-119 152 147 152 274 268 274 T F T <---
164322024-04-17T17:20:44.389Z 6 120-139 151 151 151 273 273 273 F F F
164332024-04-17T17:20:44.389Z 7 140-159 150 150 150 271 271 271 T F T <---
164342024-04-17T17:20:44.389Z 8 160-179 149 149 149 270 270 270 T F T <---
164352024-04-17T17:20:44.389Z 9 180-199 150 150 150 271 271 271 F F F
164362024-04-17T17:20:44.389Z 10 200-219 152 150 152 274 271 274 F T F <---
164372024-04-17T17:20:44.389Z 11 220-239 150 150 150 271 271 271 F F F
164382024-04-17T17:20:44.389Z 12 240-259 150 150 150 271 271 271 F F F
164392024-04-17T17:20:44.389Z 13 260-279 150 150 150 271 271 271 F F F
164402024-04-17T17:20:44.389Z 14 280-299 147 147 147 268 268 268 T F T <---
164412024-04-17T17:20:44.389Z 15 300-319 152 150 152 275 271 275 F F F <---
164422024-04-17T17:20:44.389Z 16 320-339 152 147 152 275 268 275 F F F <---
164432024-04-17T17:20:44.389Z 17 340-359 150 150 150 271 271 271 F F F
164442024-04-17T17:20:44.390Z 18 360-379 152 151 152 275 273 275 F F F <---
164452024-04-17T17:20:44.390Z 19 380-399 144 144 144 263 263 263 F F F
164462024-04-17T17:20:44.390Z 20 400-419 149 149 149 270 270 270 F F F
164472024-04-17T17:20:44.390Z 21 420-439 146 146 146 267 267 267 F F F
164482024-04-17T17:20:44.390Z 22 440-459 146 146 146 267 267 267 F F F
164492024-04-17T17:20:44.390Z 23 460-479 143 143 143 261 261 261 F F F
164502024-04-17T17:20:44.390Z 24 480-499 152 146 152 275 267 275 F F F <---
164512024-04-17T17:20:44.390Z 25 500-519 152 146 152 275 267 275 F F F <---
164522024-04-17T17:20:44.390Z 26 520-539 152 149 152 275 270 275 F F F <---
164532024-04-17T17:20:44.390Z 27 540-559 152 148 152 274 269 274 F F F <---
164542024-04-17T17:20:44.390Z 28 560-579 146 146 146 267 267 267 F F F
164552024-04-17T17:20:44.390Z 29 580-599 150 150 150 272 272 272 F F F
164562024-04-17T17:20:44.390ZMax gen: 152, Max flush: 275
164572024-04-17T17:20:44.390ZError: Difference in extent metadata found!
164582024-04-17T17:20:44.390ZA Difference in extent metadata is expected here
164592024-04-17T17:20:44.390ZOn loop 76
164602024-04-17T17:20:44.391Z
164612024-04-17T17:20:44.396ZVerifying data now
164622024-04-17T17:20:49.044ZNo change after verify
164632024-04-17T17:20:49.045ZLoop: 76 Downstairs dump after verify (and repair):
164642024-04-17T17:20:49.059ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
164652024-04-17T17:20:49.059Z 0 000-019 153 153 153 276 276 276 F F F
164662024-04-17T17:20:49.059Z 1 020-039 153 153 153 276 276 276 F F F
164672024-04-17T17:20:49.059Z 2 040-059 150 150 150 271 271 271 F F F
164682024-04-17T17:20:49.063Z 3 060-079 149 149 149 270 270 270 F F F
164692024-04-17T17:20:49.063Z 4 080-099 151 151 151 273 273 273 F F F
164702024-04-17T17:20:49.063Z 5 100-119 153 153 153 276 276 276 F F F
164712024-04-17T17:20:49.063Z 6 120-139 151 151 151 273 273 273 F F F
164722024-04-17T17:20:49.063Z 7 140-159 153 153 153 276 276 276 F F F
164732024-04-17T17:20:49.063Z 8 160-179 153 153 153 276 276 276 F F F
164742024-04-17T17:20:49.063Z 9 180-199 150 150 150 271 271 271 F F F
164752024-04-17T17:20:49.063Z 10 200-219 152 152 152 274 274 274 F F F
164762024-04-17T17:20:49.063Z 11 220-239 150 150 150 271 271 271 F F F
164772024-04-17T17:20:49.063Z 12 240-259 150 150 150 271 271 271 F F F
164782024-04-17T17:20:49.063Z 13 260-279 150 150 150 271 271 271 F F F
164792024-04-17T17:20:49.063Z 14 280-299 153 153 153 276 276 276 F F F
164802024-04-17T17:20:49.063Z 15 300-319 152 152 152 275 275 275 F F F
164812024-04-17T17:20:49.063Z 16 320-339 152 152 152 275 275 275 F F F
164822024-04-17T17:20:49.063Z 17 340-359 150 150 150 271 271 271 F F F
164832024-04-17T17:20:49.063Z 18 360-379 152 152 152 275 275 275 F F F
164842024-04-17T17:20:49.063Z 19 380-399 144 144 144 263 263 263 F F F
164852024-04-17T17:20:49.063Z 20 400-419 149 149 149 270 270 270 F F F
164862024-04-17T17:20:49.064Z 21 420-439 146 146 146 267 267 267 F F F
164872024-04-17T17:20:49.064Z 22 440-459 146 146 146 267 267 267 F F F
164882024-04-17T17:20:49.064Z 23 460-479 143 143 143 261 261 261 F F F
164892024-04-17T17:20:49.064Z 24 480-499 152 152 152 275 275 275 F F F
164902024-04-17T17:20:49.064Z 25 500-519 152 152 152 275 275 275 F F F
164912024-04-17T17:20:49.064Z 26 520-539 152 152 152 275 275 275 F F F
164922024-04-17T17:20:49.064Z 27 540-559 152 152 152 274 274 274 F F F
164932024-04-17T17:20:49.064Z 28 560-579 146 146 146 267 267 267 F F F
164942024-04-17T17:20:49.064Z 29 580-599 150 150 150 272 272 272 F F F
164952024-04-17T17:20:49.064ZMax gen: 153, Max flush: 276
164962024-04-17T17:20:49.064Z
164972024-04-17T17:20:49.064ZBegin loop 77
164982024-04-17T17:20:49.064ZDownstairs to restart: 0
164992024-04-17T17:20:49.082Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.691483601Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787}
165002024-04-17T17:20:49.082Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.691738167Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787}
165012024-04-17T17:20:49.088Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":Crucible runtime is spawned
165022024-04-17T17:20:49.088Z"2024-04-17T17:20:47.691778139Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787}
165032024-04-17T17:20:49.088Z{"msg":"Crucible e9b8b8b1-1bf9-412b-81aa-666bed011c44 has session id: 1e157051-dc46-47c3-ae81-a28a1e2cdbaf","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.691828622Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165042024-04-17T17:20:49.088Z{"msg":"Upstairs opts: Upstairs UUID: e9b8b8b1-1bf9-412b-81aa-666bed011c44, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.691886976Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165052024-04-17T17:20:49.088Z{"msg":"Crucible stats registered with UUID: e9b8b8b1-1bf9-412b-81aa-666bed011c44","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.691908897Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165062024-04-17T17:20:49.088Z{"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.691934769Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787}
165072024-04-17T17:20:49.088Z{"msg":"e9b8b8b1-1bf9-412b-81aa-666bed011c44 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.692005823Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165082024-04-17T17:20:49.088Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.692043325Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"client":"0","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165092024-04-17T17:20:49.088Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.692080577Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"client":"0","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165102024-04-17T17:20:49.088Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.692153942Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"client":"1","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165112024-04-17T17:20:49.088Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.692184874Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"client":"1","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165122024-04-17T17:20:49.088Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.692206495Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"client":"2","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165132024-04-17T17:20:49.088Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.692275529Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"client":"2","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165142024-04-17T17:20:49.088Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.692327392Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"":"io task","client":"2","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165152024-04-17T17:20:49.088Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.692361834Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"":"io task","client":"0","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165162024-04-17T17:20:49.088Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.692388876Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165172024-04-17T17:20:49.088Z{"msg":"ds_connection connected from Ok(127.0.0.1:38108)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.692414768Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"":"io task","client":"2","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165182024-04-17T17:20:49.088Z{"msg":"ds_connection connected from Ok(127.0.0.1:33950)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.69245698Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"":"io task","client":"0","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165192024-04-17T17:20:49.088Z{"msg":"ds_connection connected from Ok(127.0.0.1:60138)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.692474131Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165202024-04-17T17:20:49.088Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.693043546Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"client":"1","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165212024-04-17T17:20:49.089Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.693091609Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"client":"1","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165222024-04-17T17:20:49.089Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.693122301Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"client":"2","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165232024-04-17T17:20:49.089Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.693151142Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"client":"2","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165242024-04-17T17:20:49.089Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.693382976Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"client":"0","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165252024-04-17T17:20:49.089Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.693420179Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"client":"0","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165262024-04-17T17:20:49.089Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.693665144Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"client":"2","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165272024-04-17T17:20:49.089Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.693778611Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"client":"1","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165282024-04-17T17:20:49.089Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.694029436Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"client":"0","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165292024-04-17T17:20:49.089Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.694224467Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"client":"2","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165302024-04-17T17:20:49.089Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.69427071Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"client":"2","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165312024-04-17T17:20:49.089Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.694333004Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165322024-04-17T17:20:49.089Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.694365046Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"client":"1","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165332024-04-17T17:20:49.089Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.694381227Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"client":"1","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165342024-04-17T17:20:49.089Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.694395138Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165352024-04-17T17:20:49.089Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.694531566Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"client":"0","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165362024-04-17T17:20:49.089Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.694567558Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"client":"0","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165372024-04-17T17:20:49.089Z{"msg":"[0]R flush_numbers[0..12]: [276, 276, 271, 270, 273, 276, 273, 276, 276, 271, 274, 271]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.694584669Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"":"downstairs","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165382024-04-17T17:20:49.089Z{"msg":"[0]R generation[0..12]: [153, 153, 150, 149, 151, 153, 151, 153, 153, 150, 152, 150]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.69459948Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"":"downstairs","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165392024-04-17T17:20:49.089Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.694621911Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"":"downstairs","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165402024-04-17T17:20:49.089Z{"msg":"[1]R flush_numbers[0..12]: [276, 276, 271, 270, 273, 276, 273, 276, 276, 271, 274, 271]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.694636282Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"":"downstairs","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165412024-04-17T17:20:49.089Z{"msg":"[1]R generation[0..12]: [153, 153, 150, 149, 151, 153, 151, 153, 153, 150, 152, 150]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.694650183Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"":"downstairs","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165422024-04-17T17:20:49.089Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
165432024-04-17T17:20:49.090Z,"time":"2024-04-17T17:20:47.694671454Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"":"downstairs","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165442024-04-17T17:20:49.090Z{"msg":"[2]R flush_numbers[0..12]: [276, 276, 271, 270, 273, 276, 273, 276, 276, 271, 274, 271]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.694689036Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"":"downstairs","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165452024-04-17T17:20:49.090Z{"msg":"[2]R generation[0..12]: [153, 153, 150, 149, 151, 153, 151, 153, 153, 150, 152, 150]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.694713697Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"":"downstairs","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165462024-04-17T17:20:49.090Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.694730898Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"":"downstairs","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165472024-04-17T17:20:49.090Z{"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.694751359Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"":"downstairs","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165482024-04-17T17:20:49.090Z{"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.69476478Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"":"downstairs","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165492024-04-17T17:20:49.090Z{"msg":"Next flush: 277","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.694777671Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"":"downstairs","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165502024-04-17T17:20:49.090Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.694797622Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"":"downstairs","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165512024-04-17T17:20:49.090Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.694810763Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165522024-04-17T17:20:49.090Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.694826724Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"client":"0","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165532024-04-17T17:20:49.090Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.694859676Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"client":"1","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165542024-04-17T17:20:49.090Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.694881207Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"client":"2","":"downstairs","session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165552024-04-17T17:20:49.090Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.694915569Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165562024-04-17T17:20:49.090Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.69493122Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165572024-04-17T17:20:49.090Z{"msg":"e9b8b8b1-1bf9-412b-81aa-666bed011c44 is now active with session: 1e157051-dc46-47c3-ae81-a28a1e2cdbaf","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.694963182Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165582024-04-17T17:20:49.090Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
165592024-04-17T17:20:49.090ZSet Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.694982273Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787,"session_id":"1e157051-dc46-47c3-ae81-a28a1e2cdbaf"}
165602024-04-17T17:20:49.090Z{"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:47.695011805Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787}
165612024-04-17T17:20:49.094ZLoading write count information from file "/tmp/test_repair_verify.data"
165622024-04-17T17:20:49.094ZRun Repair workload
165632024-04-17T17:20:49.094Z01/30 Write block 178 len 12288 data: 15 12 9
165642024-04-17T17:20:49.094Z02/30 Write block 437 len 20480 data: 13 12 15 11 11
165652024-04-17T17:20:49.094Z03/30 Flush
165662024-04-17T17:20:49.094Z04/30 Flush
165672024-04-17T17:20:49.094Z05/30 Flush
165682024-04-17T17:20:49.094Z06/30 Write block 0 len 12288 data: 5 6 5
165692024-04-17T17:20:49.094Z07/30 Write block 542 len 12288 data: 16 15 15
165702024-04-17T17:20:49.094Z08/30 Read block 328 len 16384
165712024-04-17T17:20:49.098Z09/30 Write block 260 len 28672 data: 10 12 12 11 11 10 10
165722024-04-17T17:20:49.098Z10/30 Read block 99 len 24576
165732024-04-17T17:20:49.098Z11/30 Write block 437 len 20480 data: 14 13 16 12 12
165742024-04-17T17:20:49.098Z12/30 Read block 142 len 28672
165752024-04-17T17:20:49.101Z13/30 Write block 249 len 32768 data: 12 11 11 13 11 10 8 12
165762024-04-17T17:20:49.101Z14/30 Read block 363 len 16384
165772024-04-17T17:20:49.101Z15/30 Read block 124 len 36864
165782024-04-17T17:20:49.101Z16/30 Read block 130 len 24576
165792024-04-17T17:20:49.105Z17/30 Read block 569 len 8192
165802024-04-17T17:20:49.105Z18/30 Read block 203 len 16384
165812024-04-17T17:20:49.105Z19/30 Write block 398 len 16384 data: 10 10 9 10
165822024-04-17T17:20:49.105Z20/30 Write block 19 len 32768 data: 13 14 17 16 14 13 12 11
165832024-04-17T17:20:49.105Z21/30 Read block 212 len 24576
165842024-04-17T17:20:49.109Z22/30 Write block 519 len 24576 data: 7 9 11 12 12 9
165852024-04-17T17:20:49.109Z23/30 Read block 369 len 24576
165862024-04-17T17:20:49.109Z24/30 Write block 5 len 32768 data: 12 10 10 10 10 10 9 11
165872024-04-17T17:20:49.109Z25/30 Write block 552 len 4096 data: 16
165882024-04-17T17:20:49.113Z26/30 Read block 48 len 40960
165892024-04-17T17:20:49.113Z27/30 Write block 30 len 20480 data: 11 14 19 19 23
165902024-04-17T17:20:49.113Z28/30 Write block 2 len 28672 data: 6 10 12 13 11 11 11
165912024-04-17T17:20:49.113Z29/30 Read block 404 len 8192
165922024-04-17T17:20:49.117Z30/30 Read block 481 len 28672
165932024-04-17T17:20:49.117Z----------------------------------------------------------------
165942024-04-17T17:20:49.117Z Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30
165952024-04-17T17:20:49.117ZStates: Active Active Active
165962024-04-17T17:20:49.117ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
165972024-04-17T17:20:49.117Z 1 Acked 1000 Write 3 Sent Done Done false
165982024-04-17T17:20:49.117Z 2 Acked 1001 Write 5 Sent Done Done false
165992024-04-17T17:20:49.117Z 3 Acked 1002 Flush 0 Sent Done Done false
166002024-04-17T17:20:49.117Z 4 Acked 1003 Flush 0 Sent Done Done false
166012024-04-17T17:20:49.117Z 5 Acked 1004 Flush 0 Sent Done Done false
166022024-04-17T17:20:49.117Z 6 Acked 1005 Write 3 Sent Done Done false
166032024-04-17T17:20:49.118Z 7 Acked 1006 Write 3 Sent Done Done false
166042024-04-17T17:20:49.118Z 8 Acked 1007 Read 4 Sent Done Done false
166052024-04-17T17:20:49.118Z 9 Acked 1008 Write 7 Sent Done Done false
166062024-04-17T17:20:49.118Z 10 Acked 1009 Read 6 Sent Done Done false
166072024-04-17T17:20:49.118Z 11 Acked 1010 Write 5 Sent Done Done false
166082024-04-17T17:20:49.118Z 12 Acked 1011 Read 7 Sent Done Done false
166092024-04-17T17:20:49.118Z 13 Acked 1012 Write 8 Sent Done Done false
166102024-04-17T17:20:49.118Z 14 Acked 1013 Read 4 Sent Done Done false
166112024-04-17T17:20:49.118Z 15 Acked 1014 Read 9 Sent Done Done false
166122024-04-17T17:20:49.118Z 16 Acked 1015 Read 6 Sent Done Done false
166132024-04-17T17:20:49.118Z 17 Acked 1016 Read 2 Sent Done Done false
166142024-04-17T17:20:49.118Z 18 Acked 1017 Read 4 Sent Done Done false
166152024-04-17T17:20:49.118Z 19 Acked 1018 Write 4 Sent Done Done false
166162024-04-17T17:20:49.118Z 20 Acked 1019 Write 8 Sent Done Done false
166172024-04-17T17:20:49.118Z 21 Acked 1020 Read 6 Sent Done Done false
166182024-04-17T17:20:49.118Z 22 Acked 1021 Write 6 Sent Done Done false
166192024-04-17T17:20:49.118Z 23 Acked 1022 Read 6 Sent Done Done false
166202024-04-17T17:20:49.118Z 24 Acked 1023 Write 8 Sent Done Done false
166212024-04-17T17:20:49.118Z 25 Acked 1024 Write 1 Sent Done Done false
166222024-04-17T17:20:49.118Z 26 Acked 1025 Read 10 Sent Done Done false
166232024-04-17T17:20:49.118Z 27 Acked 1026 Write 5 Sent Done Done false
166242024-04-17T17:20:49.118Z 28 Acked 1027 Write 7 Sent Done Done false
166252024-04-17T17:20:49.118Z 29 Acked 1028 Read 2 Sent Done Done false
166262024-04-17T17:20:49.119Z 30 Acked 1029 Read 7 Sent Done Done false
166272024-04-17T17:20:49.119Z STATES DS:0 DS:1 DS:2 TOTAL
166282024-04-17T17:20:49.119Z New 0 0 0 0
166292024-04-17T17:20:49.119Z Sent 30 0 0 30
166302024-04-17T17:20:49.119Z Done 0 30 30 60
166312024-04-17T17:20:49.119Z Skipped 0 0 0 0
166322024-04-17T17:20:49.119Z Error 0 0 0 0
166332024-04-17T17:20:49.119ZLast Flush: 0 1004 1004
166342024-04-17T17:20:49.119ZDownstairs last five completed:
166352024-04-17T17:20:49.119ZUpstairs last five completed: 30 29 28 27 26
166362024-04-17T17:20:49.119Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:20:47.72412557Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1787}
166372024-04-17T17:20:49.119ZWrote out file "/tmp/test_repair_verify.data"
166382024-04-17T17:20:49.123Z
166392024-04-17T17:20:49.123ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
166402024-04-17T17:20:49.143ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
166412024-04-17T17:20:49.143Z 0 000-019 153 153 153 276 276 276 F T T <---
166422024-04-17T17:20:49.143Z 1 020-039 153 153 153 276 276 276 F T T <---
166432024-04-17T17:20:49.143Z 2 040-059 150 150 150 271 271 271 F F F
166442024-04-17T17:20:49.143Z 3 060-079 149 149 149 270 270 270 F F F
166452024-04-17T17:20:49.143Z 4 080-099 151 151 151 273 273 273 F F F
166462024-04-17T17:20:49.147Z 5 100-119 153 153 153 276 276 276 F F F
166472024-04-17T17:20:49.147Z 6 120-139 151 151 151 273 273 273 F F F
166482024-04-17T17:20:49.147Z 7 140-159 153 153 153 276 276 276 F F F
166492024-04-17T17:20:49.147Z 8 160-179 153 154 154 276 277 277 F F F <---
166502024-04-17T17:20:49.147Z 9 180-199 150 154 154 271 277 277 F F F <---
166512024-04-17T17:20:49.147Z 10 200-219 152 152 152 274 274 274 F F F
166522024-04-17T17:20:49.147Z 11 220-239 150 150 150 271 271 271 F F F
166532024-04-17T17:20:49.147Z 12 240-259 150 150 150 271 271 271 F T T <---
166542024-04-17T17:20:49.147Z 13 260-279 150 150 150 271 271 271 F T T <---
166552024-04-17T17:20:49.147Z 14 280-299 153 153 153 276 276 276 F F F
166562024-04-17T17:20:49.147Z 15 300-319 152 152 152 275 275 275 F F F
166572024-04-17T17:20:49.147Z 16 320-339 152 152 152 275 275 275 F F F
166582024-04-17T17:20:49.147Z 17 340-359 150 150 150 271 271 271 F F F
166592024-04-17T17:20:49.147Z 18 360-379 152 152 152 275 275 275 F F F
166602024-04-17T17:20:49.147Z 19 380-399 144 144 144 263 263 263 F T T <---
166612024-04-17T17:20:49.147Z 20 400-419 149 149 149 270 270 270 F T T <---
166622024-04-17T17:20:49.147Z 21 420-439 146 154 154 267 277 277 F T T <---
166632024-04-17T17:20:49.147Z 22 440-459 146 154 154 267 277 277 F T T <---
166642024-04-17T17:20:49.148Z 23 460-479 143 143 143 261 261 261 F F F
166652024-04-17T17:20:49.148Z 24 480-499 152 152 152 275 275 275 F F F
166662024-04-17T17:20:49.148Z 25 500-519 152 152 152 275 275 275 F T T <---
166672024-04-17T17:20:49.148Z 26 520-539 152 152 152 275 275 275 F T T <---
166682024-04-17T17:20:49.148Z 27 540-559 152 152 152 274 274 274 F T T <---
166692024-04-17T17:20:49.148Z 28 560-579 146 146 146 267 267 267 F F F
166702024-04-17T17:20:49.148Z 29 580-599 150 150 150 272 272 272 F F F
166712024-04-17T17:20:49.148ZMax gen: 154, Max flush: 277
166722024-04-17T17:20:49.148ZError: Difference in extent metadata found!
166732024-04-17T17:20:49.148ZA Difference in extent metadata is expected here
166742024-04-17T17:20:49.148ZOn loop 77
166752024-04-17T17:20:49.148Z
166762024-04-17T17:20:49.148ZVerifying data now
166772024-04-17T17:20:53.767ZNo change after verify
166782024-04-17T17:20:53.767ZLoop: 77 Downstairs dump after verify (and repair):
166792024-04-17T17:20:53.780ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
166802024-04-17T17:20:53.781Z 0 000-019 155 155 155 278 278 278 F F F
166812024-04-17T17:20:53.785Z 1 020-039 155 155 155 278 278 278 F F F
166822024-04-17T17:20:53.785Z 2 040-059 150 150 150 271 271 271 F F F
166832024-04-17T17:20:53.785Z 3 060-079 149 149 149 270 270 270 F F F
166842024-04-17T17:20:53.785Z 4 080-099 151 151 151 273 273 273 F F F
166852024-04-17T17:20:53.785Z 5 100-119 153 153 153 276 276 276 F F F
166862024-04-17T17:20:53.785Z 6 120-139 151 151 151 273 273 273 F F F
166872024-04-17T17:20:53.785Z 7 140-159 153 153 153 276 276 276 F F F
166882024-04-17T17:20:53.785Z 8 160-179 154 154 154 277 277 277 F F F
166892024-04-17T17:20:53.785Z 9 180-199 154 154 154 277 277 277 F F F
166902024-04-17T17:20:53.785Z 10 200-219 152 152 152 274 274 274 F F F
166912024-04-17T17:20:53.785Z 11 220-239 150 150 150 271 271 271 F F F
166922024-04-17T17:20:53.785Z 12 240-259 155 155 155 278 278 278 F F F
166932024-04-17T17:20:53.785Z 13 260-279 155 155 155 278 278 278 F F F
166942024-04-17T17:20:53.785Z 14 280-299 153 153 153 276 276 276 F F F
166952024-04-17T17:20:53.785Z 15 300-319 152 152 152 275 275 275 F F F
166962024-04-17T17:20:53.785Z 16 320-339 152 152 152 275 275 275 F F F
166972024-04-17T17:20:53.785Z 17 340-359 150 150 150 271 271 271 F F F
166982024-04-17T17:20:53.785Z 18 360-379 152 152 152 275 275 275 F F F
166992024-04-17T17:20:53.785Z 19 380-399 155 155 155 278 278 278 F F F
167002024-04-17T17:20:53.786Z 20 400-419 155 155 155 278 278 278 F F F
167012024-04-17T17:20:53.786Z 21 420-439 155 155 155 278 278 278 F F F
167022024-04-17T17:20:53.786Z 22 440-459 155 155 155 278 278 278 F F F
167032024-04-17T17:20:53.786Z 23 460-479 143 143 143 261 261 261 F F F
167042024-04-17T17:20:53.786Z 24 480-499 152 152 152 275 275 275 F F F
167052024-04-17T17:20:53.786Z 25 500-519 155 155 155 278 278 278 F F F
167062024-04-17T17:20:53.786Z 26 520-539 155 155 155 278 278 278 F F F
167072024-04-17T17:20:53.786Z 27 540-559 155 155 155 278 278 278 F F F
167082024-04-17T17:20:53.786Z 28 560-579 146 146 146 267 267 267 F F F
167092024-04-17T17:20:53.786Z 29 580-599 150 150 150 272 272 272 F F F
167102024-04-17T17:20:53.786ZMax gen: 155, Max flush: 278
167112024-04-17T17:20:53.786Z
167122024-04-17T17:20:53.786ZBegin loop 78
167132024-04-17T17:20:53.786ZDownstairs to restart: 1
167142024-04-17T17:20:53.803Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.412611717Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795}
167152024-04-17T17:20:53.810Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.412924996Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795}
167162024-04-17T17:20:53.810ZCrucible runtime is spawned
167172024-04-17T17:20:53.810Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.412980949Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795}
167182024-04-17T17:20:53.810Z{"msg":"Crucible 67d981f6-b8c6-4f3a-89df-ca5a2b637f36 has session id: 2460cc76-1d6e-499b-89fa-9f2883b305c5","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.413043303Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167192024-04-17T17:20:53.810Z{"msg":"Upstairs opts: Upstairs UUID: 67d981f6-b8c6-4f3a-89df-ca5a2b637f36, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.413104667Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167202024-04-17T17:20:53.810Z{"msg":"Crucible stats registered with UUID: 67d981f6-b8c6-4f3a-89df-ca5a2b637f36","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.413148919Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167212024-04-17T17:20:53.810Z{"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.413175101Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795}
167222024-04-17T17:20:53.810Z{"msg":"67d981f6-b8c6-4f3a-89df-ca5a2b637f36 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.413205833Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167232024-04-17T17:20:53.810Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.413219374Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"client":"0","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167242024-04-17T17:20:53.810Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.413240425Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"client":"0","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167252024-04-17T17:20:53.810Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.413267956Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"client":"1","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167262024-04-17T17:20:53.810Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.413287907Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"client":"1","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167272024-04-17T17:20:53.810Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.413309079Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"client":"2","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167282024-04-17T17:20:53.810Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.413406315Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"client":"2","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167292024-04-17T17:20:53.810Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.413450848Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"":"io task","client":"0","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167302024-04-17T17:20:53.810Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.413480459Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"":"io task","client":"1","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167312024-04-17T17:20:53.810Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.413523172Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"":"io task","client":"2","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167322024-04-17T17:20:53.810Z{"msg":"ds_connection connected from Ok(127.0.0.1:56292)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.413560484Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"":"io task","client":"2","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167332024-04-17T17:20:53.810Z{"msg":"ds_connection connected from Ok(127.0.0.1:37722)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.413603827Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"":"io task","client":"1","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167342024-04-17T17:20:53.810Z{"msg":"ds_connection connected from Ok(127.0.0.1:47745)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.413631908Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"":"io task","client":"0","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167352024-04-17T17:20:53.811Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.414384414Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"client":"0","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167362024-04-17T17:20:53.811Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.414421766Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"client":"0","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167372024-04-17T17:20:53.811Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.414438607Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"client":"2","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167382024-04-17T17:20:53.811Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.414471869Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"client":"2","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167392024-04-17T17:20:53.811Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.414556184Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"client":"1","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167402024-04-17T17:20:53.811Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.414596547Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"client":"1","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167412024-04-17T17:20:53.811Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.415025273Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"client":"1","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167422024-04-17T17:20:53.811Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.415076106Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"client":"0","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167432024-04-17T17:20:53.811Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.415121609Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"client":"2","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167442024-04-17T17:20:53.811Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.41564431Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"client":"0","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167452024-04-17T17:20:53.811Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.415688353Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"client":"0","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167462024-04-17T17:20:53.811Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.415711594Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167472024-04-17T17:20:53.811Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.415732806Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"client":"1","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167482024-04-17T17:20:53.811Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.415747317Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"client":"1","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167492024-04-17T17:20:53.811Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.415760577Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167502024-04-17T17:20:53.811Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.415779689Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"client":"2","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167512024-04-17T17:20:53.811Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.41579307Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"client":"2","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167522024-04-17T17:20:53.811Z{"msg":"[0]R flush_numbers[0..12]: [278, 278, 271, 270, 273, 276, 273, 276, 277, 277, 274, 271]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.41580661Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"":"downstairs","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167532024-04-17T17:20:53.811Z{"msg":"[0]R generation[0..12]: [155, 155, 150, 149, 151, 153, 151, 153, 154, 154, 152, 150]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.415827832Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"":"downstairs","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167542024-04-17T17:20:53.811Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.415842412Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"":"downstairs","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167552024-04-17T17:20:53.811Z{"msg":"[1]R flush_numbers[0..12]: [278, 278, 271, 270, 273, 276, 273, 276, 277, 277, 274, 271]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.415856813Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"":"downstairs","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167562024-04-17T17:20:53.811Z{"msg":"[1]R generation[0..12]: [155, 155, 150, 149, 151, 153, 151, 153, 154, 154, 152, 150]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.415870494Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"":"downstairs","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167572024-04-17T17:20:53.811Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.415891265Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"":"downstairs","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167582024-04-17T17:20:53.812Z{"msg":"[2]R flush_numbers[0..12]: [278, 278, 271, 270, 273, 276, 273, 276, 277, 277, 274, 271]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.415904966Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"":"downstairs","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167592024-04-17T17:20:53.812Z{"msg":"[2]R generation[0..12]: [155, 155, 150, 149, 151, 153, 151, 153, 154, 154, 152, 150]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.415918677Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"":"downstairs","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167602024-04-17T17:20:53.812Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.415939398Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"":"downstairs","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167612024-04-17T17:20:53.812Z{"msgWait for a query_work_queue command to finish before sending IO
167622024-04-17T17:20:53.812Z":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.415955319Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"":"downstairs","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167632024-04-17T17:20:53.812Z{"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.41597521Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"":"downstairs","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167642024-04-17T17:20:53.812Z{"msg":"Next flush: 279","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.415988141Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"":"downstairs","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167652024-04-17T17:20:53.812Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.416035714Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"":"downstairs","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167662024-04-17T17:20:53.812Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.416048865Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167672024-04-17T17:20:53.812Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.416060466Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"client":"0","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167682024-04-17T17:20:53.812Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.416073396Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"client":"1","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167692024-04-17T17:20:53.812Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.416093698Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"client":"2","":"downstairs","session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167702024-04-17T17:20:53.812Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.416107178Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167712024-04-17T17:20:53.812Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.416125679Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167722024-04-17T17:20:53.812Z{"msg":"67d981f6-b8c6-4f3a-89df-ca5a2b637f36 is now active with session: 2460cc76-1d6e-499b-89fa-9f2883b305c5","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.41613845Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167732024-04-17T17:20:53.812Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.416150551Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795,"session_id":"2460cc76-1d6e-499b-89fa-9f2883b305c5"}
167742024-04-17T17:20:53.812Z{"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:52.416163702Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795}
167752024-04-17T17:20:53.812ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
167762024-04-17T17:20:53.812ZLoading write count information from file "/tmp/test_repair_verify.data"
167772024-04-17T17:20:53.812ZRun Repair workload
167782024-04-17T17:20:53.816Z01/30 Write block 446 len 12288 data: 9 7 8
167792024-04-17T17:20:53.816Z02/30 Read block 72 len 12288
167802024-04-17T17:20:53.816Z03/30 Write block 310 len 24576 data: 13 10 12 13 16 15
167812024-04-17T17:20:53.816Z04/30 Read block 254 len 36864
167822024-04-17T17:20:53.816Z05/30 Write block 201 len 36864 data: 14 17 18 20 20 19 22 13 14
167832024-04-17T17:20:53.816Z06/30 Flush
167842024-04-17T17:20:53.816Z07/30 Flush
167852024-04-17T17:20:53.819Z08/30 Write block 290 len 16384 data: 15 14 11 14
167862024-04-17T17:20:53.820Z09/30 Read block 569 len 36864
167872024-04-17T17:20:53.820Z10/30 Read block 454 len 32768
167882024-04-17T17:20:53.820Z11/30 Read block 98 len 8192
167892024-04-17T17:20:53.820Z12/30 Read block 597 len 12288
167902024-04-17T17:20:53.823Z13/30 Write block 87 len 16384 data: 16 14 15 16
167912024-04-17T17:20:53.823Z14/30 Write block 66 len 36864 data: 14 14 13 11 11 11 11 8 11
167922024-04-17T17:20:53.823Z15/30 Write block 269 len 28672 data: 17 14 16 15 18 15 13
167932024-04-17T17:20:53.827Z16/30 Write block 231 len 32768 data: 11 11 11 12 16 14 11 9
167942024-04-17T17:20:53.827Z17/30 Write block 37 len 20480 data: 14 12 12 13 14
167952024-04-17T17:20:53.827Z18/30 Read block 583 len 36864
167962024-04-17T17:20:53.827Z19/30 Read block 489 len 8192
167972024-04-17T17:20:53.827Z20/30 Read block 250 len 32768
167982024-04-17T17:20:53.830Z21/30 Read block 212 len 16384
167992024-04-17T17:20:53.830Z22/30 Write block 219 len 24576 data: 10 12 11 10 12 13
168002024-04-17T17:20:53.831Z23/30 Read block 306 len 40960
168012024-04-17T17:20:53.831Z24/30 Flush
168022024-04-17T17:20:53.834Z25/30 Write block 269 len 8192 data: 18 15
168032024-04-17T17:20:53.838Z26/30 Write block 168 len 36864 data: 14 12 11 14 13 14 11 13 13
168042024-04-17T17:20:53.838Z27/30 Read block 445 len 32768
168052024-04-17T17:20:53.842Z28/30 Write block 279 len 36864 data: 9 10 9 10 9 10 10 8 9
168062024-04-17T17:20:53.842Z29/30 Write block 7 len 16384 data: 12 12 11 11
168072024-04-17T17:20:53.842Z30/30 Write block 560 len 8192 data: 12 12
168082024-04-17T17:20:53.842Z----------------------------------------------------------------
168092024-04-17T17:20:53.842Z Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:23
168102024-04-17T17:20:53.843ZStates: Active Active Active
168112024-04-17T17:20:53.843ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
168122024-04-17T17:20:53.843Z 8 Acked 1007 Write 4 Done Sent Done false
168132024-04-17T17:20:53.843Z 9 Acked 1008 Read 9 Done Sent Done false
168142024-04-17T17:20:53.843Z 10 Acked 1009 Read 8 Done Sent Done false
168152024-04-17T17:20:53.843Z 11 Acked 1010 Read 2 Done Sent Done false
168162024-04-17T17:20:53.843Z 12 Acked 1011 Read 3 Done Sent Done false
168172024-04-17T17:20:53.843Z 13 Acked 1012 Write 4 Done Sent Done false
168182024-04-17T17:20:53.843Z 14 Acked 1013 Write 9 Done Sent Done false
168192024-04-17T17:20:53.843Z 15 Acked 1014 Write 7 Done Sent Done false
168202024-04-17T17:20:53.843Z 16 Acked 1015 Write 8 Done Sent Done false
168212024-04-17T17:20:53.843Z 17 Acked 1016 Write 5 Done Sent Done false
168222024-04-17T17:20:53.843Z 18 Acked 1017 Read 9 Done Sent Done false
168232024-04-17T17:20:53.843Z 19 Acked 1018 Read 2 Done Sent Done false
168242024-04-17T17:20:53.843Z 20 Acked 1019 Read 8 Done Sent Done false
168252024-04-17T17:20:53.843Z 21 Acked 1020 Read 4 Done Sent Done false
168262024-04-17T17:20:53.843Z 22 Acked 1021 Write 6 Done Sent Done false
168272024-04-17T17:20:53.843Z 23 Acked 1022 Read 10 Done Sent Done false
168282024-04-17T17:20:53.843Z 24 Acked 1023 Flush 0 Done Sent Done false
168292024-04-17T17:20:53.843Z 25 Acked 1024 Write 2 Done Sent Done false
168302024-04-17T17:20:53.843Z 26 Acked 1025 Write 9 Done Sent Done false
168312024-04-17T17:20:53.843Z 27 Acked 1026 Read 8 Done Sent Done false
168322024-04-17T17:20:53.843Z 28 Acked 1027 Write 9 Done Sent Done false
168332024-04-17T17:20:53.844Z 29 Acked 1028 Write 4 Done Sent Done false
168342024-04-17T17:20:53.844Z 30 Acked 1029 Write 2 Sent Sent Sent false
168352024-04-17T17:20:53.844Z STATES DS:0 DS:1 DS:2 TOTAL
168362024-04-17T17:20:53.844Z New 0 0 0 0
168372024-04-17T17:20:53.844Z Sent 1 23 1 25
168382024-04-17T17:20:53.844Z Done 22 0 22 44
168392024-04-17T17:20:53.844Z Skipped 0 0 0 0
168402024-04-17T17:20:53.844Z Error 0 0 0 0
168412024-04-17T17:20:53.844ZLast Flush: 1023 1006 1023
168422024-04-17T17:20:53.844ZDownstairs last five completed: 1006 1005 1004 1003 1002
168432024-04-17T17:20:53.844ZUpstairs last five completed: 30 29 28 27 26
168442024-04-17T17:20:53.844Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:20:52.449607579Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1795}
168452024-04-17T17:20:53.844ZWrote out file "/tmp/test_repair_verify.data"
168462024-04-17T17:20:53.848Z
168472024-04-17T17:20:53.851ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
168482024-04-17T17:20:53.865ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
168492024-04-17T17:20:53.865Z 0 000-019 155 155 155 278 278 278 T F T <---
168502024-04-17T17:20:53.865Z 1 020-039 156 155 156 281 278 281 F F F <---
168512024-04-17T17:20:53.865Z 2 040-059 156 150 156 281 271 281 F F F <---
168522024-04-17T17:20:53.865Z 3 060-079 156 149 156 281 270 281 F F F <---
168532024-04-17T17:20:53.865Z 4 080-099 156 151 156 281 273 281 F F F <---
168542024-04-17T17:20:53.865Z 5 100-119 153 153 153 276 276 276 F F F
168552024-04-17T17:20:53.865Z 6 120-139 151 151 151 273 273 273 F F F
168562024-04-17T17:20:53.865Z 7 140-159 153 153 153 276 276 276 F F F
168572024-04-17T17:20:53.865Z 8 160-179 154 154 154 277 277 277 T F T <---
168582024-04-17T17:20:53.870Z 9 180-199 154 154 154 277 277 277 F F F
168592024-04-17T17:20:53.870Z 10 200-219 156 156 156 281 279 281 F F F <---
168602024-04-17T17:20:53.870Z 11 220-239 156 150 156 281 271 281 F F F <---
168612024-04-17T17:20:53.870Z 12 240-259 155 155 155 278 278 278 F F F
168622024-04-17T17:20:53.870Z 13 260-279 156 155 156 281 278 281 T F T <---
168632024-04-17T17:20:53.870Z 14 280-299 156 153 156 281 276 281 T F T <---
168642024-04-17T17:20:53.870Z 15 300-319 156 156 156 279 279 279 F F F
168652024-04-17T17:20:53.870Z 16 320-339 152 152 152 275 275 275 F F F
168662024-04-17T17:20:53.870Z 17 340-359 150 150 150 271 271 271 F F F
168672024-04-17T17:20:53.870Z 18 360-379 152 152 152 275 275 275 F F F
168682024-04-17T17:20:53.870Z 19 380-399 155 155 155 278 278 278 F F F
168692024-04-17T17:20:53.870Z 20 400-419 155 155 155 278 278 278 F F F
168702024-04-17T17:20:53.870Z 21 420-439 155 155 155 278 278 278 F F F
168712024-04-17T17:20:53.870Z 22 440-459 156 156 156 279 279 279 F F F
168722024-04-17T17:20:53.870Z 23 460-479 143 143 143 261 261 261 F F F
168732024-04-17T17:20:53.870Z 24 480-499 152 152 152 275 275 275 F F F
168742024-04-17T17:20:53.871Z 25 500-519 155 155 155 278 278 278 F F F
168752024-04-17T17:20:53.871Z 26 520-539 155 155 155 278 278 278 F F F
168762024-04-17T17:20:53.871Z 27 540-559 155 155 155 278 278 278 F F F
168772024-04-17T17:20:53.871Z 28 560-579 146 146 146 267 267 267 T F T <---
168782024-04-17T17:20:53.871Z 29 580-599 150 150 150 272 272 272 F F F
168792024-04-17T17:20:53.871ZMax gen: 156, Max flush: 281
168802024-04-17T17:20:53.871ZError: Difference in extent metadata found!
168812024-04-17T17:20:53.871ZA Difference in extent metadata is expected here
168822024-04-17T17:20:53.871ZOn loop 78
168832024-04-17T17:20:53.871Z
168842024-04-17T17:20:53.874ZVerifying data now
168852024-04-17T17:20:58.394ZNo change after verify
168862024-04-17T17:20:58.394ZLoop: 78 Downstairs dump after verify (and repair):
168872024-04-17T17:20:58.407ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
168882024-04-17T17:20:58.407Z 0 000-019 157 157 157 282 282 282 F F F
168892024-04-17T17:20:58.407Z 1 020-039 156 156 156 281 281 281 F F F
168902024-04-17T17:20:58.407Z 2 040-059 156 156 156 281 281 281 F F F
168912024-04-17T17:20:58.407Z 3 060-079 156 156 156 281 281 281 F F F
168922024-04-17T17:20:58.407Z 4 080-099 156 156 156 281 281 281 F F F
168932024-04-17T17:20:58.407Z 5 100-119 153 153 153 276 276 276 F F F
168942024-04-17T17:20:58.407Z 6 120-139 151 151 151 273 273 273 F F F
168952024-04-17T17:20:58.407Z 7 140-159 153 153 153 276 276 276 F F F
168962024-04-17T17:20:58.407Z 8 160-179 157 157 157 282 282 282 F F F
168972024-04-17T17:20:58.407Z 9 180-199 154 154 154 277 277 277 F F F
168982024-04-17T17:20:58.411Z 10 200-219 156 156 156 281 281 281 F F F
168992024-04-17T17:20:58.411Z 11 220-239 156 156 156 281 281 281 F F F
169002024-04-17T17:20:58.411Z 12 240-259 155 155 155 278 278 278 F F F
169012024-04-17T17:20:58.412Z 13 260-279 157 157 157 282 282 282 F F F
169022024-04-17T17:20:58.412Z 14 280-299 157 157 157 282 282 282 F F F
169032024-04-17T17:20:58.412Z 15 300-319 156 156 156 279 279 279 F F F
169042024-04-17T17:20:58.412Z 16 320-339 152 152 152 275 275 275 F F F
169052024-04-17T17:20:58.412Z 17 340-359 150 150 150 271 271 271 F F F
169062024-04-17T17:20:58.412Z 18 360-379 152 152 152 275 275 275 F F F
169072024-04-17T17:20:58.412Z 19 380-399 155 155 155 278 278 278 F F F
169082024-04-17T17:20:58.412Z 20 400-419 155 155 155 278 278 278 F F F
169092024-04-17T17:20:58.412Z 21 420-439 155 155 155 278 278 278 F F F
169102024-04-17T17:20:58.412Z 22 440-459 156 156 156 279 279 279 F F F
169112024-04-17T17:20:58.412Z 23 460-479 143 143 143 261 261 261 F F F
169122024-04-17T17:20:58.412Z 24 480-499 152 152 152 275 275 275 F F F
169132024-04-17T17:20:58.412Z 25 500-519 155 155 155 278 278 278 F F F
169142024-04-17T17:20:58.412Z 26 520-539 155 155 155 278 278 278 F F F
169152024-04-17T17:20:58.412Z 27 540-559 155 155 155 278 278 278 F F F
169162024-04-17T17:20:58.412Z 28 560-579 157 157 157 282 282 282 F F F
169172024-04-17T17:20:58.412Z 29 580-599 150 150 150 272 272 272 F F F
169182024-04-17T17:20:58.412ZMax gen: 157, Max flush: 282
169192024-04-17T17:20:58.412Z
169202024-04-17T17:20:58.412ZBegin loop 79
169212024-04-17T17:20:58.412ZDownstairs to restart: 1
169222024-04-17T17:20:58.429Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.038550013Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803}
169232024-04-17T17:20:58.429Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.03882707Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803}
169242024-04-17T17:20:58.435Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.038867322Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803}
169252024-04-17T17:20:58.435Z{"msg":"Crucible 0f923a3e-ae32-4993-b97b-0030058665b4 has session id: fa110b97-6065-47eb-a362-03cd89c78be6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.038897444Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169262024-04-17T17:20:58.435Z{Crucible runtime is spawned
169272024-04-17T17:20:58.435Z"msg":"Upstairs opts: Upstairs UUID: 0f923a3e-ae32-4993-b97b-0030058665b4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.038952538Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169282024-04-17T17:20:58.435Z{"msg":"Crucible stats registered with UUID: 0f923a3e-ae32-4993-b97b-0030058665b4","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.03899268Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169292024-04-17T17:20:58.435Z{"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.039020262Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803}
169302024-04-17T17:20:58.435Z{"msg":"0f923a3e-ae32-4993-b97b-0030058665b4 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.039189592Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169312024-04-17T17:20:58.436Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.039244365Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"client":"0","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169322024-04-17T17:20:58.436Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.039278167Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"client":"0","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169332024-04-17T17:20:58.436Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.039309579Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"client":"1","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169342024-04-17T17:20:58.436Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.039358412Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"client":"1","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169352024-04-17T17:20:58.436Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.039406995Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"client":"2","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169362024-04-17T17:20:58.436Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.039442117Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"client":"2","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169372024-04-17T17:20:58.436Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.039471869Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"":"io task","client":"0","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169382024-04-17T17:20:58.436Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.039523412Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"":"io task","client":"1","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169392024-04-17T17:20:58.436Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.039567365Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"":"io task","client":"2","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169402024-04-17T17:20:58.436Z{"msg":"ds_connection connected from Ok(127.0.0.1:35546)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.039620408Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"":"io task","client":"0","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169412024-04-17T17:20:58.436Z{"msg":"ds_connection connected from Ok(127.0.0.1:45176)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.039715964Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"":"io task","client":"2","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169422024-04-17T17:20:58.436Z{"msg":"ds_connection connected from Ok(127.0.0.1:38617)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.039765137Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"":"io task","client":"1","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169432024-04-17T17:20:58.436Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.040279498Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"client":"2","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169442024-04-17T17:20:58.436Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.04031598Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"client":"2","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169452024-04-17T17:20:58.436Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.040331501Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"client":"1","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169462024-04-17T17:20:58.436Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.040356073Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"client":"1","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169472024-04-17T17:20:58.436Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.040379424Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"client":"0","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169482024-04-17T17:20:58.436Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.040441738Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"client":"0","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169492024-04-17T17:20:58.436Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.040897866Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"client":"2","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169502024-04-17T17:20:58.436Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.040947439Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"client":"0","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169512024-04-17T17:20:58.436Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.041211515Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"client":"1","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169522024-04-17T17:20:58.436Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.041440698Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"client":"2","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169532024-04-17T17:20:58.436Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.041482321Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"client":"2","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169542024-04-17T17:20:58.437Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.041499282Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169552024-04-17T17:20:58.437Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.041573566Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"client":"0","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169562024-04-17T17:20:58.437Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.041600068Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"client":"0","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169572024-04-17T17:20:58.437Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.041614199Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169582024-04-17T17:20:58.437Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.041745217Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"client":"1","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169592024-04-17T17:20:58.437Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.041776939Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"client":"1","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169602024-04-17T17:20:58.437Z{"msg":"[0]R flush_numbers[0..12]: [282, 281, 281, 281, 281, 276, 273, 276, 282, 277, 281, 281]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.04179223Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"":"downstairs","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169612024-04-17T17:20:58.437Z{"msg":"[0]R generation[0..12]: [157, 156, 156, 156, 156, 153, 151, 153, 157, 154, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.041813391Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"":"downstairs","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169622024-04-17T17:20:58.437Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.041834853Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"":"downstairs","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169632024-04-17T17:20:58.437Z{"msg":"[1]R flush_numbers[0..12]: [282, 281, 281, 281, 281, 276, 273, 276, 282, 277, 281, 281]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.041849733Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"":"downstairs","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169642024-04-17T17:20:58.437Z{"msg":"[1]R generation[0..12]: [157, 156, 156, 156, 156, 153, 151, 153, 157, 154, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.041863584Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"":"downstairs","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169652024-04-17T17:20:58.437Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.041884705Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"":"downstairs","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169662024-04-17T17:20:58.437Z{"msg":"[2]R flush_numbers[0..12]: [282, 281, 281, 281, 281, 276, 273, 276, 282, 277, 281, 281]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.041898756Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"":"downstairs","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169672024-04-17T17:20:58.437Z{"msg":"[2]R generation[0..12]: [157, 156, 156, 156, 156, 153, 151, 153, 157, 154, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.041912407Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"":"downstairs","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169682024-04-17T17:20:58.437Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.041933058Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"":"downstairs","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169692024-04-17T17:20:58.437Z{"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.041946359Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"":"downstairs","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169702024-04-17T17:20:58.437Z{"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.04195925Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"":"downstairs","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169712024-04-17T17:20:58.437Z{"msg":"Next flush: 283","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.041984082Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"":"downstairs","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169722024-04-17T17:20:58.437Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.041998562Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"":"downstairs","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169732024-04-17T17:20:58.437Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
169742024-04-17T17:20:58.437Z,"time":"2024-04-17T17:20:57.042011803Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169752024-04-17T17:20:58.437Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.042026184Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"client":"0","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169762024-04-17T17:20:58.437Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.042039485Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"client":"1","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169772024-04-17T17:20:58.438Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.042052966Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"client":"2","":"downstairs","session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169782024-04-17T17:20:58.438Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.042073297Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169792024-04-17T17:20:58.438Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.042085628Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169802024-04-17T17:20:58.438Z{"msg":"0f923a3e-ae32-4993-b97b-0030058665b4 is now active with session: fa110b97-6065-47eb-a362-03cd89c78be6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.042098138Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169812024-04-17T17:20:58.438Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.042116439Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803,"session_id":"fa110b97-6065-47eb-a362-03cd89c78be6"}
169822024-04-17T17:20:58.438Z{"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:20:57.04212867Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803}
169832024-04-17T17:20:58.442ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
169842024-04-17T17:20:58.442ZLoading write count information from file "/tmp/test_repair_verify.data"
169852024-04-17T17:20:58.443ZRun Repair workload
169862024-04-17T17:20:58.443Z01/30 Write block 86 len 20480 data: 12 17 15 16 17
169872024-04-17T17:20:58.443Z02/30 Read block 451 len 20480
169882024-04-17T17:20:58.443Z03/30 Write block 72 len 20480 data: 12 9 12 13 12
169892024-04-17T17:20:58.443Z04/30 Read block 182 len 16384
169902024-04-17T17:20:58.443Z05/30 Read block 333 len 40960
169912024-04-17T17:20:58.443Z06/30 Read block 360 len 24576
169922024-04-17T17:20:58.443Z07/30 Write block 315 len 12288 data: 16 14 15
169932024-04-17T17:20:58.443Z08/30 Read block 577 len 32768
169942024-04-17T17:20:58.443Z09/30 Write block 208 len 24576 data: 14 15 15 15 15 16
169952024-04-17T17:20:58.446Z10/30 Write block 430 len 36864 data: 8 9 11 15 14 14 14 15 14
169962024-04-17T17:20:58.446Z11/30 Write block 540 len 12288 data: 12 13 17
169972024-04-17T17:20:58.446Z12/30 Write block 291 len 40960 data: 15 12 15 12 9 7 7 8 9 9
169982024-04-17T17:20:58.447Z13/30 Read block 302 len 20480
169992024-04-17T17:20:58.450Z14/30 Write block 58 len 24576 data: 17 16 16 14 13 16
170002024-04-17T17:20:58.450Z15/30 Read block 463 len 36864
170012024-04-17T17:20:58.450Z16/30 Read block 459 len 32768
170022024-04-17T17:20:58.450Z17/30 Read block 112 len 36864
170032024-04-17T17:20:58.453Z18/30 Write block 128 len 36864 data: 9 10 12 13 13 9 10 9 12
170042024-04-17T17:20:58.453Z19/30 Read block 379 len 4096
170052024-04-17T17:20:58.453Z20/30 Read block 44 len 12288
170062024-04-17T17:20:58.453Z21/30 Write block 476 len 24576 data: 13 17 18 18 19 17
170072024-04-17T17:20:58.453Z22/30 Read block 560 len 28672
170082024-04-17T17:20:58.457Z23/30 Write block 507 len 24576 data: 11 11 10 11 10 12
170092024-04-17T17:20:58.457Z24/30 Write block 290 len 40960 data: 16 16 13 16 13 10 8 8 9 10
170102024-04-17T17:20:58.457Z25/30 Read block 20 len 12288
170112024-04-17T17:20:58.460Z26/30 Write block 217 len 36864 data: 10 11 11 13 12 11 13 14 12
170122024-04-17T17:20:58.460Z27/30 Read block 303 len 4096
170132024-04-17T17:20:58.460Z28/30 Read block 434 len 20480
170142024-04-17T17:20:58.464Z29/30 Read block 564 len 4096
170152024-04-17T17:20:58.464Z30/30 Write block 235 len 36864 data: 17 15 12 10 11 10 13 12 14
170162024-04-17T17:20:58.464Z----------------------------------------------------------------
170172024-04-17T17:20:58.464Z Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30
170182024-04-17T17:20:58.464ZStates: Active Active Active
170192024-04-17T17:20:58.464ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
170202024-04-17T17:20:58.464Z 1 Acked 1000 Write 5 Done Done Done false
170212024-04-17T17:20:58.464Z 2 Acked 1001 Read 5 Done Done Done false
170222024-04-17T17:20:58.465Z 3 Acked 1002 Write 5 Done Sent Done false
170232024-04-17T17:20:58.465Z 4 Acked 1003 Read 4 Done Sent Done false
170242024-04-17T17:20:58.465Z 5 Acked 1004 Read 10 Done Sent Done false
170252024-04-17T17:20:58.465Z 6 Acked 1005 Read 6 Done Sent Done false
170262024-04-17T17:20:58.465Z 7 Acked 1006 Write 3 Done Sent Done false
170272024-04-17T17:20:58.465Z 8 Acked 1007 Read 8 Done Sent Done false
170282024-04-17T17:20:58.465Z 9 Acked 1008 Write 6 Done Sent Done false
170292024-04-17T17:20:58.465Z 10 Acked 1009 Write 9 Done Sent Done false
170302024-04-17T17:20:58.465Z 11 Acked 1010 Write 3 Done Sent Done false
170312024-04-17T17:20:58.465Z 12 Acked 1011 Write 10 Done Sent Done false
170322024-04-17T17:20:58.465Z 13 Acked 1012 Read 5 Done Sent Done false
170332024-04-17T17:20:58.465Z 14 Acked 1013 Write 6 Done Sent Done false
170342024-04-17T17:20:58.465Z 15 Acked 1014 Read 9 Done Sent Done false
170352024-04-17T17:20:58.465Z 16 Acked 1015 Read 8 Done Sent Done false
170362024-04-17T17:20:58.465Z 17 Acked 1016 Read 9 Done Sent Done false
170372024-04-17T17:20:58.465Z 18 Acked 1017 Write 9 Done Sent Done false
170382024-04-17T17:20:58.465Z 19 Acked 1018 Read 1 Done Sent Done false
170392024-04-17T17:20:58.465Z 20 Acked 1019 Read 3 Done Sent Done false
170402024-04-17T17:20:58.465Z 21 Acked 1020 Write 6 Done Sent Done false
170412024-04-17T17:20:58.465Z 22 Acked 1021 Read 7 Done Sent Done false
170422024-04-17T17:20:58.465Z 23 Acked 1022 Write 6 Done Sent Done false
170432024-04-17T17:20:58.465Z 24 Acked 1023 Write 10 Done Sent Done false
170442024-04-17T17:20:58.465Z 25 Acked 1024 Read 3 Done Sent Done false
170452024-04-17T17:20:58.465Z 26 Acked 1025 Write 9 Done Sent Done false
170462024-04-17T17:20:58.466Z 27 Acked 1026 Read 1 Done Sent Done false
170472024-04-17T17:20:58.466Z 28 Acked 1027 Read 5 Done Sent Done false
170482024-04-17T17:20:58.466Z 29 Acked 1028 Read 1 Done Sent Done false
170492024-04-17T17:20:58.466Z 30 Acked 1029 Write 9 Sent Sent Sent false
170502024-04-17T17:20:58.466Z STATES DS:0 DS:1 DS:2 TOTAL
170512024-04-17T17:20:58.466Z New 0 0 0 0
170522024-04-17T17:20:58.466Z Sent 1 28 1 30
170532024-04-17T17:20:58.466Z Done 29 2 29 60
170542024-04-17T17:20:58.466Z Skipped 0 0 0 0
170552024-04-17T17:20:58.466Z Error 0 0 0 0
170562024-04-17T17:20:58.466ZLast Flush: 0 0 0
170572024-04-17T17:20:58.466ZDownstairs last five completed:
170582024-04-17T17:20:58.466ZUpstairs last five completed: 30 29 28 27 26
170592024-04-17T17:20:58.466Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:20:57.072245105Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1803}
170602024-04-17T17:20:58.466ZWrote out file "/tmp/test_repair_verify.data"
170612024-04-17T17:20:58.471Z
170622024-04-17T17:20:58.471ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
170632024-04-17T17:20:58.488ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
170642024-04-17T17:20:58.488Z 0 000-019 157 157 157 282 282 282 F F F
170652024-04-17T17:20:58.489Z 1 020-039 156 156 156 281 281 281 F F F
170662024-04-17T17:20:58.489Z 2 040-059 156 156 156 281 281 281 T F T <---
170672024-04-17T17:20:58.489Z 3 060-079 156 156 156 281 281 281 T F T <---
170682024-04-17T17:20:58.489Z 4 080-099 156 156 156 281 281 281 T T T
170692024-04-17T17:20:58.489Z 5 100-119 153 153 153 276 276 276 F F F
170702024-04-17T17:20:58.493Z 6 120-139 151 151 151 273 273 273 T F T <---
170712024-04-17T17:20:58.493Z 7 140-159 153 153 153 276 276 276 F F F
170722024-04-17T17:20:58.493Z 8 160-179 157 157 157 282 282 282 F F F
170732024-04-17T17:20:58.493Z 9 180-199 154 154 154 277 277 277 F F F
170742024-04-17T17:20:58.493Z 10 200-219 156 156 156 281 281 281 T F T <---
170752024-04-17T17:20:58.493Z 11 220-239 156 156 156 281 281 281 T F T <---
170762024-04-17T17:20:58.493Z 12 240-259 155 155 155 278 278 278 T F T <---
170772024-04-17T17:20:58.493Z 13 260-279 157 157 157 282 282 282 F F F
170782024-04-17T17:20:58.493Z 14 280-299 157 157 157 282 282 282 T F T <---
170792024-04-17T17:20:58.493Z 15 300-319 156 156 156 279 279 279 T F T <---
170802024-04-17T17:20:58.493Z 16 320-339 152 152 152 275 275 275 F F F
170812024-04-17T17:20:58.493Z 17 340-359 150 150 150 271 271 271 F F F
170822024-04-17T17:20:58.493Z 18 360-379 152 152 152 275 275 275 F F F
170832024-04-17T17:20:58.493Z 19 380-399 155 155 155 278 278 278 F F F
170842024-04-17T17:20:58.493Z 20 400-419 155 155 155 278 278 278 F F F
170852024-04-17T17:20:58.493Z 21 420-439 155 155 155 278 278 278 T F T <---
170862024-04-17T17:20:58.493Z 22 440-459 156 156 156 279 279 279 F F F
170872024-04-17T17:20:58.494Z 23 460-479 143 143 143 261 261 261 T F T <---
170882024-04-17T17:20:58.494Z 24 480-499 152 152 152 275 275 275 T F T <---
170892024-04-17T17:20:58.494Z 25 500-519 155 155 155 278 278 278 T F T <---
170902024-04-17T17:20:58.494Z 26 520-539 155 155 155 278 278 278 F F F
170912024-04-17T17:20:58.494Z 27 540-559 155 155 155 278 278 278 T F T <---
170922024-04-17T17:20:58.494Z 28 560-579 157 157 157 282 282 282 F F F
170932024-04-17T17:20:58.494Z 29 580-599 150 150 150 272 272 272 F F F
170942024-04-17T17:20:58.494ZMax gen: 157, Max flush: 282
170952024-04-17T17:20:58.494ZError: Difference in extent metadata found!
170962024-04-17T17:20:58.494ZA Difference in extent metadata is expected here
170972024-04-17T17:20:58.494ZOn loop 79
170982024-04-17T17:20:58.494Z
170992024-04-17T17:20:58.497ZVerifying data now
171002024-04-17T17:21:03.163ZNo change after verify
171012024-04-17T17:21:03.163ZLoop: 79 Downstairs dump after verify (and repair):
171022024-04-17T17:21:03.177ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
171032024-04-17T17:21:03.177Z 0 000-019 157 157 157 282 282 282 F F F
171042024-04-17T17:21:03.182Z 1 020-039 156 156 156 281 281 281 F F F
171052024-04-17T17:21:03.182Z 2 040-059 158 158 158 283 283 283 F F F
171062024-04-17T17:21:03.182Z 3 060-079 158 158 158 283 283 283 F F F
171072024-04-17T17:21:03.182Z 4 080-099 158 158 158 283 283 283 F F F
171082024-04-17T17:21:03.182Z 5 100-119 153 153 153 276 276 276 F F F
171092024-04-17T17:21:03.182Z 6 120-139 158 158 158 283 283 283 F F F
171102024-04-17T17:21:03.182Z 7 140-159 153 153 153 276 276 276 F F F
171112024-04-17T17:21:03.182Z 8 160-179 157 157 157 282 282 282 F F F
171122024-04-17T17:21:03.182Z 9 180-199 154 154 154 277 277 277 F F F
171132024-04-17T17:21:03.182Z 10 200-219 158 158 158 283 283 283 F F F
171142024-04-17T17:21:03.182Z 11 220-239 158 158 158 283 283 283 F F F
171152024-04-17T17:21:03.182Z 12 240-259 158 158 158 283 283 283 F F F
171162024-04-17T17:21:03.182Z 13 260-279 157 157 157 282 282 282 F F F
171172024-04-17T17:21:03.182Z 14 280-299 158 158 158 283 283 283 F F F
171182024-04-17T17:21:03.182Z 15 300-319 158 158 158 283 283 283 F F F
171192024-04-17T17:21:03.182Z 16 320-339 152 152 152 275 275 275 F F F
171202024-04-17T17:21:03.182Z 17 340-359 150 150 150 271 271 271 F F F
171212024-04-17T17:21:03.182Z 18 360-379 152 152 152 275 275 275 F F F
171222024-04-17T17:21:03.183Z 19 380-399 155 155 155 278 278 278 F F F
171232024-04-17T17:21:03.183Z 20 400-419 155 155 155 278 278 278 F F F
171242024-04-17T17:21:03.183Z 21 420-439 158 158 158 283 283 283 F F F
171252024-04-17T17:21:03.183Z 22 440-459 156 156 156 279 279 279 F F F
171262024-04-17T17:21:03.183Z 23 460-479 158 158 158 283 283 283 F F F
171272024-04-17T17:21:03.183Z 24 480-499 158 158 158 283 283 283 F F F
171282024-04-17T17:21:03.183Z 25 500-519 158 158 158 283 283 283 F F F
171292024-04-17T17:21:03.183Z 26 520-539 155 155 155 278 278 278 F F F
171302024-04-17T17:21:03.183Z 27 540-559 158 158 158 283 283 283 F F F
171312024-04-17T17:21:03.183Z 28 560-579 157 157 157 282 282 282 F F F
171322024-04-17T17:21:03.183Z 29 580-599 150 150 150 272 272 272 F F F
171332024-04-17T17:21:03.183ZMax gen: 158, Max flush: 283
171342024-04-17T17:21:03.183Z
171352024-04-17T17:21:03.183ZBegin loop 80
171362024-04-17T17:21:03.183ZDownstairs to restart: 0
171372024-04-17T17:21:03.201Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.81083438Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811}
171382024-04-17T17:21:03.208Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.811099765Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811}
171392024-04-17T17:21:03.208ZCrucible runtime is spawned
171402024-04-17T17:21:03.208Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.811180681Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811}
171412024-04-17T17:21:03.208Z{"msg":"Crucible 8f55f827-1ca7-4148-a329-67b37555343a has session id: 2d0f9423-7881-48da-bc7f-248cc20d7e99","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.811211172Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171422024-04-17T17:21:03.208Z{"msg":"Upstairs opts: Upstairs UUID: 8f55f827-1ca7-4148-a329-67b37555343a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.811247494Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171432024-04-17T17:21:03.208Z{"msg":"Crucible stats registered with UUID: 8f55f827-1ca7-4148-a329-67b37555343a","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.811266386Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171442024-04-17T17:21:03.209Z{"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.811283467Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811}
171452024-04-17T17:21:03.209Z{"msg":"8f55f827-1ca7-4148-a329-67b37555343a active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.81134369Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171462024-04-17T17:21:03.209Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.811373602Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"client":"0","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171472024-04-17T17:21:03.209Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.811416405Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"client":"0","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171482024-04-17T17:21:03.209Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.811468718Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"client":"1","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171492024-04-17T17:21:03.209Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.81150273Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"client":"1","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171502024-04-17T17:21:03.209Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.811613507Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"client":"2","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171512024-04-17T17:21:03.209Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.81167017Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"client":"2","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171522024-04-17T17:21:03.209Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.811716933Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"":"io task","client":"2","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171532024-04-17T17:21:03.209Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.811803148Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"":"io task","client":"0","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171542024-04-17T17:21:03.209Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.811851591Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"":"io task","client":"1","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171552024-04-17T17:21:03.209Z{"msg":"ds_connection connected from Ok(127.0.0.1:43893)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.811879843Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"":"io task","client":"1","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171562024-04-17T17:21:03.209Z{"msg":"ds_connection connected from Ok(127.0.0.1:42732)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.811925295Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"":"io task","client":"2","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171572024-04-17T17:21:03.209Z{"msg":"ds_connection connected from Ok(127.0.0.1:53419)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.811956997Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"":"io task","client":"0","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171582024-04-17T17:21:03.209Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.812596306Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"client":"1","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171592024-04-17T17:21:03.209Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.812646679Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"client":"1","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171602024-04-17T17:21:03.209Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.812672201Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"client":"2","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171612024-04-17T17:21:03.209Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.812693332Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"client":"2","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171622024-04-17T17:21:03.209Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.813090686Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"client":"0","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171632024-04-17T17:21:03.209Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.813128868Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"client":"0","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171642024-04-17T17:21:03.209Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.813145669Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"client":"1","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171652024-04-17T17:21:03.209Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.813165551Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"client":"2","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171662024-04-17T17:21:03.209Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.81381424Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"client":"0","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171672024-04-17T17:21:03.210Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.813863473Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"client":"1","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171682024-04-17T17:21:03.210Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.813889755Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"client":"1","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171692024-04-17T17:21:03.210Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.813918746Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171702024-04-17T17:21:03.210Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.814000711Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"client":"2","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171712024-04-17T17:21:03.210Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.814036434Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"client":"2","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171722024-04-17T17:21:03.210Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.814053055Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171732024-04-17T17:21:03.210Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.81447286Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"client":"0","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171742024-04-17T17:21:03.210Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.814528513Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"client":"0","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171752024-04-17T17:21:03.210Z{"msg":"[0]R flush_numbers[0..12]: [282, 281, 283, 283, 283, 276, 283, 276, 282, 277, 283, 283]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.814570526Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"":"downstairs","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171762024-04-17T17:21:03.210Z{"msg":"[0]R generation[0..12]: [157, 156, 158, 158, 158, 153, 158, 153, 157, 154, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.814592287Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"":"downstairs","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171772024-04-17T17:21:03.210ZWait for a query_work_queue command to finish before sending IO
171782024-04-17T17:21:03.210Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.814624429Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"":"downstairs","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171792024-04-17T17:21:03.210Z{"msg":"[1]R flush_numbers[0..12]: [282, 281, 283, 283, 283, 276, 283, 276, 282, 277, 283, 283]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.8146408Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"":"downstairs","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171802024-04-17T17:21:03.210Z{"msg":"[1]R generation[0..12]: [157, 156, 158, 158, 158, 153, 158, 153, 157, 154, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.814672762Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"":"downstairs","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171812024-04-17T17:21:03.210Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.814692523Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"":"downstairs","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171822024-04-17T17:21:03.210Z{"msg":"[2]R flush_numbers[0..12]: [282, 281, 283, 283, 283, 276, 283, 276, 282, 277, 283, 283]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.814722625Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"":"downstairs","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171832024-04-17T17:21:03.210Z{"msg":"[2]R generation[0..12]: [157, 156, 158, 158, 158, 153, 158, 153, 157, 154, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.814737986Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"":"downstairs","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171842024-04-17T17:21:03.210Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.814769258Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"":"downstairs","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171852024-04-17T17:21:03.210Z{"msg":"Max found gen is 159","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.814789649Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"":"downstairs","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171862024-04-17T17:21:03.210Z{"msg":"Generation requested: 160 >= found:159","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.814817961Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"":"downstairs","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171872024-04-17T17:21:03.210Z{"msg":"Next flush: 284","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.814832042Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"":"downstairs","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171882024-04-17T17:21:03.210Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.814862104Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"":"downstairs","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171892024-04-17T17:21:03.211Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.814878345Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171902024-04-17T17:21:03.211Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.814907046Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"client":"0","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171912024-04-17T17:21:03.211Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
171922024-04-17T17:21:03.211Z,"time":"2024-04-17T17:21:01.814922437Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"client":"1","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171932024-04-17T17:21:03.211Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.814952799Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"client":"2","":"downstairs","session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171942024-04-17T17:21:03.211Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.81496773Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171952024-04-17T17:21:03.211Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.815007322Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171962024-04-17T17:21:03.211Z{"msg":"8f55f827-1ca7-4148-a329-67b37555343a is now active with session: 2d0f9423-7881-48da-bc7f-248cc20d7e99","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.815040164Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171972024-04-17T17:21:03.211Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.815054245Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811,"session_id":"2d0f9423-7881-48da-bc7f-248cc20d7e99"}
171982024-04-17T17:21:03.211Z{"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:01.815083857Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811}
171992024-04-17T17:21:03.216ZLoading write count information from file "/tmp/test_repair_verify.data"
172002024-04-17T17:21:03.216ZRun Repair workload
172012024-04-17T17:21:03.216Z01/30 Write block 194 len 12288 data: 6 7 8
172022024-04-17T17:21:03.216Z02/30 Write block 150 len 8192 data: 9 7
172032024-04-17T17:21:03.216Z03/30 Write block 307 len 4096 data: 12
172042024-04-17T17:21:03.216Z04/30 Read block 81 len 36864
172052024-04-17T17:21:03.216Z05/30 Read block 352 len 36864
172062024-04-17T17:21:03.216Z06/30 Read block 429 len 12288
172072024-04-17T17:21:03.216Z07/30 Write block 393 len 4096 data: 13
172082024-04-17T17:21:03.216Z08/30 Write block 194 len 20480 data: 7 8 9 9 12
172092024-04-17T17:21:03.216Z09/30 Write block 148 len 28672 data: 10 11 10 8 7 7 10
172102024-04-17T17:21:03.216Z10/30 Write block 426 len 28672 data: 9 9 8 10 9 10 12
172112024-04-17T17:21:03.216Z11/30 Write block 546 len 12288 data: 12 12 15
172122024-04-17T17:21:03.221Z12/30 Write block 311 len 24576 data: 11 13 14 17 17 15
172132024-04-17T17:21:03.222Z13/30 Read block 287 len 40960
172142024-04-17T17:21:03.222Z14/30 Write block 440 len 20480 data: 13 13 13 13 12
172152024-04-17T17:21:03.222Z15/30 Flush
172162024-04-17T17:21:03.225Z16/30 Write block 285 len 40960 data: 11 9 10 10 14 17 17 14 17 14
172172024-04-17T17:21:03.225Z17/30 Write block 416 len 24576 data: 14 13 11 13 12 8
172182024-04-17T17:21:03.225Z18/30 Read block 336 len 40960
172192024-04-17T17:21:03.229Z19/30 Read block 222 len 20480
172202024-04-17T17:21:03.229Z20/30 Write block 305 len 28672 data: 16 13 13 14 13 14 12
172212024-04-17T17:21:03.229Z21/30 Write block 333 len 32768 data: 8 10 12 16 17 17 16 15
172222024-04-17T17:21:03.229Z22/30 Read block 367 len 24576
172232024-04-17T17:21:03.234Z23/30 Read block 158 len 28672
172242024-04-17T17:21:03.234Z24/30 Read block 483 len 28672
172252024-04-17T17:21:03.234Z25/30 Write block 455 len 28672 data: 8 8 8 10 11 12 11
172262024-04-17T17:21:03.238Z26/30 Write block 522 len 16384 data: 13 13 10 10
172272024-04-17T17:21:03.238Z27/30 Read block 115 len 40960
172282024-04-17T17:21:03.238Z28/30 Read block 448 len 32768
172292024-04-17T17:21:03.238Z29/30 Write block 277 len 32768 data: 8 9 10 11 10 11 10 11
172302024-04-17T17:21:03.238Z30/30 Write block 361 len 40960 data: 9 8 7 10 10 10 10 14 14 15
172312024-04-17T17:21:03.244Z----------------------------------------------------------------
172322024-04-17T17:21:03.244Z Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30
172332024-04-17T17:21:03.244ZStates: Active Active Active
172342024-04-17T17:21:03.244ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
172352024-04-17T17:21:03.244Z 1 Acked 1000 Write 3 Done Done Done false
172362024-04-17T17:21:03.244Z 2 Acked 1001 Write 2 Done Done Done false
172372024-04-17T17:21:03.244Z 3 Acked 1002 Write 1 Sent Done Done false
172382024-04-17T17:21:03.244Z 4 Acked 1003 Read 9 Sent Done Done false
172392024-04-17T17:21:03.244Z 5 Acked 1004 Read 9 Sent Done Done false
172402024-04-17T17:21:03.244Z 6 Acked 1005 Read 3 Sent Done Done false
172412024-04-17T17:21:03.244Z 7 Acked 1006 Write 1 Sent Done Done false
172422024-04-17T17:21:03.244Z 8 Acked 1007 Write 5 Sent Done Done false
172432024-04-17T17:21:03.244Z 9 Acked 1008 Write 7 Sent Done Done false
172442024-04-17T17:21:03.244Z 10 Acked 1009 Write 7 Sent Done Done false
172452024-04-17T17:21:03.244Z 11 Acked 1010 Write 3 Sent Done Done false
172462024-04-17T17:21:03.244Z 12 Acked 1011 Write 6 Sent Done Done false
172472024-04-17T17:21:03.244Z 13 Acked 1012 Read 10 Sent Done Done false
172482024-04-17T17:21:03.244Z 14 Acked 1013 Write 5 Sent Done Done false
172492024-04-17T17:21:03.244Z 15 Acked 1014 Flush 0 Sent Done Done false
172502024-04-17T17:21:03.244Z 16 Acked 1015 Write 10 Sent Done Done false
172512024-04-17T17:21:03.245Z 17 Acked 1016 Write 6 Sent Done Done false
172522024-04-17T17:21:03.245Z 18 Acked 1017 Read 10 Sent Done Done false
172532024-04-17T17:21:03.245Z 19 Acked 1018 Read 5 Sent Done Done false
172542024-04-17T17:21:03.245Z 20 Acked 1019 Write 7 Sent Done Done false
172552024-04-17T17:21:03.245Z 21 Acked 1020 Write 8 Sent Done Done false
172562024-04-17T17:21:03.245Z 22 Acked 1021 Read 6 Sent Done Done false
172572024-04-17T17:21:03.245Z 23 Acked 1022 Read 7 Sent Done Done false
172582024-04-17T17:21:03.245Z 24 Acked 1023 Read 7 Sent Done Done false
172592024-04-17T17:21:03.245Z 25 Acked 1024 Write 7 Sent Done Done false
172602024-04-17T17:21:03.245Z 26 Acked 1025 Write 4 Sent Done Done false
172612024-04-17T17:21:03.245Z 27 Acked 1026 Read 10 Sent Done Done false
172622024-04-17T17:21:03.245Z 28 Acked 1027 Read 8 Sent Done Done false
172632024-04-17T17:21:03.245Z 29 Acked 1028 Write 8 Sent Done Done false
172642024-04-17T17:21:03.245Z 30 Acked 1029 Write 10 Sent Sent Sent false
172652024-04-17T17:21:03.245Z STATES DS:0 DS:1 DS:2 TOTAL
172662024-04-17T17:21:03.245Z New 0 0 0 0
172672024-04-17T17:21:03.245Z Sent 28 1 1 30
172682024-04-17T17:21:03.245Z Done 2 29 29 60
172692024-04-17T17:21:03.245Z Skipped 0 0 0 0
172702024-04-17T17:21:03.245Z Error 0 0 0 0
172712024-04-17T17:21:03.245ZLast Flush: 0 1014 1014
172722024-04-17T17:21:03.245ZDownstairs last five completed:
172732024-04-17T17:21:03.245ZUpstairs last five completed: 30 29 28 27 26
172742024-04-17T17:21:03.246Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:21:01.847934868Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1811}
172752024-04-17T17:21:03.246ZWrote out file "/tmp/test_repair_verify.data"
172762024-04-17T17:21:03.250Z
172772024-04-17T17:21:03.250ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
172782024-04-17T17:21:03.266ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
172792024-04-17T17:21:03.266Z 0 000-019 157 157 157 282 282 282 F F F
172802024-04-17T17:21:03.266Z 1 020-039 156 156 156 281 281 281 F F F
172812024-04-17T17:21:03.266Z 2 040-059 158 158 158 283 283 283 F F F
172822024-04-17T17:21:03.266Z 3 060-079 158 158 158 283 283 283 F F F
172832024-04-17T17:21:03.267Z 4 080-099 158 158 158 283 283 283 F F F
172842024-04-17T17:21:03.267Z 5 100-119 153 153 153 276 276 276 F F F
172852024-04-17T17:21:03.267Z 6 120-139 158 158 158 283 283 283 F F F
172862024-04-17T17:21:03.267Z 7 140-159 153 160 160 276 284 284 T F F <---
172872024-04-17T17:21:03.267Z 8 160-179 157 157 157 282 282 282 F F F
172882024-04-17T17:21:03.267Z 9 180-199 154 160 160 277 284 284 T F F <---
172892024-04-17T17:21:03.267Z 10 200-219 158 158 158 283 283 283 F F F
172902024-04-17T17:21:03.271Z 11 220-239 158 158 158 283 283 283 F F F
172912024-04-17T17:21:03.271Z 12 240-259 158 158 158 283 283 283 F F F
172922024-04-17T17:21:03.271Z 13 260-279 157 157 157 282 282 282 F T T <---
172932024-04-17T17:21:03.271Z 14 280-299 158 158 158 283 283 283 F T T <---
172942024-04-17T17:21:03.271Z 15 300-319 158 160 160 283 284 284 F T T <---
172952024-04-17T17:21:03.271Z 16 320-339 152 152 152 275 275 275 F T T <---
172962024-04-17T17:21:03.271Z 17 340-359 150 150 150 271 271 271 F T T <---
172972024-04-17T17:21:03.271Z 18 360-379 152 152 152 275 275 275 F T T <---
172982024-04-17T17:21:03.271Z 19 380-399 155 160 160 278 284 284 F F F <---
172992024-04-17T17:21:03.271Z 20 400-419 155 155 155 278 278 278 F T T <---
173002024-04-17T17:21:03.271Z 21 420-439 158 160 160 283 284 284 F T T <---
173012024-04-17T17:21:03.271Z 22 440-459 156 160 160 279 284 284 F T T <---
173022024-04-17T17:21:03.271Z 23 460-479 158 158 158 283 283 283 F T T <---
173032024-04-17T17:21:03.272Z 24 480-499 158 158 158 283 283 283 F F F
173042024-04-17T17:21:03.272Z 25 500-519 158 158 158 283 283 283 F F F
173052024-04-17T17:21:03.272Z 26 520-539 155 155 155 278 278 278 F T T <---
173062024-04-17T17:21:03.272Z 27 540-559 158 160 160 283 284 284 F F F <---
173072024-04-17T17:21:03.272Z 28 560-579 157 157 157 282 282 282 F F F
173082024-04-17T17:21:03.272Z 29 580-599 150 150 150 272 272 272 F F F
173092024-04-17T17:21:03.272ZMax gen: 160, Max flush: 284
173102024-04-17T17:21:03.272ZError: Difference in extent metadata found!
173112024-04-17T17:21:03.272ZA Difference in extent metadata is expected here
173122024-04-17T17:21:03.272ZOn loop 80
173132024-04-17T17:21:03.272Z
173142024-04-17T17:21:03.276ZVerifying data now
173152024-04-17T17:21:17.974ZNo change after verify
173162024-04-17T17:21:17.974ZLoop: 80 Downstairs dump after verify (and repair):
173172024-04-17T17:21:17.989ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
173182024-04-17T17:21:17.989Z 0 000-019 157 157 157 282 282 282 F F F
173192024-04-17T17:21:17.993Z 1 020-039 156 156 156 281 281 281 F F F
173202024-04-17T17:21:17.993Z 2 040-059 158 158 158 283 283 283 F F F
173212024-04-17T17:21:17.993Z 3 060-079 158 158 158 283 283 283 F F F
173222024-04-17T17:21:17.993Z 4 080-099 158 158 158 283 283 283 F F F
173232024-04-17T17:21:17.993Z 5 100-119 153 153 153 276 276 276 F F F
173242024-04-17T17:21:17.993Z 6 120-139 158 158 158 283 283 283 F F F
173252024-04-17T17:21:17.993Z 7 140-159 160 160 160 284 284 284 F F F
173262024-04-17T17:21:17.993Z 8 160-179 157 157 157 282 282 282 F F F
173272024-04-17T17:21:17.993Z 9 180-199 160 160 160 284 284 284 F F F
173282024-04-17T17:21:17.994Z 10 200-219 158 158 158 283 283 283 F F F
173292024-04-17T17:21:17.994Z 11 220-239 158 158 158 283 283 283 F F F
173302024-04-17T17:21:17.994Z 12 240-259 158 158 158 283 283 283 F F F
173312024-04-17T17:21:17.994Z 13 260-279 161 161 161 285 285 285 F F F
173322024-04-17T17:21:17.994Z 14 280-299 161 161 161 285 285 285 F F F
173332024-04-17T17:21:17.994Z 15 300-319 161 161 161 285 285 285 F F F
173342024-04-17T17:21:17.994Z 16 320-339 161 161 161 285 285 285 F F F
173352024-04-17T17:21:17.994Z 17 340-359 161 161 161 285 285 285 F F F
173362024-04-17T17:21:17.994Z 18 360-379 161 161 161 285 285 285 F F F
173372024-04-17T17:21:17.994Z 19 380-399 160 160 160 284 284 284 F F F
173382024-04-17T17:21:17.994Z 20 400-419 161 161 161 285 285 285 F F F
173392024-04-17T17:21:17.994Z 21 420-439 161 161 161 285 285 285 F F F
173402024-04-17T17:21:17.994Z 22 440-459 161 161 161 285 285 285 F F F
173412024-04-17T17:21:17.994Z 23 460-479 161 161 161 285 285 285 F F F
173422024-04-17T17:21:17.994Z 24 480-499 158 158 158 283 283 283 F F F
173432024-04-17T17:21:17.994Z 25 500-519 158 158 158 283 283 283 F F F
173442024-04-17T17:21:17.994Z 26 520-539 161 161 161 285 285 285 F F F
173452024-04-17T17:21:17.994Z 27 540-559 160 160 160 284 284 284 F F F
173462024-04-17T17:21:17.994Z 28 560-579 157 157 157 282 282 282 F F F
173472024-04-17T17:21:17.994Z 29 580-599 150 150 150 272 272 272 F F F
173482024-04-17T17:21:17.994ZMax gen: 161, Max flush: 285
173492024-04-17T17:21:17.994Z
173502024-04-17T17:21:17.994ZBegin loop 81
173512024-04-17T17:21:17.995ZDownstairs to restart: 0
173522024-04-17T17:21:18.012Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.621259358Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819}
173532024-04-17T17:21:18.017Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.621576238Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819}
173542024-04-17T17:21:18.018Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.62160927Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819}
173552024-04-17T17:21:18.018ZCrucible runtime is spawned
173562024-04-17T17:21:18.018Z{"msg":"Crucible e2099aea-f183-4ceb-b6cc-96c7125563b2 has session id: 362aacdc-2e32-46b1-a04f-d5bf90f9142f","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.621659693Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173572024-04-17T17:21:18.018Z{"msg":"Upstairs opts: Upstairs UUID: e2099aea-f183-4ceb-b6cc-96c7125563b2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.621732687Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173582024-04-17T17:21:18.018Z{"msg":"Crucible stats registered with UUID: e2099aea-f183-4ceb-b6cc-96c7125563b2","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.621768089Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173592024-04-17T17:21:18.018Z{"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.621805452Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819}
173602024-04-17T17:21:18.018Z{"msg":"e2099aea-f183-4ceb-b6cc-96c7125563b2 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.621839964Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173612024-04-17T17:21:18.018Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.621863535Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"client":"0","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173622024-04-17T17:21:18.018Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.621893107Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"client":"0","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173632024-04-17T17:21:18.018Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.621922899Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"client":"1","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173642024-04-17T17:21:18.018Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.621967401Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"client":"1","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173652024-04-17T17:21:18.018Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.622032565Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"client":"2","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173662024-04-17T17:21:18.018Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.622121301Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"client":"2","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173672024-04-17T17:21:18.018Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.622178794Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"":"io task","client":"2","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173682024-04-17T17:21:18.018Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.622233008Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"":"io task","client":"0","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173692024-04-17T17:21:18.018Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.622281271Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"":"io task","client":"1","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173702024-04-17T17:21:18.018Z{"msg":"ds_connection connected from Ok(127.0.0.1:34288)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.622322543Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"":"io task","client":"2","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173712024-04-17T17:21:18.018Z{"msg":"ds_connection connected from Ok(127.0.0.1:32891)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.622353325Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"":"io task","client":"0","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173722024-04-17T17:21:18.018Z{"msg":"ds_connection connected from Ok(127.0.0.1:57746)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.622398868Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"":"io task","client":"1","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173732024-04-17T17:21:18.018Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.622943161Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"client":"2","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173742024-04-17T17:21:18.018Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.622976153Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"client":"2","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173752024-04-17T17:21:18.019Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.622991063Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"client":"1","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173762024-04-17T17:21:18.019Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.623015045Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"client":"1","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173772024-04-17T17:21:18.019Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.62310014Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"client":"0","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173782024-04-17T17:21:18.019Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.623152813Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"client":"0","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173792024-04-17T17:21:18.019Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.623569029Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"client":"1","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173802024-04-17T17:21:18.019Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.623610791Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"client":"0","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173812024-04-17T17:21:18.019Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.623654634Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"client":"2","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173822024-04-17T17:21:18.019Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.624158024Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"client":"2","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173832024-04-17T17:21:18.019Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.624195647Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"client":"2","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173842024-04-17T17:21:18.019Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.624220748Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173852024-04-17T17:21:18.019Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.624244259Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"client":"0","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173862024-04-17T17:21:18.019Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.62425885Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"client":"0","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173872024-04-17T17:21:18.019Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.624272111Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173882024-04-17T17:21:18.019Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.624291202Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"client":"1","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173892024-04-17T17:21:18.019Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.624304903Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"client":"1","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173902024-04-17T17:21:18.019Z{"msg":"[0]R flush_numbers[0..12]: [282, 281, 283, 283, 283, 276, 283, 284, 282, 284, 283, 283]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.624318704Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"":"downstairs","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173912024-04-17T17:21:18.019Z{"msg":"[0]R generation[0..12]: [157, 156, 158, 158, 158, 153, 158, 160, 157, 160, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.624340025Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"":"downstairs","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173922024-04-17T17:21:18.019Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.624354386Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"":"downstairs","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173932024-04-17T17:21:18.019Z{"msg":"[1]R flush_numbers[0..12]: [282, 281, 283, 283, 283, 276, 283, 284, 282, 284, 283, 283]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.624368567Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"":"downstairs","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173942024-04-17T17:21:18.019Z{"msg":"[1]R generation[0..12]: [157, 156, 158, 158, 158, 153, 158, 160, 157, 160, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.624388668Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"":"downstairs","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173952024-04-17T17:21:18.019Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.624402889Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":Wait for a query_work_queue command to finish before sending IO
173962024-04-17T17:21:18.019Z1819,"":"downstairs","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173972024-04-17T17:21:18.019Z{"msg":"[2]R flush_numbers[0..12]: [282, 281, 283, 283, 283, 276, 283, 284, 282, 284, 283, 283]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.624427121Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"":"downstairs","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173982024-04-17T17:21:18.019Z{"msg":"[2]R generation[0..12]: [157, 156, 158, 158, 158, 153, 158, 160, 157, 160, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.624441201Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"":"downstairs","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
173992024-04-17T17:21:18.020Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.624461903Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"":"downstairs","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
174002024-04-17T17:21:18.020Z{"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.624474864Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"":"downstairs","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
174012024-04-17T17:21:18.020Z{"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.624487624Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"":"downstairs","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
174022024-04-17T17:21:18.020Z{"msg":"Next flush: 286","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.624506765Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"":"downstairs","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
174032024-04-17T17:21:18.020Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.624519806Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"":"downstairs","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
174042024-04-17T17:21:18.020Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.624533257Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
174052024-04-17T17:21:18.020Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.624551858Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"client":"0","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
174062024-04-17T17:21:18.020Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.624565049Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"client":"1","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
174072024-04-17T17:21:18.020Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.62457794Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"client":"2","":"downstairs","session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
174082024-04-17T17:21:18.020Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.624597501Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
174092024-04-17T17:21:18.020Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.624609922Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
174102024-04-17T17:21:18.020Z{"msg":"e2099aea-f183-4ceb-b6cc-96c7125563b2 is now active with session: 362aacdc-2e32-46b1-a04f-d5bf90f9142f","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.624622402Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
174112024-04-17T17:21:18.020Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.624640743Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819,"session_id":"362aacdc-2e32-46b1-a04f-d5bf90f9142f"}
174122024-04-17T17:21:18.020Z{"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:16.624653024Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819}
174132024-04-17T17:21:18.020ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
174142024-04-17T17:21:18.024ZLoading write count information from file "/tmp/test_repair_verify.data"
174152024-04-17T17:21:18.024ZRun Repair workload
174162024-04-17T17:21:18.024Z01/30 Write block 27 len 40960 data: 7 8 13 12 15 20 20 24 19 16
174172024-04-17T17:21:18.024Z02/30 Write block 585 len 12288 data: 14 15 14
174182024-04-17T17:21:18.024Z03/30 Write block 337 len 24576 data: 18 18 17 16 13 14
174192024-04-17T17:21:18.024Z04/30 Write block 95 len 20480 data: 11 11 10 12 16
174202024-04-17T17:21:18.024Z05/30 Read block 276 len 28672
174212024-04-17T17:21:18.024Z06/30 Read block 290 len 24576
174222024-04-17T17:21:18.024Z07/30 Flush
174232024-04-17T17:21:18.028Z08/30 Write block 108 len 8192 data: 12 12
174242024-04-17T17:21:18.028Z09/30 Flush
174252024-04-17T17:21:18.032Z10/30 Write block 449 len 16384 data: 9 8 7 8
174262024-04-17T17:21:18.032Z11/30 Read block 348 len 8192
174272024-04-17T17:21:18.032Z12/30 Read block 260 len 12288
174282024-04-17T17:21:18.032Z13/30 Write block 430 len 12288 data: 10 11 13
174292024-04-17T17:21:18.032Z14/30 Read block 151 len 24576
174302024-04-17T17:21:18.032Z15/30 Read block 227 len 24576
174312024-04-17T17:21:18.035Z16/30 Write block 516 len 4096 data: 8
174322024-04-17T17:21:18.035Z17/30 Write block 593 len 8192 data: 10 9
174332024-04-17T17:21:18.035Z18/30 Write block 181 len 40960 data: 11 12 10 10 10 10 16 19 18 14
174342024-04-17T17:21:18.035Z19/30 Read block 346 len 16384
174352024-04-17T17:21:18.035Z20/30 Read block 469 len 12288
174362024-04-17T17:21:18.039Z21/30 Write block 61 len 4096 data: 15
174372024-04-17T17:21:18.039Z22/30 Read block 109 len 36864
174382024-04-17T17:21:18.039Z23/30 Write block 406 len 16384 data: 11 9 7 9
174392024-04-17T17:21:18.039Z24/30 Read block 359 len 28672
174402024-04-17T17:21:18.039Z25/30 Read block 494 len 12288
174412024-04-17T17:21:18.043Z26/30 Write block 287 len 36864 data: 11 11 15 18 18 15 18 15 11
174422024-04-17T17:21:18.043Z27/30 Read block 368 len 28672
174432024-04-17T17:21:18.043Z28/30 Read block 31 len 28672
174442024-04-17T17:21:18.043Z29/30 Write block 459 len 8192 data: 12 13
174452024-04-17T17:21:18.047Z30/30 Read block 186 len 32768
174462024-04-17T17:21:18.047Z----------------------------------------------------------------
174472024-04-17T17:21:18.047Z Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:21
174482024-04-17T17:21:18.047ZStates: Active Active Active
174492024-04-17T17:21:18.047ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
174502024-04-17T17:21:18.047Z 10 Acked 1009 Write 4 Sent Done Done false
174512024-04-17T17:21:18.047Z 11 Acked 1010 Read 2 Sent Done Done false
174522024-04-17T17:21:18.047Z 12 Acked 1011 Read 3 Sent Done Done false
174532024-04-17T17:21:18.047Z 13 Acked 1012 Write 3 Sent Done Done false
174542024-04-17T17:21:18.047Z 14 Acked 1013 Read 6 Sent Done Done false
174552024-04-17T17:21:18.047Z 15 Acked 1014 Read 6 Sent Done Done false
174562024-04-17T17:21:18.047Z 16 Acked 1015 Write 1 Sent Done Done false
174572024-04-17T17:21:18.047Z 17 Acked 1016 Write 2 Sent Done Done false
174582024-04-17T17:21:18.047Z 18 Acked 1017 Write 10 Sent Done Done false
174592024-04-17T17:21:18.048Z 19 Acked 1018 Read 4 Sent Done Done false
174602024-04-17T17:21:18.048Z 20 Acked 1019 Read 3 Sent Done Done false
174612024-04-17T17:21:18.048Z 21 Acked 1020 Write 1 Sent Done Done false
174622024-04-17T17:21:18.048Z 22 Acked 1021 Read 9 Sent Done Done false
174632024-04-17T17:21:18.048Z 23 Acked 1022 Write 4 Sent Done Done false
174642024-04-17T17:21:18.048Z 24 Acked 1023 Read 7 Sent Done Done false
174652024-04-17T17:21:18.048Z 25 Acked 1024 Read 3 Sent Done Done false
174662024-04-17T17:21:18.048Z 26 Acked 1025 Write 9 Sent Done Done false
174672024-04-17T17:21:18.048Z 27 Acked 1026 Read 7 Sent Done Done false
174682024-04-17T17:21:18.048Z 28 Acked 1027 Read 7 Sent Done Done false
174692024-04-17T17:21:18.048Z 29 Acked 1028 Write 2 Sent Done Done false
174702024-04-17T17:21:18.048Z 30 Acked 1029 Read 8 Sent Done Sent false
174712024-04-17T17:21:18.048Z STATES DS:0 DS:1 DS:2 TOTAL
174722024-04-17T17:21:18.048Z New 0 0 0 0
174732024-04-17T17:21:18.048Z Sent 21 0 1 22
174742024-04-17T17:21:18.048Z Done 0 21 20 41
174752024-04-17T17:21:18.048Z Skipped 0 0 0 0
174762024-04-17T17:21:18.048Z Error 0 0 0 0
174772024-04-17T17:21:18.048ZLast Flush: 1008 1008 1008
174782024-04-17T17:21:18.048ZDownstairs last five completed: 1008 1007 1006 1005 1004
174792024-04-17T17:21:18.048ZUpstairs last five completed: 30 29 28 27 26
174802024-04-17T17:21:18.048Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:21:16.654191085Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1819}
174812024-04-17T17:21:18.048ZWrote out file "/tmp/test_repair_verify.data"
174822024-04-17T17:21:18.052Z
174832024-04-17T17:21:18.052ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
174842024-04-17T17:21:18.072ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
174852024-04-17T17:21:18.073Z 0 000-019 157 157 157 282 282 282 F F F
174862024-04-17T17:21:18.073Z 1 020-039 162 162 162 286 286 286 F F F
174872024-04-17T17:21:18.073Z 2 040-059 158 158 158 283 283 283 F F F
174882024-04-17T17:21:18.073Z 3 060-079 158 158 158 283 283 283 F T T <---
174892024-04-17T17:21:18.073Z 4 080-099 162 162 162 286 286 286 F F F
174902024-04-17T17:21:18.073Z 5 100-119 162 162 162 287 287 287 F F F
174912024-04-17T17:21:18.077Z 6 120-139 158 158 158 283 283 283 F F F
174922024-04-17T17:21:18.077Z 7 140-159 160 160 160 284 284 284 F F F
174932024-04-17T17:21:18.077Z 8 160-179 157 157 157 282 282 282 F F F
174942024-04-17T17:21:18.077Z 9 180-199 160 160 160 284 284 284 F T T <---
174952024-04-17T17:21:18.078Z 10 200-219 158 158 158 283 283 283 F F F
174962024-04-17T17:21:18.078Z 11 220-239 158 158 158 283 283 283 F F F
174972024-04-17T17:21:18.078Z 12 240-259 158 158 158 283 283 283 F F F
174982024-04-17T17:21:18.078Z 13 260-279 161 161 161 285 285 285 F F F
174992024-04-17T17:21:18.078Z 14 280-299 161 161 161 285 285 285 F T T <---
175002024-04-17T17:21:18.078Z 15 300-319 161 161 161 285 285 285 F F F
175012024-04-17T17:21:18.078Z 16 320-339 162 162 162 286 286 286 F F F
175022024-04-17T17:21:18.078Z 17 340-359 162 162 162 286 286 286 F F F
175032024-04-17T17:21:18.078Z 18 360-379 161 161 161 285 285 285 F F F
175042024-04-17T17:21:18.078Z 19 380-399 160 160 160 284 284 284 F F F
175052024-04-17T17:21:18.078Z 20 400-419 161 161 161 285 285 285 F T T <---
175062024-04-17T17:21:18.078Z 21 420-439 161 161 161 285 285 285 F T T <---
175072024-04-17T17:21:18.078Z 22 440-459 161 161 161 285 285 285 F T T <---
175082024-04-17T17:21:18.078Z 23 460-479 161 161 161 285 285 285 F T T <---
175092024-04-17T17:21:18.078Z 24 480-499 158 158 158 283 283 283 F F F
175102024-04-17T17:21:18.078Z 25 500-519 158 158 158 283 283 283 F T T <---
175112024-04-17T17:21:18.078Z 26 520-539 161 161 161 285 285 285 F F F
175122024-04-17T17:21:18.078Z 27 540-559 160 160 160 284 284 284 F F F
175132024-04-17T17:21:18.078Z 28 560-579 157 157 157 282 282 282 F F F
175142024-04-17T17:21:18.078Z 29 580-599 162 162 162 286 286 286 F T T <---
175152024-04-17T17:21:18.078ZMax gen: 162, Max flush: 287
175162024-04-17T17:21:18.078ZError: Difference in extent metadata found!
175172024-04-17T17:21:18.078ZA Difference in extent metadata is expected here
175182024-04-17T17:21:18.079ZOn loop 81
175192024-04-17T17:21:18.079Z
175202024-04-17T17:21:18.079ZVerifying data now
175212024-04-17T17:21:22.527ZNo change after verify
175222024-04-17T17:21:22.528ZLoop: 81 Downstairs dump after verify (and repair):
175232024-04-17T17:21:22.543ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
175242024-04-17T17:21:22.543Z 0 000-019 157 157 157 282 282 282 F F F
175252024-04-17T17:21:22.547Z 1 020-039 162 162 162 286 286 286 F F F
175262024-04-17T17:21:22.547Z 2 040-059 158 158 158 283 283 283 F F F
175272024-04-17T17:21:22.547Z 3 060-079 163 163 163 288 288 288 F F F
175282024-04-17T17:21:22.547Z 4 080-099 162 162 162 286 286 286 F F F
175292024-04-17T17:21:22.547Z 5 100-119 162 162 162 287 287 287 F F F
175302024-04-17T17:21:22.547Z 6 120-139 158 158 158 283 283 283 F F F
175312024-04-17T17:21:22.547Z 7 140-159 160 160 160 284 284 284 F F F
175322024-04-17T17:21:22.547Z 8 160-179 157 157 157 282 282 282 F F F
175332024-04-17T17:21:22.547Z 9 180-199 163 163 163 288 288 288 F F F
175342024-04-17T17:21:22.547Z 10 200-219 158 158 158 283 283 283 F F F
175352024-04-17T17:21:22.547Z 11 220-239 158 158 158 283 283 283 F F F
175362024-04-17T17:21:22.547Z 12 240-259 158 158 158 283 283 283 F F F
175372024-04-17T17:21:22.547Z 13 260-279 161 161 161 285 285 285 F F F
175382024-04-17T17:21:22.547Z 14 280-299 163 163 163 288 288 288 F F F
175392024-04-17T17:21:22.547Z 15 300-319 161 161 161 285 285 285 F F F
175402024-04-17T17:21:22.547Z 16 320-339 162 162 162 286 286 286 F F F
175412024-04-17T17:21:22.547Z 17 340-359 162 162 162 286 286 286 F F F
175422024-04-17T17:21:22.547Z 18 360-379 161 161 161 285 285 285 F F F
175432024-04-17T17:21:22.547Z 19 380-399 160 160 160 284 284 284 F F F
175442024-04-17T17:21:22.547Z 20 400-419 163 163 163 288 288 288 F F F
175452024-04-17T17:21:22.547Z 21 420-439 163 163 163 288 288 288 F F F
175462024-04-17T17:21:22.547Z 22 440-459 163 163 163 288 288 288 F F F
175472024-04-17T17:21:22.548Z 23 460-479 163 163 163 288 288 288 F F F
175482024-04-17T17:21:22.548Z 24 480-499 158 158 158 283 283 283 F F F
175492024-04-17T17:21:22.548Z 25 500-519 163 163 163 288 288 288 F F F
175502024-04-17T17:21:22.548Z 26 520-539 161 161 161 285 285 285 F F F
175512024-04-17T17:21:22.548Z 27 540-559 160 160 160 284 284 284 F F F
175522024-04-17T17:21:22.548Z 28 560-579 157 157 157 282 282 282 F F F
175532024-04-17T17:21:22.548Z 29 580-599 163 163 163 288 288 288 F F F
175542024-04-17T17:21:22.548ZMax gen: 163, Max flush: 288
175552024-04-17T17:21:22.548Z
175562024-04-17T17:21:22.548ZBegin loop 82
175572024-04-17T17:21:22.548ZDownstairs to restart: 1
175582024-04-17T17:21:22.576Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.176476479Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827}
175592024-04-17T17:21:22.600Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.176761026Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827}
175602024-04-17T17:21:22.600Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.176801249Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827}
175612024-04-17T17:21:22.600Z{"msg":"Crucible 7f973d72-7648-4fa2-af5d-2fdcf0797401 has session id: 48546766-83e8-446a-be1a-ecffa7f4284e","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.17683019Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175622024-04-17T17:21:22.600Z{"msg":"Crucible runtime is spawned
175632024-04-17T17:21:22.600ZUpstairs opts: Upstairs UUID: 7f973d72-7648-4fa2-af5d-2fdcf0797401, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.176913275Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175642024-04-17T17:21:22.600Z{"msg":"Crucible stats registered with UUID: 7f973d72-7648-4fa2-af5d-2fdcf0797401","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.176967849Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175652024-04-17T17:21:22.600Z{"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.17699435Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827}
175662024-04-17T17:21:22.600Z{"msg":"7f973d72-7648-4fa2-af5d-2fdcf0797401 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.177093006Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175672024-04-17T17:21:22.600Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.177144729Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"client":"0","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175682024-04-17T17:21:22.600Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.177185412Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"client":"0","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175692024-04-17T17:21:22.601Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.177212944Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"client":"1","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175702024-04-17T17:21:22.601Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.177252426Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"client":"1","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175712024-04-17T17:21:22.601Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.177291188Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"client":"2","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175722024-04-17T17:21:22.601Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.1773176Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"client":"2","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175732024-04-17T17:21:22.601Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.177348982Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"":"io task","client":"0","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175742024-04-17T17:21:22.601Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.177379084Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"":"io task","client":"2","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175752024-04-17T17:21:22.601Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.177423996Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"":"io task","client":"1","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175762024-04-17T17:21:22.601Z{"msg":"ds_connection connected from Ok(127.0.0.1:42050)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.177460869Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"":"io task","client":"2","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175772024-04-17T17:21:22.601Z{"msg":"ds_connection connected from Ok(127.0.0.1:55237)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.17748611Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"":"io task","client":"0","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175782024-04-17T17:21:22.601Z{"msg":"ds_connection connected from Ok(127.0.0.1:52052)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.177778738Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"":"io task","client":"1","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175792024-04-17T17:21:22.601Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.178320821Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"client":"2","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175802024-04-17T17:21:22.601Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.178356413Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"client":"2","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175812024-04-17T17:21:22.601Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.178372514Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"client":"1","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175822024-04-17T17:21:22.601Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.178387225Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"client":"1","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175832024-04-17T17:21:22.601Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.178455999Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"client":"0","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175842024-04-17T17:21:22.601Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.178525223Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"client":"0","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175852024-04-17T17:21:22.601Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.178908186Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"client":"0","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175862024-04-17T17:21:22.601Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.178961099Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"client":"2","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175872024-04-17T17:21:22.601Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.179074846Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"client":"1","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175882024-04-17T17:21:22.601Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.179549395Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"client":"2","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175892024-04-17T17:21:22.601Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.179602098Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"client":"2","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175902024-04-17T17:21:22.601Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.17962972Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175912024-04-17T17:21:22.601Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.179655962Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"client":"0","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175922024-04-17T17:21:22.602Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.179670793Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"client":"0","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175932024-04-17T17:21:22.602Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.179691074Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175942024-04-17T17:21:22.602Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.179703324Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"client":"1","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175952024-04-17T17:21:22.602Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.179716305Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"client":"1","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175962024-04-17T17:21:22.602Z{"msg":"[0]R flush_numbers[0..12]: [282, 286, 283, 288, 286, 287, 283, 284, 282, 288, 283, 283]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.179736757Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"":"downstairs","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175972024-04-17T17:21:22.602Z{"msg":"[0]R generation[0..12]: [157, 162, 158, 163, 162, 162, 158, 160, 157, 163, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.179751007Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"":"downstairs","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175982024-04-17T17:21:22.602Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.179772539Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"":"downstairs","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
175992024-04-17T17:21:22.602Z{"msg":"[1]R flush_numbers[0..12]: [282, 286, 283, 288, 286, 287, 283, 284, 282, 288, 283, 283]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
176002024-04-17T17:21:22.602Z,"time":"2024-04-17T17:21:21.179786549Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"":"downstairs","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
176012024-04-17T17:21:22.602Z{"msg":"[1]R generation[0..12]: [157, 162, 158, 163, 162, 162, 158, 160, 157, 163, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.179810261Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"":"downstairs","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
176022024-04-17T17:21:22.602Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.179824722Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"":"downstairs","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
176032024-04-17T17:21:22.602Z{"msg":"[2]R flush_numbers[0..12]: [282, 286, 283, 288, 286, 287, 283, 284, 282, 288, 283, 283]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.179844863Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"":"downstairs","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
176042024-04-17T17:21:22.602Z{"msg":"[2]R generation[0..12]: [157, 162, 158, 163, 162, 162, 158, 160, 157, 163, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.179861954Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"":"downstairs","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
176052024-04-17T17:21:22.602Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.179897496Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"":"downstairs","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
176062024-04-17T17:21:22.602Z{"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.179928138Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"":"downstairs","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
176072024-04-17T17:21:22.602Z{"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.17995982Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"":"downstairs","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
176082024-04-17T17:21:22.602Z{"msg":"Next flush: 289","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.179990492Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"":"downstairs","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
176092024-04-17T17:21:22.602Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.180024514Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"":"downstairs","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
176102024-04-17T17:21:22.602Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.180052786Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
176112024-04-17T17:21:22.602Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.180066346Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"client":"0","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
176122024-04-17T17:21:22.602Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
176132024-04-17T17:21:22.602Z,"time":"2024-04-17T17:21:21.180098088Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"client":"1","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
176142024-04-17T17:21:22.602Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.180131841Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"client":"2","":"downstairs","session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
176152024-04-17T17:21:22.603Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.180160412Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
176162024-04-17T17:21:22.603Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.180174453Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
176172024-04-17T17:21:22.603Z{"msg":"7f973d72-7648-4fa2-af5d-2fdcf0797401 is now active with session: 48546766-83e8-446a-be1a-ecffa7f4284e","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.180194004Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
176182024-04-17T17:21:22.603Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.180206625Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827,"session_id":"48546766-83e8-446a-be1a-ecffa7f4284e"}
176192024-04-17T17:21:22.603Z{"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:21.180225346Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827}
176202024-04-17T17:21:22.603ZLoading write count information from file "/tmp/test_repair_verify.data"
176212024-04-17T17:21:22.603ZRun Repair workload
176222024-04-17T17:21:22.603Z01/30 Write block 489 len 20480 data: 13 13 10 11 9
176232024-04-17T17:21:22.603Z02/30 Write block 454 len 32768 data: 7 9 9 9 11 13 14 12
176242024-04-17T17:21:22.603Z03/30 Write block 39 len 28672 data: 13 14 15 13 11 8 10
176252024-04-17T17:21:22.603Z04/30 Read block 335 len 4096
176262024-04-17T17:21:22.603Z05/30 Read block 450 len 4096
176272024-04-17T17:21:22.603Z06/30 Write block 294 len 28672 data: 16 12 9 9 10 11 10
176282024-04-17T17:21:22.603Z07/30 Read block 161 len 16384
176292024-04-17T17:21:22.603Z08/30 Write block 455 len 28672 data: 10 10 10 12 14 15 13
176302024-04-17T17:21:22.603Z09/30 Read block 292 len 8192
176312024-04-17T17:21:22.603Z10/30 Write block 135 len 12288 data: 10 13 12
176322024-04-17T17:21:22.603Z11/30 Write block 87 len 4096 data: 18
176332024-04-17T17:21:22.603Z12/30 Write block 137 len 12288 data: 13 11 11
176342024-04-17T17:21:22.603Z13/30 Read block 195 len 4096
176352024-04-17T17:21:22.603Z14/30 Read block 43 len 8192
176362024-04-17T17:21:22.603Z15/30 Read block 34 len 8192
176372024-04-17T17:21:22.603Z16/30 Read block 406 len 24576
176382024-04-17T17:21:22.604Z17/30 Write block 347 len 20480 data: 13 12 12 13 8
176392024-04-17T17:21:22.604Z18/30 Write block 344 len 16384 data: 17 18 17 14
176402024-04-17T17:21:22.604Z19/30 Read block 332 len 4096
176412024-04-17T17:21:22.604Z20/30 Write block 343 len 40960 data: 16 18 19 18 15 13 13 14 9 9
176422024-04-17T17:21:22.604Z21/30 Read block 112 len 40960
176432024-04-17T17:21:22.609Z22/30 Read block 289 len 20480
176442024-04-17T17:21:22.609Z23/30 Write block 397 len 8192 data: 8 11
176452024-04-17T17:21:22.609Z24/30 Read block 70 len 4096
176462024-04-17T17:21:22.609Z25/30 Write block 538 len 40960 data: 5 12 13 14 18 16 16 12 13 13
176472024-04-17T17:21:22.609Z26/30 Write block 151 len 12288 data: 9 8 8
176482024-04-17T17:21:22.609Z27/30 Write block 434 len 28672 data: 15 15 15 16 15 17 14
176492024-04-17T17:21:22.609Z28/30 Write block 144 len 32768 data: 11 13 13 11 11 12 11 10
176502024-04-17T17:21:22.609Z29/30 Read block 167 len 4096
176512024-04-17T17:21:22.610Z30/30 Read block 572 len 24576
176522024-04-17T17:21:22.610Z----------------------------------------------------------------
176532024-04-17T17:21:22.610Z Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:30
176542024-04-17T17:21:22.610ZStates: Active Active Active
176552024-04-17T17:21:22.610ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
176562024-04-17T17:21:22.610Z 1 Acked 1000 Write 5 Done Sent Done false
176572024-04-17T17:21:22.610Z 2 Acked 1001 Write 8 Done Sent Done false
176582024-04-17T17:21:22.610Z 3 Acked 1002 Write 7 Done Sent Done false
176592024-04-17T17:21:22.610Z 4 Acked 1003 Read 1 Done Sent Done false
176602024-04-17T17:21:22.610Z 5 Acked 1004 Read 1 Done Sent Done false
176612024-04-17T17:21:22.610Z 6 Acked 1005 Write 7 Done Sent Done false
176622024-04-17T17:21:22.610Z 7 Acked 1006 Read 4 Done Sent Done false
176632024-04-17T17:21:22.610Z 8 Acked 1007 Write 7 Done Sent Done false
176642024-04-17T17:21:22.610Z 9 Acked 1008 Read 2 Done Sent Done false
176652024-04-17T17:21:22.610Z 10 Acked 1009 Write 3 Done Sent Done false
176662024-04-17T17:21:22.610Z 11 Acked 1010 Write 1 Done Sent Done false
176672024-04-17T17:21:22.610Z 12 Acked 1011 Write 3 Done Sent Done false
176682024-04-17T17:21:22.610Z 13 Acked 1012 Read 1 Done Sent Done false
176692024-04-17T17:21:22.610Z 14 Acked 1013 Read 2 Done Sent Done false
176702024-04-17T17:21:22.610Z 15 Acked 1014 Read 2 Done Sent Done false
176712024-04-17T17:21:22.610Z 16 Acked 1015 Read 6 Done Sent Done false
176722024-04-17T17:21:22.610Z 17 Acked 1016 Write 5 Done Sent Done false
176732024-04-17T17:21:22.610Z 18 Acked 1017 Write 4 Done Sent Done false
176742024-04-17T17:21:22.610Z 19 Acked 1018 Read 1 Done Sent Done false
176752024-04-17T17:21:22.611Z 20 Acked 1019 Write 10 Done Sent Done false
176762024-04-17T17:21:22.611Z 21 Acked 1020 Read 10 Done Sent Done false
176772024-04-17T17:21:22.611Z 22 Acked 1021 Read 5 Done Sent Done false
176782024-04-17T17:21:22.611Z 23 Acked 1022 Write 2 Done Sent Done false
176792024-04-17T17:21:22.611Z 24 Acked 1023 Read 1 Done Sent Done false
176802024-04-17T17:21:22.611Z 25 Acked 1024 Write 10 Done Sent Done false
176812024-04-17T17:21:22.611Z 26 Acked 1025 Write 3 Done Sent Done false
176822024-04-17T17:21:22.611Z 27 Acked 1026 Write 7 Done Sent Done false
176832024-04-17T17:21:22.611Z 28 Acked 1027 Write 8 Done Sent Done false
176842024-04-17T17:21:22.611Z 29 Acked 1028 Read 1 Done Sent Done false
176852024-04-17T17:21:22.611Z 30 Acked 1029 Read 6 Done Sent Done false
176862024-04-17T17:21:22.611Z STATES DS:0 DS:1 DS:2 TOTAL
176872024-04-17T17:21:22.611Z New 0 0 0 0
176882024-04-17T17:21:22.611Z Sent 0 30 0 30
176892024-04-17T17:21:22.611Z Done 30 0 30 60
176902024-04-17T17:21:22.611Z Skipped 0 0 0 0
176912024-04-17T17:21:22.611Z Error 0 0 0 0
176922024-04-17T17:21:22.611ZLast Flush: 0 0 0
176932024-04-17T17:21:22.611ZDownstairs last five completed:
176942024-04-17T17:21:22.611ZUpstairs last five completed: 30 29 28 27 26
176952024-04-17T17:21:22.611Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:21:21.208617387Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1827}
176962024-04-17T17:21:22.611ZWrote out file "/tmp/test_repair_verify.data"
176972024-04-17T17:21:22.611Z
176982024-04-17T17:21:22.612ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
176992024-04-17T17:21:22.626ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
177002024-04-17T17:21:22.626Z 0 000-019 157 157 157 282 282 282 F F F
177012024-04-17T17:21:22.626Z 1 020-039 162 162 162 286 286 286 T F T <---
177022024-04-17T17:21:22.626Z 2 040-059 158 158 158 283 283 283 T F T <---
177032024-04-17T17:21:22.626Z 3 060-079 163 163 163 288 288 288 F F F
177042024-04-17T17:21:22.626Z 4 080-099 162 162 162 286 286 286 T F T <---
177052024-04-17T17:21:22.626Z 5 100-119 162 162 162 287 287 287 F F F
177062024-04-17T17:21:22.626Z 6 120-139 158 158 158 283 283 283 T F T <---
177072024-04-17T17:21:22.626Z 7 140-159 160 160 160 284 284 284 T F T <---
177082024-04-17T17:21:22.626Z 8 160-179 157 157 157 282 282 282 F F F
177092024-04-17T17:21:22.626Z 9 180-199 163 163 163 288 288 288 F F F
177102024-04-17T17:21:22.626Z 10 200-219 158 158 158 283 283 283 F F F
177112024-04-17T17:21:22.626Z 11 220-239 158 158 158 283 283 283 F F F
177122024-04-17T17:21:22.630Z 12 240-259 158 158 158 283 283 283 F F F
177132024-04-17T17:21:22.630Z 13 260-279 161 161 161 285 285 285 F F F
177142024-04-17T17:21:22.630Z 14 280-299 163 163 163 288 288 288 T F T <---
177152024-04-17T17:21:22.630Z 15 300-319 161 161 161 285 285 285 T F T <---
177162024-04-17T17:21:22.630Z 16 320-339 162 162 162 286 286 286 F F F
177172024-04-17T17:21:22.630Z 17 340-359 162 162 162 286 286 286 T F T <---
177182024-04-17T17:21:22.630Z 18 360-379 161 161 161 285 285 285 F F F
177192024-04-17T17:21:22.631Z 19 380-399 160 160 160 284 284 284 T F T <---
177202024-04-17T17:21:22.631Z 20 400-419 163 163 163 288 288 288 F F F
177212024-04-17T17:21:22.631Z 21 420-439 163 163 163 288 288 288 T F T <---
177222024-04-17T17:21:22.631Z 22 440-459 163 163 163 288 288 288 T F T <---
177232024-04-17T17:21:22.631Z 23 460-479 163 163 163 288 288 288 T F T <---
177242024-04-17T17:21:22.631Z 24 480-499 158 158 158 283 283 283 T F T <---
177252024-04-17T17:21:22.631Z 25 500-519 163 163 163 288 288 288 F F F
177262024-04-17T17:21:22.631Z 26 520-539 161 161 161 285 285 285 T F T <---
177272024-04-17T17:21:22.631Z 27 540-559 160 160 160 284 284 284 T F T <---
177282024-04-17T17:21:22.631Z 28 560-579 157 157 157 282 282 282 F F F
177292024-04-17T17:21:22.631Z 29 580-599 163 163 163 288 288 288 F F F
177302024-04-17T17:21:22.631ZMax gen: 163, Max flush: 288
177312024-04-17T17:21:22.631ZError: Difference in extent metadata found!
177322024-04-17T17:21:22.631ZA Difference in extent metadata is expected here
177332024-04-17T17:21:22.631ZOn loop 82
177342024-04-17T17:21:22.631Z
177352024-04-17T17:21:22.635ZVerifying data now
177362024-04-17T17:21:27.338ZNo change after verify
177372024-04-17T17:21:27.342ZLoop: 82 Downstairs dump after verify (and repair):
177382024-04-17T17:21:27.352ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
177392024-04-17T17:21:27.352Z 0 000-019 157 157 157 282 282 282 F F F
177402024-04-17T17:21:27.356Z 1 020-039 164 164 164 289 289 289 F F F
177412024-04-17T17:21:27.356Z 2 040-059 164 164 164 289 289 289 F F F
177422024-04-17T17:21:27.356Z 3 060-079 163 163 163 288 288 288 F F F
177432024-04-17T17:21:27.356Z 4 080-099 164 164 164 289 289 289 F F F
177442024-04-17T17:21:27.356Z 5 100-119 162 162 162 287 287 287 F F F
177452024-04-17T17:21:27.356Z 6 120-139 164 164 164 289 289 289 F F F
177462024-04-17T17:21:27.356Z 7 140-159 164 164 164 289 289 289 F F F
177472024-04-17T17:21:27.356Z 8 160-179 157 157 157 282 282 282 F F F
177482024-04-17T17:21:27.357Z 9 180-199 163 163 163 288 288 288 F F F
177492024-04-17T17:21:27.357Z 10 200-219 158 158 158 283 283 283 F F F
177502024-04-17T17:21:27.357Z 11 220-239 158 158 158 283 283 283 F F F
177512024-04-17T17:21:27.357Z 12 240-259 158 158 158 283 283 283 F F F
177522024-04-17T17:21:27.357Z 13 260-279 161 161 161 285 285 285 F F F
177532024-04-17T17:21:27.357Z 14 280-299 164 164 164 289 289 289 F F F
177542024-04-17T17:21:27.357Z 15 300-319 164 164 164 289 289 289 F F F
177552024-04-17T17:21:27.357Z 16 320-339 162 162 162 286 286 286 F F F
177562024-04-17T17:21:27.357Z 17 340-359 164 164 164 289 289 289 F F F
177572024-04-17T17:21:27.357Z 18 360-379 161 161 161 285 285 285 F F F
177582024-04-17T17:21:27.357Z 19 380-399 164 164 164 289 289 289 F F F
177592024-04-17T17:21:27.357Z 20 400-419 163 163 163 288 288 288 F F F
177602024-04-17T17:21:27.357Z 21 420-439 164 164 164 289 289 289 F F F
177612024-04-17T17:21:27.357Z 22 440-459 164 164 164 289 289 289 F F F
177622024-04-17T17:21:27.357Z 23 460-479 164 164 164 289 289 289 F F F
177632024-04-17T17:21:27.357Z 24 480-499 164 164 164 289 289 289 F F F
177642024-04-17T17:21:27.357Z 25 500-519 163 163 163 288 288 288 F F F
177652024-04-17T17:21:27.357Z 26 520-539 164 164 164 289 289 289 F F F
177662024-04-17T17:21:27.357Z 27 540-559 164 164 164 289 289 289 F F F
177672024-04-17T17:21:27.357Z 28 560-579 157 157 157 282 282 282 F F F
177682024-04-17T17:21:27.357Z 29 580-599 163 163 163 288 288 288 F F F
177692024-04-17T17:21:27.357ZMax gen: 164, Max flush: 289
177702024-04-17T17:21:27.361Z
177712024-04-17T17:21:27.361ZBegin loop 83
177722024-04-17T17:21:27.361ZDownstairs to restart: 2
177732024-04-17T17:21:27.377Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.986250336Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835}
177742024-04-17T17:21:27.377Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.986496191Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835}
177752024-04-17T17:21:27.377Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.986539653Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835}
177762024-04-17T17:21:27.386Z{"msg":"Crucible 7b995a1a-6060-448f-80b5-6636fcc1e900 has session id: 63e95c6b-3bea-4055-9d04-4af0fc345653","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.986563435Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
177772024-04-17T17:21:27.386Z{"msg":"Crucible runtime is spawned
177782024-04-17T17:21:27.386ZUpstairs opts: Upstairs UUID: 7b995a1a-6060-448f-80b5-6636fcc1e900, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.986643669Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
177792024-04-17T17:21:27.386Z{"msg":"Crucible stats registered with UUID: 7b995a1a-6060-448f-80b5-6636fcc1e900","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.986690032Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
177802024-04-17T17:21:27.386Z{"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.986727345Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835}
177812024-04-17T17:21:27.386Z{"msg":"7b995a1a-6060-448f-80b5-6636fcc1e900 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.986861873Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
177822024-04-17T17:21:27.386Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.986892744Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"client":"0","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
177832024-04-17T17:21:27.386Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.986921576Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"client":"0","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
177842024-04-17T17:21:27.386Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.986937567Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"client":"1","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
177852024-04-17T17:21:27.387Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.986973619Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"client":"1","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
177862024-04-17T17:21:27.387Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.98698814Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"client":"2","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
177872024-04-17T17:21:27.387Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.987001571Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"client":"2","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
177882024-04-17T17:21:27.387Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.987014682Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"":"io task","client":"2","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
177892024-04-17T17:21:27.387Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.987033603Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"":"io task","client":"1","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
177902024-04-17T17:21:27.387Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.987055914Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"":"io task","client":"0","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
177912024-04-17T17:21:27.387Z{"msg":"ds_connection connected from Ok(127.0.0.1:37905)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.987261817Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"":"io task","client":"2","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
177922024-04-17T17:21:27.387Z{"msg":"ds_connection connected from Ok(127.0.0.1:57152)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.987299869Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"":"io task","client":"0","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
177932024-04-17T17:21:27.387Z{"msg":"ds_connection connected from Ok(127.0.0.1:57868)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.987325301Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"":"io task","client":"1","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
177942024-04-17T17:21:27.387Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.988022143Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"client":"1","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
177952024-04-17T17:21:27.387Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.988066866Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"client":"1","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
177962024-04-17T17:21:27.387Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.988108658Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"client":"2","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
177972024-04-17T17:21:27.387Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.988153071Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"client":"2","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
177982024-04-17T17:21:27.387Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.988183083Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"client":"0","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
177992024-04-17T17:21:27.387Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.988213915Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"client":"0","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178002024-04-17T17:21:27.387Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.988620029Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"client":"1","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178012024-04-17T17:21:27.387Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.988760398Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"client":"0","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178022024-04-17T17:21:27.387Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.98879023Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"client":"2","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178032024-04-17T17:21:27.387Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.989142851Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"client":"1","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178042024-04-17T17:21:27.387Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.989195684Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"client":"1","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178052024-04-17T17:21:27.387Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.989227386Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178062024-04-17T17:21:27.387Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.989258228Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"client":"2","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178072024-04-17T17:21:27.387Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.989281249Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"client":"2","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178082024-04-17T17:21:27.388Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.989310211Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178092024-04-17T17:21:27.388Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.989324472Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"client":"0","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178102024-04-17T17:21:27.388Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.989348673Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"client":"0","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178112024-04-17T17:21:27.388Z{"msg":"[0]R flush_numbers[0..12]: [282, 289, 289, 288, 289, 287, 289, 289, 282, 288, 283, 283]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.989365034Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"":"downstairs","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178122024-04-17T17:21:27.388Z{"msg":"[0]R generation[0..12]: [157, 164, 164, 163, 164, 162, 164, 164, 157, 163, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.989386556Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"":"downstairs","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178132024-04-17T17:21:27.388Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.989401656Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"":"downstairs","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178142024-04-17T17:21:27.388Z{"msg":"[1]R flush_numbers[0..12]: [282, 289, 289, 288, 289, 287, 289, 289, 282, 288, 283, 283]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.989422588Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"":"downstairs","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178152024-04-17T17:21:27.388Z{"msg":"[1]R generation[0..12]: [157, 164, 164, 163, 164, 162, 164, 164, 157, 163, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.989436619Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"":"downstairs","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178162024-04-17T17:21:27.388Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
178172024-04-17T17:21:27.388Z,"time":"2024-04-17T17:21:25.98945821Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"":"downstairs","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178182024-04-17T17:21:27.388Z{"msg":"[2]R flush_numbers[0..12]: [282, 289, 289, 288, 289, 287, 289, 289, 282, 288, 283, 283]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.989475791Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"":"downstairs","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178192024-04-17T17:21:27.388Z{"msg":"[2]R generation[0..12]: [157, 164, 164, 163, 164, 162, 164, 164, 157, 163, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.989496512Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"":"downstairs","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178202024-04-17T17:21:27.388Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.989510873Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"":"downstairs","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178212024-04-17T17:21:27.388Z{"msg":"Max found gen is 165","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.989530264Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"":"downstairs","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178222024-04-17T17:21:27.388Z{"msg":"Generation requested: 166 >= found:165","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.989543785Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"":"downstairs","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178232024-04-17T17:21:27.388Z{"msg":"Next flush: 290","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.989562856Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"":"downstairs","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178242024-04-17T17:21:27.388Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.989575747Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"":"downstairs","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178252024-04-17T17:21:27.388Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.989595108Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178262024-04-17T17:21:27.388Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.989607149Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"client":"0","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178272024-04-17T17:21:27.388Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.98962667Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"client":"1","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178282024-04-17T17:21:27.388Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.989640001Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"client":"2","":"downstairs","session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178292024-04-17T17:21:27.388Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.989659292Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178302024-04-17T17:21:27.388Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.989671473Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178312024-04-17T17:21:27.389Z{"msg":"7b995a1a-6060-448f-80b5-6636fcc1e900 is now active with session: 63e95c6b-3bea-4055-9d04-4af0fc345653","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.989690554Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178322024-04-17T17:21:27.389Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.989702475Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835,"session_id":"63e95c6b-3bea-4055-9d04-4af0fc345653"}
178332024-04-17T17:21:27.389Z{"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:25.989720846Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835}
178342024-04-17T17:21:27.389ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
178352024-04-17T17:21:27.389ZLoading write count information from file "/tmp/test_repair_verify.data"
178362024-04-17T17:21:27.389ZRun Repair workload
178372024-04-17T17:21:27.393Z01/30 Write block 92 len 12288 data: 19 17 14
178382024-04-17T17:21:27.393Z02/30 Read block 78 len 24576
178392024-04-17T17:21:27.393Z03/30 Flush
178402024-04-17T17:21:27.393Z04/30 Write block 128 len 28672 data: 10 11 13 14 14 10 11
178412024-04-17T17:21:27.393Z05/30 Write block 380 len 20480 data: 9 8 12 13 12
178422024-04-17T17:21:27.393Z06/30 Read block 534 len 40960
178432024-04-17T17:21:27.393Z07/30 Read block 398 len 8192
178442024-04-17T17:21:27.393Z08/30 Write block 227 len 36864 data: 15 14 15 14 12 12 12 13 18
178452024-04-17T17:21:27.393Z09/30 Write block 465 len 32768 data: 16 14 13 13 14 15 14 13
178462024-04-17T17:21:27.393Z10/30 Write block 64 len 32768 data: 17 16 15 15 14 12 12 12
178472024-04-17T17:21:27.394Z11/30 Read block 284 len 16384
178482024-04-17T17:21:27.398Z12/30 Read block 78 len 12288
178492024-04-17T17:21:27.398Z13/30 Read block 243 len 32768
178502024-04-17T17:21:27.398Z14/30 Write block 514 len 16384 data: 12 10 9 8
178512024-04-17T17:21:27.398Z15/30 Write block 117 len 8192 data: 11 7
178522024-04-17T17:21:27.399Z16/30 Read block 75 len 32768
178532024-04-17T17:21:27.399Z17/30 Read block 186 len 16384
178542024-04-17T17:21:27.399Z18/30 Flush
178552024-04-17T17:21:27.402Z19/30 Flush
178562024-04-17T17:21:27.402Z20/30 Write block 42 len 16384 data: 14 12 9 11
178572024-04-17T17:21:27.402Z21/30 Flush
178582024-04-17T17:21:27.406Z22/30 Write block 120 len 40960 data: 11 12 9 9 10 12 12 12 11 12
178592024-04-17T17:21:27.406Z23/30 Write block 432 len 4096 data: 14
178602024-04-17T17:21:27.406Z24/30 Flush
178612024-04-17T17:21:27.409Z25/30 Write block 368 len 4096 data: 15
178622024-04-17T17:21:27.412Z26/30 Write block 305 len 36864 data: 17 14 14 15 14 15 13 14 15
178632024-04-17T17:21:27.412Z27/30 Read block 225 len 16384
178642024-04-17T17:21:27.413Z28/30 Write block 33 len 16384 data: 21 25 20 17
178652024-04-17T17:21:27.417Z29/30 Write block 154 len 24576 data: 11 10 12 13 12 14
178662024-04-17T17:21:27.417Z30/30 Read block 98 len 4096
178672024-04-17T17:21:27.417Z----------------------------------------------------------------
178682024-04-17T17:21:27.417Z Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:30
178692024-04-17T17:21:27.417ZStates: Active Active Active
178702024-04-17T17:21:27.417ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
178712024-04-17T17:21:27.417Z 1 Acked 1000 Write 3 Done Done Sent false
178722024-04-17T17:21:27.417Z 2 Acked 1001 Read 6 Done Done Sent false
178732024-04-17T17:21:27.417Z 3 Acked 1002 Flush 0 Done Done Sent false
178742024-04-17T17:21:27.417Z 4 Acked 1003 Write 7 Done Done Sent false
178752024-04-17T17:21:27.417Z 5 Acked 1004 Write 5 Done Done Sent false
178762024-04-17T17:21:27.417Z 6 Acked 1005 Read 10 Done Done Sent false
178772024-04-17T17:21:27.417Z 7 Acked 1006 Read 2 Done Done Sent false
178782024-04-17T17:21:27.417Z 8 Acked 1007 Write 9 Done Done Sent false
178792024-04-17T17:21:27.417Z 9 Acked 1008 Write 8 Done Done Sent false
178802024-04-17T17:21:27.418Z 10 Acked 1009 Write 8 Done Done Sent false
178812024-04-17T17:21:27.418Z 11 Acked 1010 Read 4 Done Done Sent false
178822024-04-17T17:21:27.418Z 12 Acked 1011 Read 3 Done Done Sent false
178832024-04-17T17:21:27.418Z 13 Acked 1012 Read 8 Done Done Sent false
178842024-04-17T17:21:27.418Z 14 Acked 1013 Write 4 Done Done Sent false
178852024-04-17T17:21:27.418Z 15 Acked 1014 Write 2 Done Done Sent false
178862024-04-17T17:21:27.418Z 16 Acked 1015 Read 8 Done Done Sent false
178872024-04-17T17:21:27.418Z 17 Acked 1016 Read 4 Done Done Sent false
178882024-04-17T17:21:27.418Z 18 Acked 1017 Flush 0 Done Done Sent false
178892024-04-17T17:21:27.418Z 19 Acked 1018 Flush 0 Done Done Sent false
178902024-04-17T17:21:27.418Z 20 Acked 1019 Write 4 Done Done Sent false
178912024-04-17T17:21:27.418Z 21 Acked 1020 Flush 0 Done Done Sent false
178922024-04-17T17:21:27.418Z 22 Acked 1021 Write 10 Done Done Sent false
178932024-04-17T17:21:27.418Z 23 Acked 1022 Write 1 Done Done Sent false
178942024-04-17T17:21:27.418Z 24 Acked 1023 Flush 0 Done Done Sent false
178952024-04-17T17:21:27.418Z 25 Acked 1024 Write 1 Done Done Sent false
178962024-04-17T17:21:27.418Z 26 Acked 1025 Write 9 Done Done Sent false
178972024-04-17T17:21:27.418Z 27 Acked 1026 Read 4 Done Done Sent false
178982024-04-17T17:21:27.418Z 28 Acked 1027 Write 4 Done Done Sent false
178992024-04-17T17:21:27.419Z 29 Acked 1028 Write 6 Done Done Sent false
179002024-04-17T17:21:27.419Z 30 Acked 1029 Read 1 Done Done Sent false
179012024-04-17T17:21:27.419Z STATES DS:0 DS:1 DS:2 TOTAL
179022024-04-17T17:21:27.419Z New 0 0 0 0
179032024-04-17T17:21:27.419Z Sent 0 0 30 30
179042024-04-17T17:21:27.419Z Done 30 30 0 60
179052024-04-17T17:21:27.419Z Skipped 0 0 0 0
179062024-04-17T17:21:27.419Z Error 0 0 0 0
179072024-04-17T17:21:27.419ZLast Flush: 1023 1023 0
179082024-04-17T17:21:27.419ZDownstairs last five completed:
179092024-04-17T17:21:27.419ZUpstairs last five completed: 30 29 28 27 26
179102024-04-17T17:21:27.419Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:21:26.024342554Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1835}
179112024-04-17T17:21:27.419ZWrote out file "/tmp/test_repair_verify.data"
179122024-04-17T17:21:27.425Z
179132024-04-17T17:21:27.425ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
179142024-04-17T17:21:27.439ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
179152024-04-17T17:21:27.439Z 0 000-019 157 157 157 282 282 282 F F F
179162024-04-17T17:21:27.443Z 1 020-039 164 164 164 289 289 289 T T F <---
179172024-04-17T17:21:27.443Z 2 040-059 166 166 164 293 293 289 F F F <---
179182024-04-17T17:21:27.443Z 3 060-079 166 166 163 291 291 288 F F F <---
179192024-04-17T17:21:27.443Z 4 080-099 166 166 164 290 290 289 F F F <---
179202024-04-17T17:21:27.443Z 5 100-119 166 166 162 291 291 287 F F F <---
179212024-04-17T17:21:27.443Z 6 120-139 166 166 164 294 294 289 F F F <---
179222024-04-17T17:21:27.443Z 7 140-159 164 164 164 289 289 289 T T F <---
179232024-04-17T17:21:27.444Z 8 160-179 157 157 157 282 282 282 F F F
179242024-04-17T17:21:27.444Z 9 180-199 163 163 163 288 288 288 F F F
179252024-04-17T17:21:27.444Z 10 200-219 158 158 158 283 283 283 F F F
179262024-04-17T17:21:27.444Z 11 220-239 166 166 158 291 291 283 F F F <---
179272024-04-17T17:21:27.444Z 12 240-259 158 158 158 283 283 283 F F F
179282024-04-17T17:21:27.444Z 13 260-279 161 161 161 285 285 285 F F F
179292024-04-17T17:21:27.444Z 14 280-299 164 164 164 289 289 289 F F F
179302024-04-17T17:21:27.444Z 15 300-319 164 164 164 289 289 289 T T F <---
179312024-04-17T17:21:27.444Z 16 320-339 162 162 162 286 286 286 F F F
179322024-04-17T17:21:27.444Z 17 340-359 164 164 164 289 289 289 F F F
179332024-04-17T17:21:27.444Z 18 360-379 161 161 161 285 285 285 T T F <---
179342024-04-17T17:21:27.444Z 19 380-399 166 166 164 291 291 289 F F F <---
179352024-04-17T17:21:27.444Z 20 400-419 163 163 163 288 288 288 F F F
179362024-04-17T17:21:27.444Z 21 420-439 166 166 164 294 294 289 F F F <---
179372024-04-17T17:21:27.444Z 22 440-459 164 164 164 289 289 289 F F F
179382024-04-17T17:21:27.444Z 23 460-479 166 166 164 291 291 289 F F F <---
179392024-04-17T17:21:27.444Z 24 480-499 164 164 164 289 289 289 F F F
179402024-04-17T17:21:27.444Z 25 500-519 166 166 163 291 291 288 F F F <---
179412024-04-17T17:21:27.444Z 26 520-539 164 164 164 289 289 289 F F F
179422024-04-17T17:21:27.444Z 27 540-559 164 164 164 289 289 289 F F F
179432024-04-17T17:21:27.444Z 28 560-579 157 157 157 282 282 282 F F F
179442024-04-17T17:21:27.444Z 29 580-599 163 163 163 288 288 288 F F F
179452024-04-17T17:21:27.444ZMax gen: 166, Max flush: 294
179462024-04-17T17:21:27.445ZError: Difference in extent metadata found!
179472024-04-17T17:21:27.445ZA Difference in extent metadata is expected here
179482024-04-17T17:21:27.445ZOn loop 83
179492024-04-17T17:21:27.445Z
179502024-04-17T17:21:27.449ZVerifying data now
179512024-04-17T17:21:32.093ZNo change after verify
179522024-04-17T17:21:32.093ZLoop: 83 Downstairs dump after verify (and repair):
179532024-04-17T17:21:32.107ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
179542024-04-17T17:21:32.107Z 0 000-019 157 157 157 282 282 282 F F F
179552024-04-17T17:21:32.110Z 1 020-039 167 167 167 295 295 295 F F F
179562024-04-17T17:21:32.110Z 2 040-059 166 166 166 293 293 293 F F F
179572024-04-17T17:21:32.110Z 3 060-079 166 166 166 291 291 291 F F F
179582024-04-17T17:21:32.110Z 4 080-099 166 166 166 290 290 290 F F F
179592024-04-17T17:21:32.110Z 5 100-119 166 166 166 291 291 291 F F F
179602024-04-17T17:21:32.110Z 6 120-139 166 166 166 294 294 294 F F F
179612024-04-17T17:21:32.111Z 7 140-159 167 167 167 295 295 295 F F F
179622024-04-17T17:21:32.111Z 8 160-179 157 157 157 282 282 282 F F F
179632024-04-17T17:21:32.111Z 9 180-199 163 163 163 288 288 288 F F F
179642024-04-17T17:21:32.111Z 10 200-219 158 158 158 283 283 283 F F F
179652024-04-17T17:21:32.111Z 11 220-239 166 166 166 291 291 291 F F F
179662024-04-17T17:21:32.111Z 12 240-259 158 158 158 283 283 283 F F F
179672024-04-17T17:21:32.111Z 13 260-279 161 161 161 285 285 285 F F F
179682024-04-17T17:21:32.111Z 14 280-299 164 164 164 289 289 289 F F F
179692024-04-17T17:21:32.111Z 15 300-319 167 167 167 295 295 295 F F F
179702024-04-17T17:21:32.111Z 16 320-339 162 162 162 286 286 286 F F F
179712024-04-17T17:21:32.111Z 17 340-359 164 164 164 289 289 289 F F F
179722024-04-17T17:21:32.111Z 18 360-379 167 167 167 295 295 295 F F F
179732024-04-17T17:21:32.111Z 19 380-399 166 166 166 291 291 291 F F F
179742024-04-17T17:21:32.111Z 20 400-419 163 163 163 288 288 288 F F F
179752024-04-17T17:21:32.111Z 21 420-439 166 166 166 294 294 294 F F F
179762024-04-17T17:21:32.111Z 22 440-459 164 164 164 289 289 289 F F F
179772024-04-17T17:21:32.111Z 23 460-479 166 166 166 291 291 291 F F F
179782024-04-17T17:21:32.111Z 24 480-499 164 164 164 289 289 289 F F F
179792024-04-17T17:21:32.111Z 25 500-519 166 166 166 291 291 291 F F F
179802024-04-17T17:21:32.111Z 26 520-539 164 164 164 289 289 289 F F F
179812024-04-17T17:21:32.111Z 27 540-559 164 164 164 289 289 289 F F F
179822024-04-17T17:21:32.111Z 28 560-579 157 157 157 282 282 282 F F F
179832024-04-17T17:21:32.111Z 29 580-599 163 163 163 288 288 288 F F F
179842024-04-17T17:21:32.112ZMax gen: 167, Max flush: 295
179852024-04-17T17:21:32.115Z
179862024-04-17T17:21:32.115ZBegin loop 84
179872024-04-17T17:21:32.115ZDownstairs to restart: 2
179882024-04-17T17:21:32.131Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.741072602Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843}
179892024-04-17T17:21:32.137Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.741430303Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843}
179902024-04-17T17:21:32.137Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30Crucible runtime is spawned
179912024-04-17T17:21:32.137Z,"time":"2024-04-17T17:21:30.741462775Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843}
179922024-04-17T17:21:32.137Z{"msg":"Crucible 3f1ad826-369d-405b-aca7-7329e61bdeec has session id: 9d9cb162-b230-4ab5-b627-8e8a20b7ca0f","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.741495877Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
179932024-04-17T17:21:32.137Z{"msg":"Upstairs opts: Upstairs UUID: 3f1ad826-369d-405b-aca7-7329e61bdeec, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.741551591Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
179942024-04-17T17:21:32.137Z{"msg":"Crucible stats registered with UUID: 3f1ad826-369d-405b-aca7-7329e61bdeec","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.741595313Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
179952024-04-17T17:21:32.138Z{"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.741634636Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843}
179962024-04-17T17:21:32.138Z{"msg":"3f1ad826-369d-405b-aca7-7329e61bdeec active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.741656837Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
179972024-04-17T17:21:32.138Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.741691009Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"client":"0","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
179982024-04-17T17:21:32.138Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.74171055Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"client":"0","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
179992024-04-17T17:21:32.138Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.741748973Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"client":"1","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180002024-04-17T17:21:32.138Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.741787505Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"client":"1","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180012024-04-17T17:21:32.138Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.741815977Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"client":"2","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180022024-04-17T17:21:32.138Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.741859089Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"client":"2","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180032024-04-17T17:21:32.138Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.741889931Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"":"io task","client":"2","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180042024-04-17T17:21:32.138Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.741939614Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"":"io task","client":"0","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180052024-04-17T17:21:32.138Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.741966266Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"":"io task","client":"1","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180062024-04-17T17:21:32.138Z{"msg":"ds_connection connected from Ok(127.0.0.1:37855)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.741992427Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"":"io task","client":"0","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180072024-04-17T17:21:32.138Z{"msg":"ds_connection connected from Ok(127.0.0.1:36891)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.7420359Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"":"io task","client":"1","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180082024-04-17T17:21:32.138Z{"msg":"ds_connection connected from Ok(127.0.0.1:57219)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.742083173Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"":"io task","client":"2","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180092024-04-17T17:21:32.138Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.742680159Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"client":"0","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180102024-04-17T17:21:32.138Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.742707231Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"client":"0","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180112024-04-17T17:21:32.138Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.742722502Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"client":"1","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180122024-04-17T17:21:32.138Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.742736092Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"client":"1","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180132024-04-17T17:21:32.138Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.743075643Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"client":"2","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180142024-04-17T17:21:32.138Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.743112205Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"client":"2","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180152024-04-17T17:21:32.138Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.743270305Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"client":"1","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180162024-04-17T17:21:32.138Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.743308567Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"client":"0","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180172024-04-17T17:21:32.138Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.743631227Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"client":"2","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180182024-04-17T17:21:32.139Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.743798297Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"client":"0","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180192024-04-17T17:21:32.139Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.7438439Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"client":"0","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180202024-04-17T17:21:32.139Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.743875272Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180212024-04-17T17:21:32.139Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.743891253Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"client":"1","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180222024-04-17T17:21:32.139Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.743905323Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"client":"1","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180232024-04-17T17:21:32.139Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.743924675Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180242024-04-17T17:21:32.139Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.744061533Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"client":"2","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180252024-04-17T17:21:32.139Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.744118796Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"client":"2","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180262024-04-17T17:21:32.139Z{"msg":"[0]R flush_numbers[0..12]: [282, 295, 293, 291, 290, 291, 294, 295, 282, 288, 283, 291]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.744167569Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"":"downstairs","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180272024-04-17T17:21:32.139Z{"msg":"[0]R generation[0..12]: [157, 167, 166, 166, 166, 166, 166, 167, 157, 163, 158, 166]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.744203721Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"":"downstairs","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180282024-04-17T17:21:32.139Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.744240594Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843Wait for a query_work_queue command to finish before sending IO
180292024-04-17T17:21:32.139Z,"":"downstairs","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180302024-04-17T17:21:32.139Z{"msg":"[1]R flush_numbers[0..12]: [282, 295, 293, 291, 290, 291, 294, 295, 282, 288, 283, 291]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.74433805Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"":"downstairs","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180312024-04-17T17:21:32.139Z{"msg":"[1]R generation[0..12]: [157, 167, 166, 166, 166, 166, 166, 167, 157, 163, 158, 166]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.744380792Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"":"downstairs","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180322024-04-17T17:21:32.139Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.744408224Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"":"downstairs","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180332024-04-17T17:21:32.139Z{"msg":"[2]R flush_numbers[0..12]: [282, 295, 293, 291, 290, 291, 294, 295, 282, 288, 283, 291]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.744443476Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"":"downstairs","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180342024-04-17T17:21:32.139Z{"msg":"[2]R generation[0..12]: [157, 167, 166, 166, 166, 166, 166, 167, 157, 163, 158, 166]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.744470998Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"":"downstairs","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180352024-04-17T17:21:32.139Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.74450653Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"":"downstairs","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180362024-04-17T17:21:32.139Z{"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.744523471Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"":"downstairs","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180372024-04-17T17:21:32.139Z{"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.744545102Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"":"downstairs","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180382024-04-17T17:21:32.139Z{"msg":"Next flush: 296","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.744564033Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"":"downstairs","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180392024-04-17T17:21:32.139Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.744598575Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"":"downstairs","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180402024-04-17T17:21:32.140Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.744615637Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180412024-04-17T17:21:32.140Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
180422024-04-17T17:21:32.140Z,"time":"2024-04-17T17:21:30.744642118Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"client":"0","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180432024-04-17T17:21:32.140Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.74466596Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"client":"1","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180442024-04-17T17:21:32.140Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.744699171Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"client":"2","":"downstairs","session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180452024-04-17T17:21:32.140Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.744733573Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180462024-04-17T17:21:32.140Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.744763585Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180472024-04-17T17:21:32.140Z{"msg":"3f1ad826-369d-405b-aca7-7329e61bdeec is now active with session: 9d9cb162-b230-4ab5-b627-8e8a20b7ca0f","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.744795177Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180482024-04-17T17:21:32.140Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.744823909Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843,"session_id":"9d9cb162-b230-4ab5-b627-8e8a20b7ca0f"}
180492024-04-17T17:21:32.140Z{"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:30.744853561Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843}
180502024-04-17T17:21:32.144ZLoading write count information from file "/tmp/test_repair_verify.data"
180512024-04-17T17:21:32.144ZRun Repair workload
180522024-04-17T17:21:32.144Z01/30 Write block 75 len 32768 data: 14 13 12 11 11 16 14 15
180532024-04-17T17:21:32.144Z02/30 Flush
180542024-04-17T17:21:32.144Z03/30 Write block 81 len 32768 data: 15 16 18 14 15 13 19 16
180552024-04-17T17:21:32.144Z04/30 Write block 178 len 40960 data: 16 13 10 12 13 11 11 11 11 17
180562024-04-17T17:21:32.144Z05/30 Read block 480 len 12288
180572024-04-17T17:21:32.144Z06/30 Read block 534 len 12288
180582024-04-17T17:21:32.148Z07/30 Write block 536 len 12288 data: 7 5 6
180592024-04-17T17:21:32.148Z08/30 Write block 112 len 36864 data: 14 13 11 12 12 12 8 10 12
180602024-04-17T17:21:32.148Z09/30 Write block 191 len 24576 data: 12 11 10 8 9 10
180612024-04-17T17:21:32.148Z10/30 Read block 348 len 24576
180622024-04-17T17:21:32.151Z11/30 Read block 269 len 4096
180632024-04-17T17:21:32.151Z12/30 Write block 414 len 12288 data: 14 13 15
180642024-04-17T17:21:32.151Z13/30 Flush
180652024-04-17T17:21:32.155Z14/30 Write block 93 len 4096 data: 18
180662024-04-17T17:21:32.155Z15/30 Read block 108 len 24576
180672024-04-17T17:21:32.155Z16/30 Write block 132 len 8192 data: 15 11
180682024-04-17T17:21:32.155Z17/30 Read block 341 len 24576
180692024-04-17T17:21:32.155Z18/30 Read block 406 len 24576
180702024-04-17T17:21:32.158Z19/30 Write block 127 len 12288 data: 13 12 13
180712024-04-17T17:21:32.158Z20/30 Write block 262 len 32768 data: 13 12 12 11 11 13 14 19
180722024-04-17T17:21:32.158Z21/30 Read block 170 len 40960
180732024-04-17T17:21:32.158Z22/30 Read block 252 len 36864
180742024-04-17T17:21:32.162Z23/30 Read block 67 len 36864
180752024-04-17T17:21:32.162Z24/30 Read block 233 len 8192
180762024-04-17T17:21:32.162Z25/30 Read block 171 len 20480
180772024-04-17T17:21:32.162Z26/30 Write block 155 len 12288 data: 11 13 14
180782024-04-17T17:21:32.162Z27/30 Read block 68 len 4096
180792024-04-17T17:21:32.166Z28/30 Write block 442 len 36864 data: 14 14 13 9 10 8 9 10 9
180802024-04-17T17:21:32.166Z29/30 Read block 321 len 20480
180812024-04-17T17:21:32.167Z30/30 Write block 379 len 28672 data: 9 10 9 13 14 13 14
180822024-04-17T17:21:32.167Z----------------------------------------------------------------
180832024-04-17T17:21:32.167Z Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:28
180842024-04-17T17:21:32.167ZStates: Active Active Active
180852024-04-17T17:21:32.167ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
180862024-04-17T17:21:32.167Z 3 Acked 1002 Write 8 Done Done Done false
180872024-04-17T17:21:32.167Z 4 Acked 1003 Write 10 Done Done Done false
180882024-04-17T17:21:32.167Z 5 Acked 1004 Read 3 Done Done Sent false
180892024-04-17T17:21:32.167Z 6 Acked 1005 Read 3 Done Done Sent false
180902024-04-17T17:21:32.167Z 7 Acked 1006 Write 3 Done Done Sent false
180912024-04-17T17:21:32.167Z 8 Acked 1007 Write 9 Done Done Sent false
180922024-04-17T17:21:32.167Z 9 Acked 1008 Write 6 Done Done Sent false
180932024-04-17T17:21:32.167Z 10 Acked 1009 Read 6 Done Done Sent false
180942024-04-17T17:21:32.167Z 11 Acked 1010 Read 1 Done Done Sent false
180952024-04-17T17:21:32.167Z 12 Acked 1011 Write 3 Done Done Sent false
180962024-04-17T17:21:32.167Z 13 Acked 1012 Flush 0 Done Done Sent false
180972024-04-17T17:21:32.167Z 14 Acked 1013 Write 1 Done Done Sent false
180982024-04-17T17:21:32.167Z 15 Acked 1014 Read 6 Done Done Sent false
180992024-04-17T17:21:32.167Z 16 Acked 1015 Write 2 Done Done Sent false
181002024-04-17T17:21:32.167Z 17 Acked 1016 Read 6 Done Done Sent false
181012024-04-17T17:21:32.167Z 18 Acked 1017 Read 6 Done Done Sent false
181022024-04-17T17:21:32.167Z 19 Acked 1018 Write 3 Done Done Sent false
181032024-04-17T17:21:32.167Z 20 Acked 1019 Write 8 Done Done Sent false
181042024-04-17T17:21:32.168Z 21 Acked 1020 Read 10 Done Done Sent false
181052024-04-17T17:21:32.168Z 22 Acked 1021 Read 9 Done Done Sent false
181062024-04-17T17:21:32.168Z 23 Acked 1022 Read 9 Done Done Sent false
181072024-04-17T17:21:32.168Z 24 Acked 1023 Read 2 Done Done Sent false
181082024-04-17T17:21:32.168Z 25 Acked 1024 Read 5 Done Done Sent false
181092024-04-17T17:21:32.168Z 26 Acked 1025 Write 3 Done Done Sent false
181102024-04-17T17:21:32.168Z 27 Acked 1026 Read 1 Done Done Sent false
181112024-04-17T17:21:32.168Z 28 Acked 1027 Write 9 Done Done Sent false
181122024-04-17T17:21:32.168Z 29 Acked 1028 Read 5 Done Done Sent false
181132024-04-17T17:21:32.168Z 30 Acked 1029 Write 7 Sent Sent Sent false
181142024-04-17T17:21:32.168Z STATES DS:0 DS:1 DS:2 TOTAL
181152024-04-17T17:21:32.168Z New 0 0 0 0
181162024-04-17T17:21:32.168Z Sent 1 1 26 28
181172024-04-17T17:21:32.168Z Done 27 27 2 56
181182024-04-17T17:21:32.168Z Skipped 0 0 0 0
181192024-04-17T17:21:32.168Z Error 0 0 0 0
181202024-04-17T17:21:32.168ZLast Flush: 1012 1012 1001
181212024-04-17T17:21:32.168ZDownstairs last five completed: 1001 1000
181222024-04-17T17:21:32.168ZUpstairs last five completed: 30 29 28 27 26
181232024-04-17T17:21:32.172Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:21:30.77602436Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1843}
181242024-04-17T17:21:32.172ZWrote out file "/tmp/test_repair_verify.data"
181252024-04-17T17:21:32.172Z
181262024-04-17T17:21:32.176ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
181272024-04-17T17:21:32.192ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
181282024-04-17T17:21:32.192Z 0 000-019 157 157 157 282 282 282 F F F
181292024-04-17T17:21:32.196Z 1 020-039 167 167 167 295 295 295 F F F
181302024-04-17T17:21:32.196Z 2 040-059 166 166 166 293 293 293 F F F
181312024-04-17T17:21:32.196Z 3 060-079 168 168 168 296 296 296 F F F
181322024-04-17T17:21:32.196Z 4 080-099 168 168 168 297 297 296 T T T <---
181332024-04-17T17:21:32.196Z 5 100-119 168 168 166 297 297 291 F F F <---
181342024-04-17T17:21:32.196Z 6 120-139 168 168 166 297 297 294 T T F <---
181352024-04-17T17:21:32.196Z 7 140-159 167 167 167 295 295 295 T T F <---
181362024-04-17T17:21:32.196Z 8 160-179 168 168 157 297 297 282 F F T <---
181372024-04-17T17:21:32.196Z 9 180-199 168 168 163 297 297 288 F F T <---
181382024-04-17T17:21:32.196Z 10 200-219 158 158 158 283 283 283 F F F
181392024-04-17T17:21:32.196Z 11 220-239 166 166 166 291 291 291 F F F
181402024-04-17T17:21:32.196Z 12 240-259 158 158 158 283 283 283 F F F
181412024-04-17T17:21:32.196Z 13 260-279 161 161 161 285 285 285 T T F <---
181422024-04-17T17:21:32.196Z 14 280-299 164 164 164 289 289 289 F F F
181432024-04-17T17:21:32.196Z 15 300-319 167 167 167 295 295 295 F F F
181442024-04-17T17:21:32.196Z 16 320-339 162 162 162 286 286 286 F F F
181452024-04-17T17:21:32.196Z 17 340-359 164 164 164 289 289 289 F F F
181462024-04-17T17:21:32.196Z 18 360-379 167 167 167 295 295 295 T T F <---
181472024-04-17T17:21:32.196Z 19 380-399 166 166 166 291 291 291 T T F <---
181482024-04-17T17:21:32.196Z 20 400-419 168 168 163 297 297 288 F F F <---
181492024-04-17T17:21:32.196Z 21 420-439 166 166 166 294 294 294 F F F
181502024-04-17T17:21:32.196Z 22 440-459 164 164 164 289 289 289 T T F <---
181512024-04-17T17:21:32.196Z 23 460-479 166 166 166 291 291 291 F F F
181522024-04-17T17:21:32.197Z 24 480-499 164 164 164 289 289 289 F F F
181532024-04-17T17:21:32.197Z 25 500-519 166 166 166 291 291 291 F F F
181542024-04-17T17:21:32.197Z 26 520-539 168 168 164 297 297 289 F F F <---
181552024-04-17T17:21:32.197Z 27 540-559 164 164 164 289 289 289 F F F
181562024-04-17T17:21:32.197Z 28 560-579 157 157 157 282 282 282 F F F
181572024-04-17T17:21:32.197Z 29 580-599 163 163 163 288 288 288 F F F
181582024-04-17T17:21:32.197ZMax gen: 168, Max flush: 297
181592024-04-17T17:21:32.197ZError: Difference in extent metadata found!
181602024-04-17T17:21:32.197ZA Difference in extent metadata is expected here
181612024-04-17T17:21:32.197ZOn loop 84
181622024-04-17T17:21:32.197Z
181632024-04-17T17:21:32.200ZVerifying data now
181642024-04-17T17:21:42.766ZNo change after verify
181652024-04-17T17:21:42.766ZLoop: 84 Downstairs dump after verify (and repair):
181662024-04-17T17:21:42.781ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
181672024-04-17T17:21:42.781Z 0 000-019 157 157 157 282 282 282 F F F
181682024-04-17T17:21:42.781Z 1 020-039 167 167 167 295 295 295 F F F
181692024-04-17T17:21:42.781Z 2 040-059 166 166 166 293 293 293 F F F
181702024-04-17T17:21:42.781Z 3 060-079 168 168 168 296 296 296 F F F
181712024-04-17T17:21:42.781Z 4 080-099 169 169 169 298 298 298 F F F
181722024-04-17T17:21:42.781Z 5 100-119 168 168 168 297 297 297 F F F
181732024-04-17T17:21:42.781Z 6 120-139 169 169 169 298 298 298 F F F
181742024-04-17T17:21:42.785Z 7 140-159 169 169 169 298 298 298 F F F
181752024-04-17T17:21:42.785Z 8 160-179 168 168 168 297 297 297 F F F
181762024-04-17T17:21:42.785Z 9 180-199 168 168 168 297 297 297 F F F
181772024-04-17T17:21:42.785Z 10 200-219 158 158 158 283 283 283 F F F
181782024-04-17T17:21:42.785Z 11 220-239 166 166 166 291 291 291 F F F
181792024-04-17T17:21:42.785Z 12 240-259 158 158 158 283 283 283 F F F
181802024-04-17T17:21:42.785Z 13 260-279 169 169 169 298 298 298 F F F
181812024-04-17T17:21:42.785Z 14 280-299 164 164 164 289 289 289 F F F
181822024-04-17T17:21:42.785Z 15 300-319 167 167 167 295 295 295 F F F
181832024-04-17T17:21:42.785Z 16 320-339 162 162 162 286 286 286 F F F
181842024-04-17T17:21:42.785Z 17 340-359 164 164 164 289 289 289 F F F
181852024-04-17T17:21:42.785Z 18 360-379 169 169 169 298 298 298 F F F
181862024-04-17T17:21:42.786Z 19 380-399 169 169 169 298 298 298 F F F
181872024-04-17T17:21:42.786Z 20 400-419 168 168 168 297 297 297 F F F
181882024-04-17T17:21:42.786Z 21 420-439 166 166 166 294 294 294 F F F
181892024-04-17T17:21:42.786Z 22 440-459 169 169 169 298 298 298 F F F
181902024-04-17T17:21:42.786Z 23 460-479 166 166 166 291 291 291 F F F
181912024-04-17T17:21:42.786Z 24 480-499 164 164 164 289 289 289 F F F
181922024-04-17T17:21:42.786Z 25 500-519 166 166 166 291 291 291 F F F
181932024-04-17T17:21:42.786Z 26 520-539 168 168 168 297 297 297 F F F
181942024-04-17T17:21:42.786Z 27 540-559 164 164 164 289 289 289 F F F
181952024-04-17T17:21:42.786Z 28 560-579 157 157 157 282 282 282 F F F
181962024-04-17T17:21:42.786Z 29 580-599 163 163 163 288 288 288 F F F
181972024-04-17T17:21:42.786ZMax gen: 169, Max flush: 298
181982024-04-17T17:21:42.786Z
181992024-04-17T17:21:42.786ZBegin loop 85
182002024-04-17T17:21:42.786ZDownstairs to restart: 2
182012024-04-17T17:21:42.804Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.413909909Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851}
182022024-04-17T17:21:42.804Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\nCrucible runtime is spawned
182032024-04-17T17:21:42.804Z opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.414201017Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851}
182042024-04-17T17:21:42.804Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.414226719Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851}
182052024-04-17T17:21:42.810Z{"msg":"Crucible 90f3905c-fa99-4fd2-87e3-4a817643c95c has session id: 9969fba0-d79e-4c03-8ab0-38553f12c904","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.41424755Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182062024-04-17T17:21:42.811Z{"msg":"Upstairs opts: Upstairs UUID: 90f3905c-fa99-4fd2-87e3-4a817643c95c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.414296623Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182072024-04-17T17:21:42.811Z{"msg":"Crucible stats registered with UUID: 90f3905c-fa99-4fd2-87e3-4a817643c95c","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.414329895Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182082024-04-17T17:21:42.811Z{"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.414350726Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851}
182092024-04-17T17:21:42.811Z{"msg":"90f3905c-fa99-4fd2-87e3-4a817643c95c active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.414369217Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182102024-04-17T17:21:42.811Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.414400979Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"client":"0","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182112024-04-17T17:21:42.811Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.414425091Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"client":"0","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182122024-04-17T17:21:42.811Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.414462923Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"client":"1","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182132024-04-17T17:21:42.811Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.414487144Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"client":"1","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182142024-04-17T17:21:42.811Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.414517856Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"client":"2","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182152024-04-17T17:21:42.811Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.414540718Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"client":"2","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182162024-04-17T17:21:42.811Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.41458102Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"":"io task","client":"0","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182172024-04-17T17:21:42.811Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.414613732Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"":"io task","client":"1","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182182024-04-17T17:21:42.811Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.414645924Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"":"io task","client":"2","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182192024-04-17T17:21:42.811Z{"msg":"ds_connection connected from Ok(127.0.0.1:47542)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.41475108Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"":"io task","client":"0","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182202024-04-17T17:21:42.811Z{"msg":"ds_connection connected from Ok(127.0.0.1:65215)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.414789743Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"":"io task","client":"2","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182212024-04-17T17:21:42.811Z{"msg":"ds_connection connected from Ok(127.0.0.1:55925)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.414827705Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"":"io task","client":"1","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182222024-04-17T17:21:42.811Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.41557369Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"client":"0","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182232024-04-17T17:21:42.811Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.415622943Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"client":"0","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182242024-04-17T17:21:42.811Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.415653105Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"client":"1","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182252024-04-17T17:21:42.811Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.415681847Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"client":"1","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182262024-04-17T17:21:42.811Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.416103332Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"client":"0","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182272024-04-17T17:21:42.811Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.416257212Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"client":"2","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182282024-04-17T17:21:42.812Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.416293664Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"client":"2","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182292024-04-17T17:21:42.812Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.416356858Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"client":"1","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182302024-04-17T17:21:42.812Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.416654906Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"client":"0","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182312024-04-17T17:21:42.812Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.416693778Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"client":"0","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182322024-04-17T17:21:42.812Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.41671919Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182332024-04-17T17:21:42.812Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.416906171Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"client":"1","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182342024-04-17T17:21:42.812Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.416964235Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"client":"1","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182352024-04-17T17:21:42.812Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.417002387Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182362024-04-17T17:21:42.812Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.417023838Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"client":"2","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182372024-04-17T17:21:42.812Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.417429903Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"client":"2","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182382024-04-17T17:21:42.812Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.417468555Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"client":"2","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182392024-04-17T17:21:42.812Z{"msg":"[0]R flush_numbers[0..12]: [282, 295, 293, 296, 298, 297, 298, 298, 297, 297, 283, 291]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.417494807Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"":"downstairs","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182402024-04-17T17:21:42.812Z{"msg":"[0]R generation[0..12]: [157, 167, 166, 168, 169, 168, 169, 169, 168, 168, 158, 166]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.417523798Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"":"downstairs","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182412024-04-17T17:21:42.812Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.41754908Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"":"downstairs","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182422024-04-17T17:21:42.812Z{"msg":"[1]R flush_numbers[0..12]: [282, 295, 293, 296, 298, 297, 298, 298, 297, 297, 283, 291]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.417577512Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"":"downstairs","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182432024-04-17T17:21:42.812Z{"msg":"[1]R generation[0..12]: [157, 167, 166, 168, 169, 168, 169, 169, 168, 168, 158, 166]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.417602053Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"":"downstairs","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182442024-04-17T17:21:42.812Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
182452024-04-17T17:21:42.812Z,"time":"2024-04-17T17:21:41.417631105Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"":"downstairs","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182462024-04-17T17:21:42.812Z{"msg":"[2]R flush_numbers[0..12]: [282, 295, 293, 296, 298, 297, 298, 298, 297, 297, 283, 291]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.417656746Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"":"downstairs","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182472024-04-17T17:21:42.812Z{"msg":"[2]R generation[0..12]: [157, 167, 166, 168, 169, 168, 169, 169, 168, 168, 158, 166]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.417685628Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"":"downstairs","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182482024-04-17T17:21:42.812Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.41771032Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"":"downstairs","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182492024-04-17T17:21:42.812Z{"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.417735831Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"":"downstairs","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182502024-04-17T17:21:42.812Z{"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.417759153Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"":"downstairs","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182512024-04-17T17:21:42.812Z{"msg":"Next flush: 299","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.417785794Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"":"downstairs","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182522024-04-17T17:21:42.813Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.417808376Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"":"downstairs","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182532024-04-17T17:21:42.813Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.417840298Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182542024-04-17T17:21:42.813Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.417862149Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"client":"0","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182552024-04-17T17:21:42.813Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.417889371Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"client":"1","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182562024-04-17T17:21:42.813Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.417911642Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"client":"2","":"downstairs","session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182572024-04-17T17:21:42.813ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
182582024-04-17T17:21:42.813Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.417941474Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182592024-04-17T17:21:42.813Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.417974206Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182602024-04-17T17:21:42.813Z{"msg":"90f3905c-fa99-4fd2-87e3-4a817643c95c is now active with session: 9969fba0-d79e-4c03-8ab0-38553f12c904","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.418001978Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182612024-04-17T17:21:42.813Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.418023449Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851,"session_id":"9969fba0-d79e-4c03-8ab0-38553f12c904"}
182622024-04-17T17:21:42.813Z{"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:41.41804984Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851}
182632024-04-17T17:21:42.817ZLoading write count information from file "/tmp/test_repair_verify.data"
182642024-04-17T17:21:42.817ZRun Repair workload
182652024-04-17T17:21:42.817Z01/30 Write block 83 len 4096 data: 19
182662024-04-17T17:21:42.817Z02/30 Write block 272 len 16384 data: 16 19 16 14
182672024-04-17T17:21:42.817Z03/30 Write block 133 len 32768 data: 12 12 11 14 14 12 12 10
182682024-04-17T17:21:42.817Z04/30 Read block 505 len 24576
182692024-04-17T17:21:42.817Z05/30 Read block 451 len 32768
182702024-04-17T17:21:42.817Z06/30 Write block 165 len 28672 data: 11 15 16 15 13 12 15
182712024-04-17T17:21:42.817Z07/30 Read block 163 len 4096
182722024-04-17T17:21:42.817Z08/30 Read block 293 len 16384
182732024-04-17T17:21:42.821Z09/30 Write block 0 len 24576 data: 6 7 7 11 13 14
182742024-04-17T17:21:42.821Z10/30 Read block 406 len 24576
182752024-04-17T17:21:42.821Z11/30 Write block 54 len 36864 data: 18 16 16 16 18 17 17 16 14
182762024-04-17T17:21:42.821Z12/30 Read block 348 len 12288
182772024-04-17T17:21:42.824Z13/30 Write block 527 len 36864 data: 13 14 14 13 12 8 7 5 6
182782024-04-17T17:21:42.824Z14/30 Write block 405 len 12288 data: 12 12 10
182792024-04-17T17:21:42.824Z15/30 Read block 57 len 36864
182802024-04-17T17:21:42.825Z16/30 Write block 499 len 20480 data: 14 15 16 14 14
182812024-04-17T17:21:42.831Z17/30 Write block 43 len 28672 data: 13 10 12 11 10 8 11
182822024-04-17T17:21:42.831Z18/30 Read block 448 len 32768
182832024-04-17T17:21:42.831Z19/30 Read block 325 len 40960
182842024-04-17T17:21:42.835Z20/30 Write block 568 len 32768 data: 11 10 9 9 11 12 16 17
182852024-04-17T17:21:42.835Z21/30 Read block 97 len 36864
182862024-04-17T17:21:42.835Z22/30 Write block 427 len 40960 data: 10 9 11 11 12 15 16 16 16 16
182872024-04-17T17:21:42.835Z23/30 Read block 7 len 32768
182882024-04-17T17:21:42.835Z24/30 Read block 201 len 8192
182892024-04-17T17:21:42.835Z25/30 Write block 15 len 4096 data: 10
182902024-04-17T17:21:42.836Z26/30 Read block 575 len 40960
182912024-04-17T17:21:42.836Z27/30 Write block 329 len 36864 data: 10 8 10 9 9 11 13 17 19
182922024-04-17T17:21:42.840Z28/30 Read block 73 len 16384
182932024-04-17T17:21:42.840Z29/30 Write block 538 len 36864 data: 7 13 14 15 19 17 17 13 14
182942024-04-17T17:21:42.840Z30/30 Read block 464 len 24576
182952024-04-17T17:21:42.840Z----------------------------------------------------------------
182962024-04-17T17:21:42.840Z Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30
182972024-04-17T17:21:42.840ZStates: Active Active Active
182982024-04-17T17:21:42.840ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
182992024-04-17T17:21:42.840Z 1 Acked 1000 Write 1 Done Done Done false
183002024-04-17T17:21:42.840Z 2 Acked 1001 Write 4 Done Done Done false
183012024-04-17T17:21:42.841Z 3 Acked 1002 Write 8 Done Done Sent false
183022024-04-17T17:21:42.841Z 4 Acked 1003 Read 6 Done Done Sent false
183032024-04-17T17:21:42.841Z 5 Acked 1004 Read 8 Done Done Sent false
183042024-04-17T17:21:42.841Z 6 Acked 1005 Write 7 Done Done Sent false
183052024-04-17T17:21:42.841Z 7 Acked 1006 Read 1 Done Done Sent false
183062024-04-17T17:21:42.841Z 8 Acked 1007 Read 4 Done Done Sent false
183072024-04-17T17:21:42.841Z 9 Acked 1008 Write 6 Done Done Sent false
183082024-04-17T17:21:42.841Z 10 Acked 1009 Read 6 Done Done Sent false
183092024-04-17T17:21:42.841Z 11 Acked 1010 Write 9 Done Done Sent false
183102024-04-17T17:21:42.841Z 12 Acked 1011 Read 3 Done Done Sent false
183112024-04-17T17:21:42.841Z 13 Acked 1012 Write 9 Done Done Sent false
183122024-04-17T17:21:42.841Z 14 Acked 1013 Write 3 Done Done Sent false
183132024-04-17T17:21:42.841Z 15 Acked 1014 Read 9 Done Done Sent false
183142024-04-17T17:21:42.841Z 16 Acked 1015 Write 5 Done Done Sent false
183152024-04-17T17:21:42.841Z 17 Acked 1016 Write 7 Done Done Sent false
183162024-04-17T17:21:42.841Z 18 Acked 1017 Read 8 Done Done Sent false
183172024-04-17T17:21:42.841Z 19 Acked 1018 Read 10 Done Done Sent false
183182024-04-17T17:21:42.841Z 20 Acked 1019 Write 8 Done Done Sent false
183192024-04-17T17:21:42.841Z 21 Acked 1020 Read 9 Done Done Sent false
183202024-04-17T17:21:42.841Z 22 Acked 1021 Write 10 Done Done Sent false
183212024-04-17T17:21:42.841Z 23 Acked 1022 Read 8 Done Done Sent false
183222024-04-17T17:21:42.841Z 24 Acked 1023 Read 2 Done Done Sent false
183232024-04-17T17:21:42.841Z 25 Acked 1024 Write 1 Done Done Sent false
183242024-04-17T17:21:42.841Z 26 Acked 1025 Read 10 Done Done Sent false
183252024-04-17T17:21:42.842Z 27 Acked 1026 Write 9 Done Done Sent false
183262024-04-17T17:21:42.842Z 28 Acked 1027 Read 4 Done Done Sent false
183272024-04-17T17:21:42.842Z 29 Acked 1028 Write 9 Done Done Sent false
183282024-04-17T17:21:42.842Z 30 Acked 1029 Read 6 Done Sent Sent false
183292024-04-17T17:21:42.842Z STATES DS:0 DS:1 DS:2 TOTAL
183302024-04-17T17:21:42.842Z New 0 0 0 0
183312024-04-17T17:21:42.842Z Sent 0 1 28 29
183322024-04-17T17:21:42.842Z Done 30 29 2 61
183332024-04-17T17:21:42.842Z Skipped 0 0 0 0
183342024-04-17T17:21:42.842Z Error 0 0 0 0
183352024-04-17T17:21:42.842ZLast Flush: 0 0 0
183362024-04-17T17:21:42.842ZDownstairs last five completed:
183372024-04-17T17:21:42.842ZUpstairs last five completed: 30 29 28 27 26
183382024-04-17T17:21:42.842Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:21:41.44906018Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1851}
183392024-04-17T17:21:42.842ZWrote out file "/tmp/test_repair_verify.data"
183402024-04-17T17:21:42.851Z
183412024-04-17T17:21:42.851ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
183422024-04-17T17:21:42.867ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
183432024-04-17T17:21:42.867Z 0 000-019 157 157 157 282 282 282 T T F <---
183442024-04-17T17:21:42.871Z 1 020-039 167 167 167 295 295 295 F F F
183452024-04-17T17:21:42.871Z 2 040-059 166 166 166 293 293 293 T T F <---
183462024-04-17T17:21:42.871Z 3 060-079 168 168 168 296 296 296 T T F <---
183472024-04-17T17:21:42.871Z 4 080-099 169 169 169 298 298 298 T T T
183482024-04-17T17:21:42.871Z 5 100-119 168 168 168 297 297 297 F F F
183492024-04-17T17:21:42.871Z 6 120-139 169 169 169 298 298 298 T T F <---
183502024-04-17T17:21:42.871Z 7 140-159 169 169 169 298 298 298 T T F <---
183512024-04-17T17:21:42.871Z 8 160-179 168 168 168 297 297 297 T T F <---
183522024-04-17T17:21:42.871Z 9 180-199 168 168 168 297 297 297 F F F
183532024-04-17T17:21:42.871Z 10 200-219 158 158 158 283 283 283 F F F
183542024-04-17T17:21:42.871Z 11 220-239 166 166 166 291 291 291 F F F
183552024-04-17T17:21:42.871Z 12 240-259 158 158 158 283 283 283 F F F
183562024-04-17T17:21:42.871Z 13 260-279 169 169 169 298 298 298 T T T
183572024-04-17T17:21:42.871Z 14 280-299 164 164 164 289 289 289 F F F
183582024-04-17T17:21:42.871Z 15 300-319 167 167 167 295 295 295 F F F
183592024-04-17T17:21:42.871Z 16 320-339 162 162 162 286 286 286 T T F <---
183602024-04-17T17:21:42.871Z 17 340-359 164 164 164 289 289 289 F F F
183612024-04-17T17:21:42.871Z 18 360-379 169 169 169 298 298 298 F F F
183622024-04-17T17:21:42.872Z 19 380-399 169 169 169 298 298 298 F F F
183632024-04-17T17:21:42.872Z 20 400-419 168 168 168 297 297 297 T T F <---
183642024-04-17T17:21:42.872Z 21 420-439 166 166 166 294 294 294 T T F <---
183652024-04-17T17:21:42.872Z 22 440-459 169 169 169 298 298 298 F F F
183662024-04-17T17:21:42.872Z 23 460-479 166 166 166 291 291 291 F F F
183672024-04-17T17:21:42.872Z 24 480-499 164 164 164 289 289 289 T T F <---
183682024-04-17T17:21:42.872Z 25 500-519 166 166 166 291 291 291 T T F <---
183692024-04-17T17:21:42.872Z 26 520-539 168 168 168 297 297 297 T T F <---
183702024-04-17T17:21:42.872Z 27 540-559 164 164 164 289 289 289 T T F <---
183712024-04-17T17:21:42.872Z 28 560-579 157 157 157 282 282 282 T T F <---
183722024-04-17T17:21:42.872Z 29 580-599 163 163 163 288 288 288 F F F
183732024-04-17T17:21:42.872ZMax gen: 169, Max flush: 298
183742024-04-17T17:21:42.872ZError: Difference in extent metadata found!
183752024-04-17T17:21:42.876ZA Difference in extent metadata is expected here
183762024-04-17T17:21:42.876ZOn loop 85
183772024-04-17T17:21:42.876Z
183782024-04-17T17:21:42.876ZVerifying data now
183792024-04-17T17:21:47.636ZNo change after verify
183802024-04-17T17:21:47.636ZLoop: 85 Downstairs dump after verify (and repair):
183812024-04-17T17:21:47.651ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
183822024-04-17T17:21:47.651Z 0 000-019 170 170 170 299 299 299 F F F
183832024-04-17T17:21:47.655Z 1 020-039 167 167 167 295 295 295 F F F
183842024-04-17T17:21:47.655Z 2 040-059 170 170 170 299 299 299 F F F
183852024-04-17T17:21:47.655Z 3 060-079 170 170 170 299 299 299 F F F
183862024-04-17T17:21:47.655Z 4 080-099 170 170 170 299 299 299 F F F
183872024-04-17T17:21:47.655Z 5 100-119 168 168 168 297 297 297 F F F
183882024-04-17T17:21:47.655Z 6 120-139 170 170 170 299 299 299 F F F
183892024-04-17T17:21:47.655Z 7 140-159 170 170 170 299 299 299 F F F
183902024-04-17T17:21:47.655Z 8 160-179 170 170 170 299 299 299 F F F
183912024-04-17T17:21:47.655Z 9 180-199 168 168 168 297 297 297 F F F
183922024-04-17T17:21:47.655Z 10 200-219 158 158 158 283 283 283 F F F
183932024-04-17T17:21:47.655Z 11 220-239 166 166 166 291 291 291 F F F
183942024-04-17T17:21:47.655Z 12 240-259 158 158 158 283 283 283 F F F
183952024-04-17T17:21:47.656Z 13 260-279 170 170 170 299 299 299 F F F
183962024-04-17T17:21:47.656Z 14 280-299 164 164 164 289 289 289 F F F
183972024-04-17T17:21:47.656Z 15 300-319 167 167 167 295 295 295 F F F
183982024-04-17T17:21:47.656Z 16 320-339 170 170 170 299 299 299 F F F
183992024-04-17T17:21:47.656Z 17 340-359 164 164 164 289 289 289 F F F
184002024-04-17T17:21:47.656Z 18 360-379 169 169 169 298 298 298 F F F
184012024-04-17T17:21:47.656Z 19 380-399 169 169 169 298 298 298 F F F
184022024-04-17T17:21:47.656Z 20 400-419 170 170 170 299 299 299 F F F
184032024-04-17T17:21:47.656Z 21 420-439 170 170 170 299 299 299 F F F
184042024-04-17T17:21:47.656Z 22 440-459 169 169 169 298 298 298 F F F
184052024-04-17T17:21:47.656Z 23 460-479 166 166 166 291 291 291 F F F
184062024-04-17T17:21:47.656Z 24 480-499 170 170 170 299 299 299 F F F
184072024-04-17T17:21:47.656Z 25 500-519 170 170 170 299 299 299 F F F
184082024-04-17T17:21:47.656Z 26 520-539 170 170 170 299 299 299 F F F
184092024-04-17T17:21:47.656Z 27 540-559 170 170 170 299 299 299 F F F
184102024-04-17T17:21:47.656Z 28 560-579 170 170 170 299 299 299 F F F
184112024-04-17T17:21:47.656Z 29 580-599 163 163 163 288 288 288 F F F
184122024-04-17T17:21:47.656ZMax gen: 170, Max flush: 299
184132024-04-17T17:21:47.656Z
184142024-04-17T17:21:47.656ZBegin loop 86
184152024-04-17T17:21:47.656ZDownstairs to restart: 1
184162024-04-17T17:21:47.674Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.283851483Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859}
184172024-04-17T17:21:47.674Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.284071887Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859}
184182024-04-17T17:21:47.680Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.284107719Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859}
184192024-04-17T17:21:47.680Z{"msg":"Crucible fa4c48fb-3bcd-4808-ad47-28e0a2488dd8 has session id: a2ba94c9-3a5a-4790-8245-d6c2cec454d8","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.284146181Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184202024-04-17T17:21:47.680ZCrucible runtime is spawned
184212024-04-17T17:21:47.680Z{"msg":"Upstairs opts: Upstairs UUID: fa4c48fb-3bcd-4808-ad47-28e0a2488dd8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.284210075Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184222024-04-17T17:21:47.680Z{"msg":"Crucible stats registered with UUID: fa4c48fb-3bcd-4808-ad47-28e0a2488dd8","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.284249097Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184232024-04-17T17:21:47.680Z{"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.284262698Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859}
184242024-04-17T17:21:47.680Z{"msg":"fa4c48fb-3bcd-4808-ad47-28e0a2488dd8 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.284342603Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184252024-04-17T17:21:47.680Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.284367965Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"client":"0","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184262024-04-17T17:21:47.680Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.284383815Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"client":"0","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184272024-04-17T17:21:47.680Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.284402896Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"client":"1","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184282024-04-17T17:21:47.680Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.28446746Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"client":"1","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184292024-04-17T17:21:47.680Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.284496512Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"client":"2","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184302024-04-17T17:21:47.680Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.284528644Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"client":"2","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184312024-04-17T17:21:47.680Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.284551185Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"":"io task","client":"2","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184322024-04-17T17:21:47.680Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.284580837Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"":"io task","client":"0","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184332024-04-17T17:21:47.680Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.284694954Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"":"io task","client":"1","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184342024-04-17T17:21:47.680Z{"msg":"ds_connection connected from Ok(127.0.0.1:32900)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.284764158Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"":"io task","client":"1","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184352024-04-17T17:21:47.681Z{"msg":"ds_connection connected from Ok(127.0.0.1:36367)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.284811631Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"":"io task","client":"2","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184362024-04-17T17:21:47.681Z{"msg":"ds_connection connected from Ok(127.0.0.1:49163)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.284847054Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"":"io task","client":"0","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184372024-04-17T17:21:47.681Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.285598489Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"client":"0","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184382024-04-17T17:21:47.681Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.285763769Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"client":"0","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184392024-04-17T17:21:47.681Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.285812102Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"client":"2","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184402024-04-17T17:21:47.681Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.285848184Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"client":"2","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184412024-04-17T17:21:47.681Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.285878476Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"client":"1","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184422024-04-17T17:21:47.681Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.285916868Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"client":"1","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184432024-04-17T17:21:47.681Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.286202546Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"client":"2","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184442024-04-17T17:21:47.681Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.286251089Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"client":"1","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184452024-04-17T17:21:47.681Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.28628271Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"client":"0","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184462024-04-17T17:21:47.681Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.286745389Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"client":"0","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184472024-04-17T17:21:47.681Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.286795172Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"client":"0","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184482024-04-17T17:21:47.681Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.286817273Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184492024-04-17T17:21:47.681Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.286882767Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"client":"1","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184502024-04-17T17:21:47.681Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.286909688Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"client":"1","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184512024-04-17T17:21:47.681Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.286923829Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184522024-04-17T17:21:47.681Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.28694317Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"client":"2","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184532024-04-17T17:21:47.681Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.286956601Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"client":"2","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184542024-04-17T17:21:47.681Z{"msg":"[0]R flush_numbers[0..12]: [299, 295, 299, 299, 299, 297, 299, 299, 299, 297, 283, 291]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.286976843Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"":"downstairs","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184552024-04-17T17:21:47.681Z{"msg":"[0]R generation[0..12]: [170, 167, 170, 170, 170, 168, 170, 170, 170, 168, 158, 166]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.286990933Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"":"downstairs","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184562024-04-17T17:21:47.681Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.287013175Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"":"downstairs","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184572024-04-17T17:21:47.681Z{"msg":"[1]R flush_numbers[0..12]: [299, 295, 299, 299, 299, 297, 299, 299, 299, 297, 283, 291]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.287027916Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"":"downstairs","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184582024-04-17T17:21:47.682Z{"msg":"[1]R generation[0..12]: [170, 167, 170, 170, 170, 168, 170, 170, 170, 168, 158, 166]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.287051887Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"":"downstairs","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184592024-04-17T17:21:47.682Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"Wait for a query_work_queue command to finish before sending IO
184602024-04-17T17:21:47.682Zname":"crucible","level":30,"time":"2024-04-17T17:21:46.287079899Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"":"downstairs","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184612024-04-17T17:21:47.682Z{"msg":"[2]R flush_numbers[0..12]: [299, 295, 299, 299, 299, 297, 299, 299, 299, 297, 283, 291]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.287109461Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"":"downstairs","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184622024-04-17T17:21:47.682Z{"msg":"[2]R generation[0..12]: [170, 167, 170, 170, 170, 168, 170, 170, 170, 168, 158, 166]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.287134622Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"":"downstairs","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184632024-04-17T17:21:47.682Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.287162534Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"":"downstairs","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184642024-04-17T17:21:47.682Z{"msg":"Max found gen is 171","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.287185065Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"":"downstairs","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184652024-04-17T17:21:47.682Z{"msg":"Generation requested: 172 >= found:171","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.287211077Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"":"downstairs","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184662024-04-17T17:21:47.682Z{"msg":"Next flush: 300","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.287233458Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"":"downstairs","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184672024-04-17T17:21:47.682Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.28726002Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"":"downstairs","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184682024-04-17T17:21:47.682Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.287282971Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184692024-04-17T17:21:47.682Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.287308233Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"client":"0","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184702024-04-17T17:21:47.682Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.287331494Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"client":"1","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184712024-04-17T17:21:47.682Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.287358416Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"client":"2","":"downstairs","session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184722024-04-17T17:21:47.682Z{"msg":"All required reconciliation work is completed","v":0Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
184732024-04-17T17:21:47.682Z,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.287395408Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184742024-04-17T17:21:47.682Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.28742645Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184752024-04-17T17:21:47.682Z{"msg":"fa4c48fb-3bcd-4808-ad47-28e0a2488dd8 is now active with session: a2ba94c9-3a5a-4790-8245-d6c2cec454d8","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.287447541Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184762024-04-17T17:21:47.682Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.287474683Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859,"session_id":"a2ba94c9-3a5a-4790-8245-d6c2cec454d8"}
184772024-04-17T17:21:47.682Z{"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:21:46.287488114Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859}
184782024-04-17T17:21:47.686ZLoading write count information from file "/tmp/test_repair_verify.data"
184792024-04-17T17:21:47.686ZRun Repair workload
184802024-04-17T17:21:47.686Z01/30 Write block 93 len 16384 data: 19 15 12 12
184812024-04-17T17:21:47.686Z02/30 Write block 202 len 32768 data: 18 19 21 21 20 23 15 16
184822024-04-17T17:21:47.686Z03/30 Write block 232 len 16384 data: 13 13 14 19
184832024-04-17T17:21:47.686Z04/30 Write block 271 len 8192 data: 17 17
184842024-04-17T17:21:47.687Z05/30 Read block 171 len 36864
184852024-04-17T17:21:47.687Z06/30 Read block 499 len 12288
184862024-04-17T17:21:47.687Z07/30 Flush
184872024-04-17T17:21:47.687Z08/30 Flush
184882024-04-17T17:21:47.690Z09/30 Write block 331 len 36864 data: 11 10 10 12 14 18 20 19 18
184892024-04-17T17:21:47.690Z10/30 Read block 89 len 12288
184902024-04-17T17:21:47.690Z11/30 Write block 566 len 12288 data: 9 10 12
184912024-04-17T17:21:47.690Z12/30 Read block 72 len 24576
184922024-04-17T17:21:47.690Z13/30 Write block 509 len 12288 data: 11 12 11
184932024-04-17T17:21:47.694Z14/30 Write block 151 len 12288 data: 11 9 9
184942024-04-17T17:21:47.694Z15/30 Read block 386 len 16384
184952024-04-17T17:21:47.694Z16/30 Read block 193 len 40960
184962024-04-17T17:21:47.694Z17/30 Read block 458 len 32768
184972024-04-17T17:21:47.697Z18/30 Write block 354 len 40960 data: 8 6 6 7 7 8 9 10 9 8
184982024-04-17T17:21:47.697Z19/30 Read block 543 len 36864
184992024-04-17T17:21:47.697Z20/30 Read block 533 len 24576
185002024-04-17T17:21:47.700Z21/30 Write block 245 len 12288 data: 12 14 15
185012024-04-17T17:21:47.700Z22/30 Read block 104 len 12288
185022024-04-17T17:21:47.700Z23/30 Write block 67 len 40960 data: 16 15 13 13 13 13 10 13 15 14
185032024-04-17T17:21:47.700Z24/30 Read block 469 len 24576
185042024-04-17T17:21:47.704Z25/30 Flush
185052024-04-17T17:21:47.707Z26/30 Write block 537 len 8192 data: 6 8
185062024-04-17T17:21:47.707Z27/30 Write block 339 len 12288 data: 19 17 14
185072024-04-17T17:21:47.707Z28/30 Write block 339 len 20480 data: 20 18 15 15 17
185082024-04-17T17:21:47.707Z29/30 Write block 84 len 20480 data: 15 16 14 20 17
185092024-04-17T17:21:47.711Z30/30 Write block 432 len 32768 data: 16 17 17 17 17 17 16 18
185102024-04-17T17:21:47.711Z----------------------------------------------------------------
185112024-04-17T17:21:47.711Z Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30
185122024-04-17T17:21:47.711ZStates: Active Active Active
185132024-04-17T17:21:47.711ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
185142024-04-17T17:21:47.711Z 1 Acked 1000 Write 4 Done Done Done false
185152024-04-17T17:21:47.711Z 2 Acked 1001 Write 8 Done Done Done false
185162024-04-17T17:21:47.711Z 3 Acked 1002 Write 4 Done Done Done false
185172024-04-17T17:21:47.711Z 4 Acked 1003 Write 2 Done Done Done false
185182024-04-17T17:21:47.711Z 5 Acked 1004 Read 9 Done Sent Done false
185192024-04-17T17:21:47.711Z 6 Acked 1005 Read 3 Done Sent Done false
185202024-04-17T17:21:47.711Z 7 Acked 1006 Flush 0 Done Sent Done false
185212024-04-17T17:21:47.711Z 8 Acked 1007 Flush 0 Done Sent Done false
185222024-04-17T17:21:47.711Z 9 Acked 1008 Write 9 Done Sent Done false
185232024-04-17T17:21:47.711Z 10 Acked 1009 Read 3 Done Sent Done false
185242024-04-17T17:21:47.711Z 11 Acked 1010 Write 3 Done Sent Done false
185252024-04-17T17:21:47.711Z 12 Acked 1011 Read 6 Done Sent Done false
185262024-04-17T17:21:47.711Z 13 Acked 1012 Write 3 Done Sent Done false
185272024-04-17T17:21:47.711Z 14 Acked 1013 Write 3 Done Sent Done false
185282024-04-17T17:21:47.711Z 15 Acked 1014 Read 4 Done Sent Done false
185292024-04-17T17:21:47.712Z 16 Acked 1015 Read 10 Done Sent Done false
185302024-04-17T17:21:47.712Z 17 Acked 1016 Read 8 Done Sent Done false
185312024-04-17T17:21:47.712Z 18 Acked 1017 Write 10 Done Sent Done false
185322024-04-17T17:21:47.712Z 19 Acked 1018 Read 9 Done Sent Done false
185332024-04-17T17:21:47.712Z 20 Acked 1019 Read 6 Done Sent Done false
185342024-04-17T17:21:47.712Z 21 Acked 1020 Write 3 Done Sent Done false
185352024-04-17T17:21:47.712Z 22 Acked 1021 Read 3 Done Sent Done false
185362024-04-17T17:21:47.712Z 23 Acked 1022 Write 10 Done Sent Done false
185372024-04-17T17:21:47.712Z 24 Acked 1023 Read 6 Done Sent Done false
185382024-04-17T17:21:47.712Z 25 Acked 1024 Flush 0 Done Sent Done false
185392024-04-17T17:21:47.712Z 26 Acked 1025 Write 2 Done Sent Done false
185402024-04-17T17:21:47.712Z 27 Acked 1026 Write 3 Done Sent Done false
185412024-04-17T17:21:47.712Z 28 Acked 1027 Write 5 Done Sent Done false
185422024-04-17T17:21:47.712Z 29 Acked 1028 Write 5 Done Sent Done false
185432024-04-17T17:21:47.712Z 30 Acked 1029 Write 8 Sent Sent Sent false
185442024-04-17T17:21:47.712Z STATES DS:0 DS:1 DS:2 TOTAL
185452024-04-17T17:21:47.712Z New 0 0 0 0
185462024-04-17T17:21:47.712Z Sent 1 26 1 28
185472024-04-17T17:21:47.712Z Done 29 4 29 62
185482024-04-17T17:21:47.712Z Skipped 0 0 0 0
185492024-04-17T17:21:47.712Z Error 0 0 0 0
185502024-04-17T17:21:47.712ZLast Flush: 1024 0 1024
185512024-04-17T17:21:47.712ZDownstairs last five completed:
185522024-04-17T17:21:47.712ZUpstairs last five completed: 30 29 28 27 26
185532024-04-17T17:21:47.713Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:21:46.319056117Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1859}
185542024-04-17T17:21:47.713ZWrote out file "/tmp/test_repair_verify.data"
185552024-04-17T17:21:47.716Z
185562024-04-17T17:21:47.716ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
185572024-04-17T17:21:47.733ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
185582024-04-17T17:21:47.733Z 0 000-019 170 170 170 299 299 299 F F F
185592024-04-17T17:21:47.733Z 1 020-039 167 167 167 295 295 295 F F F
185602024-04-17T17:21:47.733Z 2 040-059 170 170 170 299 299 299 F F F
185612024-04-17T17:21:47.733Z 3 060-079 172 170 172 302 299 302 F F F <---
185622024-04-17T17:21:47.733Z 4 080-099 172 170 172 300 299 300 T T T <---
185632024-04-17T17:21:47.733Z 5 100-119 168 168 168 297 297 297 F F F
185642024-04-17T17:21:47.737Z 6 120-139 170 170 170 299 299 299 F F F
185652024-04-17T17:21:47.737Z 7 140-159 172 170 172 302 299 302 F F F <---
185662024-04-17T17:21:47.737Z 8 160-179 170 170 170 299 299 299 F F F
185672024-04-17T17:21:47.737Z 9 180-199 168 168 168 297 297 297 F F F
185682024-04-17T17:21:47.737Z 10 200-219 172 158 172 300 283 300 F T F <---
185692024-04-17T17:21:47.737Z 11 220-239 172 166 172 300 291 300 F T F <---
185702024-04-17T17:21:47.737Z 12 240-259 172 158 172 302 283 302 F F F <---
185712024-04-17T17:21:47.737Z 13 260-279 172 170 172 300 299 300 F T F <---
185722024-04-17T17:21:47.737Z 14 280-299 164 164 164 289 289 289 F F F
185732024-04-17T17:21:47.737Z 15 300-319 167 167 167 295 295 295 F F F
185742024-04-17T17:21:47.738Z 16 320-339 172 170 172 302 299 302 T F T <---
185752024-04-17T17:21:47.738Z 17 340-359 172 164 172 302 289 302 T F T <---
185762024-04-17T17:21:47.738Z 18 360-379 172 169 172 302 298 302 F F F <---
185772024-04-17T17:21:47.738Z 19 380-399 169 169 169 298 298 298 F F F
185782024-04-17T17:21:47.738Z 20 400-419 170 170 170 299 299 299 F F F
185792024-04-17T17:21:47.738Z 21 420-439 170 170 170 299 299 299 T F T <---
185802024-04-17T17:21:47.738Z 22 440-459 169 169 169 298 298 298 F F F
185812024-04-17T17:21:47.738Z 23 460-479 166 166 166 291 291 291 F F F
185822024-04-17T17:21:47.738Z 24 480-499 170 170 170 299 299 299 F F F
185832024-04-17T17:21:47.738Z 25 500-519 172 170 172 302 299 302 F F F <---
185842024-04-17T17:21:47.738Z 26 520-539 170 170 170 299 299 299 T F T <---
185852024-04-17T17:21:47.738Z 27 540-559 170 170 170 299 299 299 F F F
185862024-04-17T17:21:47.738Z 28 560-579 172 170 172 302 299 302 F F F <---
185872024-04-17T17:21:47.738Z 29 580-599 163 163 163 288 288 288 F F F
185882024-04-17T17:21:47.738ZMax gen: 172, Max flush: 302
185892024-04-17T17:21:47.738ZError: Difference in extent metadata found!
185902024-04-17T17:21:47.738ZA Difference in extent metadata is expected here
185912024-04-17T17:21:47.738ZOn loop 86
185922024-04-17T17:21:47.738Z
185932024-04-17T17:21:47.742ZVerifying data now
185942024-04-17T17:22:02.378ZNo change after verify
185952024-04-17T17:22:02.378ZLoop: 86 Downstairs dump after verify (and repair):
185962024-04-17T17:22:02.392ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
185972024-04-17T17:22:02.392Z 0 000-019 170 170 170 299 299 299 F F F
185982024-04-17T17:22:02.396Z 1 020-039 167 167 167 295 295 295 F F F
185992024-04-17T17:22:02.397Z 2 040-059 170 170 170 299 299 299 F F F
186002024-04-17T17:22:02.397Z 3 060-079 172 172 172 302 302 302 F F F
186012024-04-17T17:22:02.397Z 4 080-099 173 173 173 303 303 303 F F F
186022024-04-17T17:22:02.397Z 5 100-119 168 168 168 297 297 297 F F F
186032024-04-17T17:22:02.397Z 6 120-139 170 170 170 299 299 299 F F F
186042024-04-17T17:22:02.397Z 7 140-159 172 172 172 302 302 302 F F F
186052024-04-17T17:22:02.397Z 8 160-179 170 170 170 299 299 299 F F F
186062024-04-17T17:22:02.397Z 9 180-199 168 168 168 297 297 297 F F F
186072024-04-17T17:22:02.397Z 10 200-219 172 172 172 300 300 300 F F F
186082024-04-17T17:22:02.398Z 11 220-239 172 172 172 300 300 300 F F F
186092024-04-17T17:22:02.398Z 12 240-259 172 172 172 302 302 302 F F F
186102024-04-17T17:22:02.398Z 13 260-279 172 172 172 300 300 300 F F F
186112024-04-17T17:22:02.398Z 14 280-299 164 164 164 289 289 289 F F F
186122024-04-17T17:22:02.398Z 15 300-319 167 167 167 295 295 295 F F F
186132024-04-17T17:22:02.398Z 16 320-339 173 173 173 303 303 303 F F F
186142024-04-17T17:22:02.398Z 17 340-359 173 173 173 303 303 303 F F F
186152024-04-17T17:22:02.398Z 18 360-379 172 172 172 302 302 302 F F F
186162024-04-17T17:22:02.399Z 19 380-399 169 169 169 298 298 298 F F F
186172024-04-17T17:22:02.399Z 20 400-419 170 170 170 299 299 299 F F F
186182024-04-17T17:22:02.399Z 21 420-439 173 173 173 303 303 303 F F F
186192024-04-17T17:22:02.399Z 22 440-459 169 169 169 298 298 298 F F F
186202024-04-17T17:22:02.399Z 23 460-479 166 166 166 291 291 291 F F F
186212024-04-17T17:22:02.399Z 24 480-499 170 170 170 299 299 299 F F F
186222024-04-17T17:22:02.399Z 25 500-519 172 172 172 302 302 302 F F F
186232024-04-17T17:22:02.399Z 26 520-539 173 173 173 303 303 303 F F F
186242024-04-17T17:22:02.399Z 27 540-559 170 170 170 299 299 299 F F F
186252024-04-17T17:22:02.399Z 28 560-579 172 172 172 302 302 302 F F F
186262024-04-17T17:22:02.399Z 29 580-599 163 163 163 288 288 288 F F F
186272024-04-17T17:22:02.400ZMax gen: 173, Max flush: 303
186282024-04-17T17:22:02.400Z
186292024-04-17T17:22:02.400ZBegin loop 87
186302024-04-17T17:22:02.400ZDownstairs to restart: 2
186312024-04-17T17:22:02.415Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.023941166Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867}
186322024-04-17T17:22:02.415Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.024236874Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867}
186332024-04-17T17:22:02.415Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.024272556Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867}
186342024-04-17T17:22:02.423Z{"msg":"Crucible 3575e9d1-9734-46c8-b7b7-798199010b0f has session id: 580ef38c-202a-4042-99e5-ddff3c9032ba","v":0,"name":"crucible","level":30Crucible runtime is spawned
186352024-04-17T17:22:02.423Z,"time":"2024-04-17T17:22:01.024295558Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186362024-04-17T17:22:02.423Z{"msg":"Upstairs opts: Upstairs UUID: 3575e9d1-9734-46c8-b7b7-798199010b0f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.024352391Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186372024-04-17T17:22:02.423Z{"msg":"Crucible stats registered with UUID: 3575e9d1-9734-46c8-b7b7-798199010b0f","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.024390524Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186382024-04-17T17:22:02.423Z{"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.024413735Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867}
186392024-04-17T17:22:02.423Z{"msg":"3575e9d1-9734-46c8-b7b7-798199010b0f active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.024445697Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186402024-04-17T17:22:02.423Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.024480179Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"client":"0","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186412024-04-17T17:22:02.423Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.024505411Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"client":"0","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186422024-04-17T17:22:02.423Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.024540693Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"client":"1","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186432024-04-17T17:22:02.424Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.024581285Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"client":"1","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186442024-04-17T17:22:02.424Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.024608997Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"client":"2","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186452024-04-17T17:22:02.424Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.024645649Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"client":"2","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186462024-04-17T17:22:02.424Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.024680261Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"":"io task","client":"0","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186472024-04-17T17:22:02.424Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.024717403Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"":"io task","client":"1","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186482024-04-17T17:22:02.424Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.024752046Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"":"io task","client":"2","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186492024-04-17T17:22:02.424Z{"msg":"ds_connection connected from Ok(127.0.0.1:52215)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.024787217Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"":"io task","client":"0","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186502024-04-17T17:22:02.424Z{"msg":"ds_connection connected from Ok(127.0.0.1:50596)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.02482289Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"":"io task","client":"2","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186512024-04-17T17:22:02.424Z{"msg":"ds_connection connected from Ok(127.0.0.1:59028)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.024858632Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"":"io task","client":"1","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186522024-04-17T17:22:02.425Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.025542533Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"client":"0","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186532024-04-17T17:22:02.425Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.025585636Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"client":"0","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186542024-04-17T17:22:02.425Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.025621798Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"client":"1","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186552024-04-17T17:22:02.425Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.02564828Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"client":"1","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186562024-04-17T17:22:02.425Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.02581345Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"client":"2","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186572024-04-17T17:22:02.425Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.025860413Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"client":"2","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186582024-04-17T17:22:02.425Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.026238285Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"client":"0","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186592024-04-17T17:22:02.425Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.026296219Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"client":"1","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186602024-04-17T17:22:02.425Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.026335271Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"client":"2","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186612024-04-17T17:22:02.426Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.026827741Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"client":"0","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186622024-04-17T17:22:02.426Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.026865043Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"client":"0","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186632024-04-17T17:22:02.426Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.026881444Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186642024-04-17T17:22:02.426Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.026901066Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"client":"2","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186652024-04-17T17:22:02.426Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.026914797Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"client":"2","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186662024-04-17T17:22:02.426Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.026934798Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186672024-04-17T17:22:02.426Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.026947108Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"client":"1","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186682024-04-17T17:22:02.426Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.02696778Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"client":"1","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186692024-04-17T17:22:02.427Z{"msg":"[0]R flush_numbers[0..12]: [299, 295, 299, 302, 303, 297, 299, 302, 299, 297, 300, 300]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.026982061Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"":"downstairs","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186702024-04-17T17:22:02.427Z{"msg":"[0]R generation[0..12]: [170, 167, 170, 172, 173, 168, 170, 172, 170, 168, 172, 172]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
186712024-04-17T17:22:02.427Z2024-04-17T17:22:01.027017443Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"":"downstairs","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186722024-04-17T17:22:02.427Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.027037094Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"":"downstairs","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186732024-04-17T17:22:02.427Z{"msg":"[1]R flush_numbers[0..12]: [299, 295, 299, 302, 303, 297, 299, 302, 299, 297, 300, 300]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.027051655Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"":"downstairs","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186742024-04-17T17:22:02.427Z{"msg":"[1]R generation[0..12]: [170, 167, 170, 172, 173, 168, 170, 172, 170, 168, 172, 172]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.027073276Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"":"downstairs","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186752024-04-17T17:22:02.427Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.027087917Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"":"downstairs","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186762024-04-17T17:22:02.427Z{"msg":"[2]R flush_numbers[0..12]: [299, 295, 299, 302, 303, 297, 299, 302, 299, 297, 300, 300]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.027102218Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"":"downstairs","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186772024-04-17T17:22:02.428Z{"msg":"[2]R generation[0..12]: [170, 167, 170, 172, 173, 168, 170, 172, 170, 168, 172, 172]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.027123039Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"":"downstairs","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186782024-04-17T17:22:02.428Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.02713753Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"":"downstairs","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186792024-04-17T17:22:02.428Z{"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.027157251Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"":"downstairs","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186802024-04-17T17:22:02.428Z{"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.027170632Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"":"downstairs","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186812024-04-17T17:22:02.428Z{"msg":"Next flush: 304","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.027190523Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"":"downstairs","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186822024-04-17T17:22:02.428Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.027203824Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"":"downstairs","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186832024-04-17T17:22:02.428Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.027223845Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186842024-04-17T17:22:02.428Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.027235906Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"client":"0","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186852024-04-17T17:22:02.429Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.027255747Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"client":"1","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186862024-04-17T17:22:02.429Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.027269218Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"client":"2","":"downstairs","session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186872024-04-17T17:22:02.429Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.027289069Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186882024-04-17T17:22:02.429Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.02730148Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186892024-04-17T17:22:02.429ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
186902024-04-17T17:22:02.429Z{"msg":"3575e9d1-9734-46c8-b7b7-798199010b0f is now active with session: 580ef38c-202a-4042-99e5-ddff3c9032ba","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.027325101Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186912024-04-17T17:22:02.429Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.027337212Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867,"session_id":"580ef38c-202a-4042-99e5-ddff3c9032ba"}
186922024-04-17T17:22:02.429Z{"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:01.027356263Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867}
186932024-04-17T17:22:02.429ZLoading write count information from file "/tmp/test_repair_verify.data"
186942024-04-17T17:22:02.429ZRun Repair workload
186952024-04-17T17:22:02.430Z01/30 Write block 45 len 36864 data: 13 12 11 9 12 10 12 12 13
186962024-04-17T17:22:02.435Z02/30 Write block 108 len 36864 data: 13 13 13 15 15 14 12 13 13
186972024-04-17T17:22:02.435Z03/30 Read block 54 len 4096
186982024-04-17T17:22:02.435Z04/30 Read block 0 len 12288
186992024-04-17T17:22:02.436Z05/30 Read block 323 len 36864
187002024-04-17T17:22:02.436Z06/30 Read block 543 len 36864
187012024-04-17T17:22:02.436Z07/30 Read block 324 len 36864
187022024-04-17T17:22:02.436Z08/30 Write block 85 len 20480 data: 17 15 21 18 17
187032024-04-17T17:22:02.436Z09/30 Flush
187042024-04-17T17:22:02.436Z10/30 Write block 487 len 28672 data: 15 15 14 14 11 12 10
187052024-04-17T17:22:02.436Z11/30 Read block 128 len 12288
187062024-04-17T17:22:02.437Z12/30 Read block 467 len 4096
187072024-04-17T17:22:02.437Z13/30 Read block 401 len 36864
187082024-04-17T17:22:02.437Z14/30 Read block 260 len 24576
187092024-04-17T17:22:02.437Z15/30 Read block 120 len 12288
187102024-04-17T17:22:02.437Z16/30 Write block 151 len 20480 data: 12 10 10 12 12
187112024-04-17T17:22:02.437Z17/30 Read block 510 len 40960
187122024-04-17T17:22:02.442Z18/30 Write block 570 len 28672 data: 10 10 12 13 17 18 15
187132024-04-17T17:22:02.442Z19/30 Read block 184 len 36864
187142024-04-17T17:22:02.442Z20/30 Flush
187152024-04-17T17:22:02.442Z21/30 Write block 415 len 12288 data: 14 16 14
187162024-04-17T17:22:02.442Z22/30 Write block 64 len 12288 data: 18 17 16
187172024-04-17T17:22:02.442Z23/30 Write block 165 len 12288 data: 12 16 17
187182024-04-17T17:22:02.448Z24/30 Write block 37 len 4096 data: 15
187192024-04-17T17:22:02.448Z25/30 Flush
187202024-04-17T17:22:02.448Z26/30 Write block 347 len 12288 data: 16 14 14
187212024-04-17T17:22:02.449Z27/30 Write block 290 len 28672 data: 19 19 16 19 17 13 10
187222024-04-17T17:22:02.453Z28/30 Write block 313 len 32768 data: 16 18 18 16 16 17 15 13
187232024-04-17T17:22:02.454Z29/30 Read block 189 len 36864
187242024-04-17T17:22:02.454Z30/30 Write block 57 len 20480 data: 17 19 18 18 17
187252024-04-17T17:22:02.454Z----------------------------------------------------------------
187262024-04-17T17:22:02.454Z Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:30
187272024-04-17T17:22:02.454ZStates: Active Active Active
187282024-04-17T17:22:02.454ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
187292024-04-17T17:22:02.454Z 1 Acked 1000 Write 9 Done Done Done false
187302024-04-17T17:22:02.454Z 2 Acked 1001 Write 9 Done Done Done false
187312024-04-17T17:22:02.454Z 3 Acked 1002 Read 1 Done Done Done false
187322024-04-17T17:22:02.454Z 4 Acked 1003 Read 3 Done Done Sent false
187332024-04-17T17:22:02.455Z 5 Acked 1004 Read 9 Done Done Sent false
187342024-04-17T17:22:02.455Z 6 Acked 1005 Read 9 Done Done Sent false
187352024-04-17T17:22:02.455Z 7 Acked 1006 Read 9 Done Done Sent false
187362024-04-17T17:22:02.455Z 8 Acked 1007 Write 5 Done Done Sent false
187372024-04-17T17:22:02.455Z 9 Acked 1008 Flush 0 Done Done Sent false
187382024-04-17T17:22:02.455Z 10 Acked 1009 Write 7 Done Done Sent false
187392024-04-17T17:22:02.455Z 11 Acked 1010 Read 3 Done Done Sent false
187402024-04-17T17:22:02.455Z 12 Acked 1011 Read 1 Done Done Sent false
187412024-04-17T17:22:02.455Z 13 Acked 1012 Read 9 Done Done Sent false
187422024-04-17T17:22:02.456Z 14 Acked 1013 Read 6 Done Done Sent false
187432024-04-17T17:22:02.456Z 15 Acked 1014 Read 3 Done Done Sent false
187442024-04-17T17:22:02.456Z 16 Acked 1015 Write 5 Done Done Sent false
187452024-04-17T17:22:02.456Z 17 Acked 1016 Read 10 Done Done Sent false
187462024-04-17T17:22:02.456Z 18 Acked 1017 Write 7 Done Done Sent false
187472024-04-17T17:22:02.456Z 19 Acked 1018 Read 9 Done Done Sent false
187482024-04-17T17:22:02.456Z 20 Acked 1019 Flush 0 Done Done Sent false
187492024-04-17T17:22:02.456Z 21 Acked 1020 Write 3 Done Done Sent false
187502024-04-17T17:22:02.456Z 22 Acked 1021 Write 3 Done Done Sent false
187512024-04-17T17:22:02.456Z 23 Acked 1022 Write 3 Done Done Sent false
187522024-04-17T17:22:02.456Z 24 Acked 1023 Write 1 Done Done Sent false
187532024-04-17T17:22:02.457Z 25 Acked 1024 Flush 0 Done Done Sent false
187542024-04-17T17:22:02.457Z 26 Acked 1025 Write 3 Done Done Sent false
187552024-04-17T17:22:02.457Z 27 Acked 1026 Write 7 Done Done Sent false
187562024-04-17T17:22:02.457Z 28 Acked 1027 Write 8 Done Done Sent false
187572024-04-17T17:22:02.457Z 29 Acked 1028 Read 9 Done Done Sent false
187582024-04-17T17:22:02.457Z 30 Acked 1029 Write 5 Sent Sent Sent false
187592024-04-17T17:22:02.457Z STATES DS:0 DS:1 DS:2 TOTAL
187602024-04-17T17:22:02.457Z New 0 0 0 0
187612024-04-17T17:22:02.457Z Sent 1 1 27 29
187622024-04-17T17:22:02.457Z Done 29 29 3 61
187632024-04-17T17:22:02.457Z Skipped 0 0 0 0
187642024-04-17T17:22:02.457Z Error 0 0 0 0
187652024-04-17T17:22:02.458ZLast Flush: 1024 1024 0
187662024-04-17T17:22:02.458ZDownstairs last five completed:
187672024-04-17T17:22:02.458ZUpstairs last five completed: 30 29 28 27 26
187682024-04-17T17:22:02.458Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:22:01.059922577Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1867}
187692024-04-17T17:22:02.458ZWrote out file "/tmp/test_repair_verify.data"
187702024-04-17T17:22:02.465Z
187712024-04-17T17:22:02.465ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
187722024-04-17T17:22:02.475ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
187732024-04-17T17:22:02.475Z 0 000-019 170 170 170 299 299 299 F F F
187742024-04-17T17:22:02.476Z 1 020-039 174 174 167 306 306 295 F F F <---
187752024-04-17T17:22:02.482Z 2 040-059 174 174 170 304 304 299 T T T <---
187762024-04-17T17:22:02.482Z 3 060-079 174 174 172 306 306 302 T T F <---
187772024-04-17T17:22:02.482Z 4 080-099 174 174 173 304 304 303 F F F <---
187782024-04-17T17:22:02.482Z 5 100-119 174 174 168 304 304 297 F F T <---
187792024-04-17T17:22:02.483Z 6 120-139 170 170 170 299 299 299 F F F
187802024-04-17T17:22:02.483Z 7 140-159 174 174 172 305 305 302 F F F <---
187812024-04-17T17:22:02.483Z 8 160-179 174 174 170 306 306 299 F F F <---
187822024-04-17T17:22:02.483Z 9 180-199 168 168 168 297 297 297 F F F
187832024-04-17T17:22:02.483Z 10 200-219 172 172 172 300 300 300 F F F
187842024-04-17T17:22:02.483Z 11 220-239 172 172 172 300 300 300 F F F
187852024-04-17T17:22:02.483Z 12 240-259 172 172 172 302 302 302 F F F
187862024-04-17T17:22:02.484Z 13 260-279 172 172 172 300 300 300 F F F
187872024-04-17T17:22:02.484Z 14 280-299 164 164 164 289 289 289 T T F <---
187882024-04-17T17:22:02.484Z 15 300-319 167 167 167 295 295 295 T T F <---
187892024-04-17T17:22:02.484Z 16 320-339 173 173 173 303 303 303 T T F <---
187902024-04-17T17:22:02.484Z 17 340-359 173 173 173 303 303 303 T T F <---
187912024-04-17T17:22:02.484Z 18 360-379 172 172 172 302 302 302 F F F
187922024-04-17T17:22:02.484Z 19 380-399 169 169 169 298 298 298 F F F
187932024-04-17T17:22:02.484Z 20 400-419 174 174 170 306 306 299 F F F <---
187942024-04-17T17:22:02.484Z 21 420-439 173 173 173 303 303 303 F F F
187952024-04-17T17:22:02.484Z 22 440-459 169 169 169 298 298 298 F F F
187962024-04-17T17:22:02.485Z 23 460-479 166 166 166 291 291 291 F F F
187972024-04-17T17:22:02.485Z 24 480-499 174 174 170 305 305 299 F F F <---
187982024-04-17T17:22:02.485Z 25 500-519 172 172 172 302 302 302 F F F
187992024-04-17T17:22:02.485Z 26 520-539 173 173 173 303 303 303 F F F
188002024-04-17T17:22:02.485Z 27 540-559 170 170 170 299 299 299 F F F
188012024-04-17T17:22:02.485Z 28 560-579 174 174 172 305 305 302 F F F <---
188022024-04-17T17:22:02.485Z 29 580-599 163 163 163 288 288 288 F F F
188032024-04-17T17:22:02.485ZMax gen: 174, Max flush: 306
188042024-04-17T17:22:02.485ZError: Difference in extent metadata found!
188052024-04-17T17:22:02.486ZA Difference in extent metadata is expected here
188062024-04-17T17:22:02.486ZOn loop 87
188072024-04-17T17:22:02.486Z
188082024-04-17T17:22:02.486ZVerifying data now
188092024-04-17T17:22:17.117ZNo change after verify
188102024-04-17T17:22:17.117ZLoop: 87 Downstairs dump after verify (and repair):
188112024-04-17T17:22:17.131ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
188122024-04-17T17:22:17.132Z 0 000-019 170 170 170 299 299 299 F F F
188132024-04-17T17:22:17.142Z 1 020-039 174 174 174 306 306 306 F F F
188142024-04-17T17:22:17.142Z 2 040-059 175 175 175 307 307 307 F F F
188152024-04-17T17:22:17.142Z 3 060-079 175 175 175 307 307 307 F F F
188162024-04-17T17:22:17.142Z 4 080-099 174 174 174 304 304 304 F F F
188172024-04-17T17:22:17.143Z 5 100-119 174 174 174 304 304 304 F F F
188182024-04-17T17:22:17.143Z 6 120-139 170 170 170 299 299 299 F F F
188192024-04-17T17:22:17.143Z 7 140-159 174 174 174 305 305 305 F F F
188202024-04-17T17:22:17.143Z 8 160-179 174 174 174 306 306 306 F F F
188212024-04-17T17:22:17.143Z 9 180-199 168 168 168 297 297 297 F F F
188222024-04-17T17:22:17.143Z 10 200-219 172 172 172 300 300 300 F F F
188232024-04-17T17:22:17.143Z 11 220-239 172 172 172 300 300 300 F F F
188242024-04-17T17:22:17.143Z 12 240-259 172 172 172 302 302 302 F F F
188252024-04-17T17:22:17.143Z 13 260-279 172 172 172 300 300 300 F F F
188262024-04-17T17:22:17.143Z 14 280-299 175 175 175 307 307 307 F F F
188272024-04-17T17:22:17.144Z 15 300-319 175 175 175 307 307 307 F F F
188282024-04-17T17:22:17.144Z 16 320-339 175 175 175 307 307 307 F F F
188292024-04-17T17:22:17.144Z 17 340-359 175 175 175 307 307 307 F F F
188302024-04-17T17:22:17.144Z 18 360-379 172 172 172 302 302 302 F F F
188312024-04-17T17:22:17.144Z 19 380-399 169 169 169 298 298 298 F F F
188322024-04-17T17:22:17.144Z 20 400-419 174 174 174 306 306 306 F F F
188332024-04-17T17:22:17.144Z 21 420-439 173 173 173 303 303 303 F F F
188342024-04-17T17:22:17.144Z 22 440-459 169 169 169 298 298 298 F F F
188352024-04-17T17:22:17.144Z 23 460-479 166 166 166 291 291 291 F F F
188362024-04-17T17:22:17.145Z 24 480-499 174 174 174 305 305 305 F F F
188372024-04-17T17:22:17.145Z 25 500-519 172 172 172 302 302 302 F F F
188382024-04-17T17:22:17.145Z 26 520-539 173 173 173 303 303 303 F F F
188392024-04-17T17:22:17.145Z 27 540-559 170 170 170 299 299 299 F F F
188402024-04-17T17:22:17.145Z 28 560-579 174 174 174 305 305 305 F F F
188412024-04-17T17:22:17.145Z 29 580-599 163 163 163 288 288 288 F F F
188422024-04-17T17:22:17.145ZMax gen: 175, Max flush: 307
188432024-04-17T17:22:17.145Z
188442024-04-17T17:22:17.145ZBegin loop 88
188452024-04-17T17:22:17.145ZDownstairs to restart: 1
188462024-04-17T17:22:17.155Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.764566265Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875}
188472024-04-17T17:22:17.155Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.764861143Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875}
188482024-04-17T17:22:17.155Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name"Crucible runtime is spawned
188492024-04-17T17:22:17.162Z:"crucible","level":30,"time":"2024-04-17T17:22:15.764910646Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875}
188502024-04-17T17:22:17.162Z{"msg":"Crucible 0d5beb50-ef62-4320-aad0-9fb55c2ac5d8 has session id: ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.764955128Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188512024-04-17T17:22:17.162Z{"msg":"Upstairs opts: Upstairs UUID: 0d5beb50-ef62-4320-aad0-9fb55c2ac5d8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.765007402Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188522024-04-17T17:22:17.162Z{"msg":"Crucible stats registered with UUID: 0d5beb50-ef62-4320-aad0-9fb55c2ac5d8","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.765058385Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188532024-04-17T17:22:17.162Z{"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.765096567Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875}
188542024-04-17T17:22:17.163Z{"msg":"0d5beb50-ef62-4320-aad0-9fb55c2ac5d8 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.765118958Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188552024-04-17T17:22:17.163Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.76514884Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"client":"0","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188562024-04-17T17:22:17.163Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.765172732Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"client":"0","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188572024-04-17T17:22:17.163Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.765205984Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"client":"1","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188582024-04-17T17:22:17.163Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.765232255Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"client":"1","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188592024-04-17T17:22:17.163Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.765266017Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"client":"2","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188602024-04-17T17:22:17.163Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.765352103Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"client":"2","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188612024-04-17T17:22:17.163Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.765396735Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"":"io task","client":"0","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188622024-04-17T17:22:17.163Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.765451969Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"":"io task","client":"1","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188632024-04-17T17:22:17.163Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.765503022Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"":"io task","client":"2","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188642024-04-17T17:22:17.163Z{"msg":"ds_connection connected from Ok(127.0.0.1:44137)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.76564326Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"":"io task","client":"2","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188652024-04-17T17:22:17.164Z{"msg":"ds_connection connected from Ok(127.0.0.1:58767)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.765691623Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"":"io task","client":"0","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188662024-04-17T17:22:17.164Z{"msg":"ds_connection connected from Ok(127.0.0.1:42275)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.765718465Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"":"io task","client":"1","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188672024-04-17T17:22:17.164Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.766401146Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"client":"0","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188682024-04-17T17:22:17.164Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.766436068Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"client":"0","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188692024-04-17T17:22:17.164Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.766451889Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"client":"2","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188702024-04-17T17:22:17.164Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.766476691Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"client":"2","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188712024-04-17T17:22:17.164Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.766512803Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"client":"1","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188722024-04-17T17:22:17.164Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.766540564Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"client":"1","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188732024-04-17T17:22:17.164Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.767033314Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"client":"2","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188742024-04-17T17:22:17.164Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.767069817Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"client":"1","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188752024-04-17T17:22:17.164Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.767137011Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"client":"0","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188762024-04-17T17:22:17.164Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.76761222Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"client":"0","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188772024-04-17T17:22:17.164Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.767666373Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"client":"0","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188782024-04-17T17:22:17.164Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.767692725Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188792024-04-17T17:22:17.164Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.767722776Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"client":"2","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188802024-04-17T17:22:17.164Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.767747688Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"client":"2","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188812024-04-17T17:22:17.164Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.76777539Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188822024-04-17T17:22:17.164Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.767796961Z","hostname":"Wait for a query_work_queue command to finish before sending IO
188832024-04-17T17:22:17.165Zw-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"client":"1","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188842024-04-17T17:22:17.165Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.767831763Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"client":"1","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188852024-04-17T17:22:17.165Z{"msg":"[0]R flush_numbers[0..12]: [299, 306, 307, 307, 304, 304, 299, 305, 306, 297, 300, 300]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.767856344Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"":"downstairs","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188862024-04-17T17:22:17.165Z{"msg":"[0]R generation[0..12]: [170, 174, 175, 175, 174, 174, 170, 174, 174, 168, 172, 172]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.767888386Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"":"downstairs","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188872024-04-17T17:22:17.165Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.767914998Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"":"downstairs","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188882024-04-17T17:22:17.165Z{"msg":"[1]R flush_numbers[0..12]: [299, 306, 307, 307, 304, 304, 299, 305, 306, 297, 300, 300]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.76794504Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"":"downstairs","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188892024-04-17T17:22:17.165Z{"msg":"[1]R generation[0..12]: [170, 174, 175, 175, 174, 174, 170, 174, 174, 168, 172, 172]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.767970431Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"":"downstairs","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188902024-04-17T17:22:17.165Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.768000703Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
188912024-04-17T17:22:17.165Zw-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"":"downstairs","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188922024-04-17T17:22:17.165Z{"msg":"[2]R flush_numbers[0..12]: [299, 306, 307, 307, 304, 304, 299, 305, 306, 297, 300, 300]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.768029925Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"":"downstairs","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188932024-04-17T17:22:17.165Z{"msg":"[2]R generation[0..12]: [170, 174, 175, 175, 174, 174, 170, 174, 174, 168, 172, 172]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.768060357Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"":"downstairs","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188942024-04-17T17:22:17.165Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.768093439Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"":"downstairs","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188952024-04-17T17:22:17.165Z{"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.76811614Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"":"downstairs","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188962024-04-17T17:22:17.165Z{"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.768147372Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"":"downstairs","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188972024-04-17T17:22:17.165Z{"msg":"Next flush: 308","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.768169893Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"":"downstairs","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188982024-04-17T17:22:17.165Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.768196295Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"":"downstairs","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
188992024-04-17T17:22:17.165Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.768219276Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
189002024-04-17T17:22:17.165Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.768244468Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"client":"0","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
189012024-04-17T17:22:17.165Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.768267379Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"client":"1","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
189022024-04-17T17:22:17.165Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.768293261Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"client":"2","":"downstairs","session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
189032024-04-17T17:22:17.165Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.768316102Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
189042024-04-17T17:22:17.165Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.768341744Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
189052024-04-17T17:22:17.165Z{"msg":"0d5beb50-ef62-4320-aad0-9fb55c2ac5d8 is now active with session: ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.768364385Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
189062024-04-17T17:22:17.166Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.768389487Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875,"session_id":"ac1f07b5-2cd9-41ef-b1c5-e0eb5242ffb1"}
189072024-04-17T17:22:17.166Z{"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:15.768410568Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875}
189082024-04-17T17:22:17.166ZLoading write count information from file "/tmp/test_repair_verify.data"
189092024-04-17T17:22:17.166ZRun Repair workload
189102024-04-17T17:22:17.166Z01/30 Write block 438 len 12288 data: 17 19 15
189112024-04-17T17:22:17.170Z02/30 Read block 41 len 28672
189122024-04-17T17:22:17.170Z03/30 Flush
189132024-04-17T17:22:17.170Z04/30 Write block 181 len 16384 data: 13 14 12 12
189142024-04-17T17:22:17.170Z05/30 Flush
189152024-04-17T17:22:17.170Z06/30 Write block 362 len 16384 data: 10 9 11 11
189162024-04-17T17:22:17.170Z07/30 Flush
189172024-04-17T17:22:17.170Z08/30 Write block 263 len 36864 data: 13 13 12 12 14 15 20 16 18
189182024-04-17T17:22:17.170Z09/30 Read block 362 len 36864
189192024-04-17T17:22:17.174Z10/30 Read block 18 len 28672
189202024-04-17T17:22:17.174Z11/30 Write block 376 len 24576 data: 8 8 8 10 11 10
189212024-04-17T17:22:17.174Z12/30 Flush
189222024-04-17T17:22:17.178Z13/30 Write block 273 len 16384 data: 20 17 15 14
189232024-04-17T17:22:17.178Z14/30 Read block 588 len 24576
189242024-04-17T17:22:17.178Z15/30 Flush
189252024-04-17T17:22:17.181Z16/30 Write block 273 len 8192 data: 21 18
189262024-04-17T17:22:17.181Z17/30 Read block 364 len 32768
189272024-04-17T17:22:17.182Z18/30 Read block 492 len 40960
189282024-04-17T17:22:17.182Z19/30 Write block 518 len 24576 data: 8 8 10 12 14 14
189292024-04-17T17:22:17.182Z20/30 Read block 173 len 32768
189302024-04-17T17:22:17.185Z21/30 Read block 118 len 20480
189312024-04-17T17:22:17.185Z22/30 Write block 105 len 40960 data: 12 12 12 14 14 14 16 16 15 13
189322024-04-17T17:22:17.186Z23/30 Read block 239 len 40960
189332024-04-17T17:22:17.189Z24/30 Write block 25 len 24576 data: 13 12 8 9 14 13
189342024-04-17T17:22:17.189Z25/30 Read block 20 len 24576
189352024-04-17T17:22:17.189Z26/30 Flush
189362024-04-17T17:22:17.193Z27/30 Write block 397 len 32768 data: 9 12 11 10 11 9 10 11
189372024-04-17T17:22:17.193Z28/30 Write block 494 len 24576 data: 12 13 12 12 14 15
189382024-04-17T17:22:17.202Z29/30 Write block 531 len 24576 data: 13 9 8 6 7 8
189392024-04-17T17:22:17.202Z30/30 Read block 579 len 28672
189402024-04-17T17:22:17.202Z----------------------------------------------------------------
189412024-04-17T17:22:17.202Z Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:30
189422024-04-17T17:22:17.202ZStates: Active Active Active
189432024-04-17T17:22:17.202ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
189442024-04-17T17:22:17.202Z 1 Acked 1000 Write 3 Done Sent Done false
189452024-04-17T17:22:17.202Z 2 Acked 1001 Read 7 Done Sent Done false
189462024-04-17T17:22:17.202Z 3 Acked 1002 Flush 0 Done Sent Done false
189472024-04-17T17:22:17.202Z 4 Acked 1003 Write 4 Done Sent Done false
189482024-04-17T17:22:17.202Z 5 Acked 1004 Flush 0 Done Sent Done false
189492024-04-17T17:22:17.203Z 6 Acked 1005 Write 4 Done Sent Done false
189502024-04-17T17:22:17.203Z 7 Acked 1006 Flush 0 Done Sent Done false
189512024-04-17T17:22:17.203Z 8 Acked 1007 Write 9 Done Sent Done false
189522024-04-17T17:22:17.203Z 9 Acked 1008 Read 9 Done Sent Done false
189532024-04-17T17:22:17.203Z 10 Acked 1009 Read 7 Done Sent Done false
189542024-04-17T17:22:17.203Z 11 Acked 1010 Write 6 Done Sent Done false
189552024-04-17T17:22:17.203Z 12 Acked 1011 Flush 0 Done Sent Done false
189562024-04-17T17:22:17.203Z 13 Acked 1012 Write 4 Done Sent Done false
189572024-04-17T17:22:17.203Z 14 Acked 1013 Read 6 Done Sent Done false
189582024-04-17T17:22:17.203Z 15 Acked 1014 Flush 0 Done Sent Done false
189592024-04-17T17:22:17.204Z 16 Acked 1015 Write 2 Done Sent Done false
189602024-04-17T17:22:17.204Z 17 Acked 1016 Read 8 Done Sent Done false
189612024-04-17T17:22:17.204Z 18 Acked 1017 Read 10 Done Sent Done false
189622024-04-17T17:22:17.204Z 19 Acked 1018 Write 6 Done Sent Done false
189632024-04-17T17:22:17.204Z 20 Acked 1019 Read 8 Done Sent Done false
189642024-04-17T17:22:17.204Z 21 Acked 1020 Read 5 Done Sent Done false
189652024-04-17T17:22:17.204Z 22 Acked 1021 Write 10 Done Sent Done false
189662024-04-17T17:22:17.204Z 23 Acked 1022 Read 10 Done Sent Done false
189672024-04-17T17:22:17.205Z 24 Acked 1023 Write 6 Done Sent Done false
189682024-04-17T17:22:17.205Z 25 Acked 1024 Read 6 Done Sent Done false
189692024-04-17T17:22:17.205Z 26 Acked 1025 Flush 0 Done Sent Done false
189702024-04-17T17:22:17.205Z 27 Acked 1026 Write 8 Done Sent Done false
189712024-04-17T17:22:17.205Z 28 Acked 1027 Write 6 Done Sent Done false
189722024-04-17T17:22:17.205Z 29 Acked 1028 Write 6 Done Sent Done false
189732024-04-17T17:22:17.205Z 30 Acked 1029 Read 7 Sent Sent Done false
189742024-04-17T17:22:17.205Z STATES DS:0 DS:1 DS:2 TOTAL
189752024-04-17T17:22:17.205Z New 0 0 0 0
189762024-04-17T17:22:17.205Z Sent 1 30 0 31
189772024-04-17T17:22:17.205Z Done 29 0 30 59
189782024-04-17T17:22:17.205Z Skipped 0 0 0 0
189792024-04-17T17:22:17.205Z Error 0 0 0 0
189802024-04-17T17:22:17.205ZLast Flush: 1025 0 1025
189812024-04-17T17:22:17.205ZDownstairs last five completed:
189822024-04-17T17:22:17.205ZUpstairs last five completed: 30 29 28 27 26
189832024-04-17T17:22:17.205Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:22:15.805143244Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1875}
189842024-04-17T17:22:17.206ZWrote out file "/tmp/test_repair_verify.data"
189852024-04-17T17:22:17.211Z
189862024-04-17T17:22:17.212ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
189872024-04-17T17:22:17.219ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
189882024-04-17T17:22:17.219Z 0 000-019 170 170 170 299 299 299 F F F
189892024-04-17T17:22:17.224Z 1 020-039 176 174 176 313 306 313 F F F <---
189902024-04-17T17:22:17.224Z 2 040-059 175 175 175 307 307 307 F F F
189912024-04-17T17:22:17.224Z 3 060-079 175 175 175 307 307 307 F F F
189922024-04-17T17:22:17.224Z 4 080-099 174 174 174 304 304 304 F F F
189932024-04-17T17:22:17.224Z 5 100-119 176 174 176 313 304 313 F F F <---
189942024-04-17T17:22:17.224Z 6 120-139 170 170 170 299 299 299 F F F
189952024-04-17T17:22:17.224Z 7 140-159 174 174 174 305 305 305 F F F
189962024-04-17T17:22:17.224Z 8 160-179 174 174 174 306 306 306 F F F
189972024-04-17T17:22:17.224Z 9 180-199 176 168 176 309 297 309 F F F <---
189982024-04-17T17:22:17.224Z 10 200-219 172 172 172 300 300 300 F F F
189992024-04-17T17:22:17.224Z 11 220-239 172 172 172 300 300 300 F F F
190002024-04-17T17:22:17.224Z 12 240-259 172 172 172 302 302 302 F F F
190012024-04-17T17:22:17.224Z 13 260-279 176 172 176 313 300 313 F F F <---
190022024-04-17T17:22:17.225Z 14 280-299 175 175 175 307 307 307 F F F
190032024-04-17T17:22:17.225Z 15 300-319 175 175 175 307 307 307 F F F
190042024-04-17T17:22:17.225Z 16 320-339 175 175 175 307 307 307 F F F
190052024-04-17T17:22:17.225Z 17 340-359 175 175 175 307 307 307 F F F
190062024-04-17T17:22:17.225Z 18 360-379 176 172 176 311 302 311 F F F <---
190072024-04-17T17:22:17.225Z 19 380-399 176 169 176 311 298 311 T F T <---
190082024-04-17T17:22:17.225Z 20 400-419 174 174 174 306 306 306 T F T <---
190092024-04-17T17:22:17.225Z 21 420-439 176 173 176 308 303 308 F F F <---
190102024-04-17T17:22:17.225Z 22 440-459 176 169 176 308 298 308 F F F <---
190112024-04-17T17:22:17.225Z 23 460-479 166 166 166 291 291 291 F F F
190122024-04-17T17:22:17.226Z 24 480-499 174 174 174 305 305 305 T F T <---
190132024-04-17T17:22:17.226Z 25 500-519 176 172 176 313 302 313 F F F <---
190142024-04-17T17:22:17.226Z 26 520-539 176 173 176 313 303 313 T F T <---
190152024-04-17T17:22:17.226Z 27 540-559 170 170 170 299 299 299 F F F
190162024-04-17T17:22:17.226Z 28 560-579 174 174 174 305 305 305 F F F
190172024-04-17T17:22:17.226Z 29 580-599 163 163 163 288 288 288 F F F
190182024-04-17T17:22:17.226ZMax gen: 176, Max flush: 313
190192024-04-17T17:22:17.226ZError: Difference in extent metadata found!
190202024-04-17T17:22:17.226ZA Difference in extent metadata is expected here
190212024-04-17T17:22:17.226ZOn loop 88
190222024-04-17T17:22:17.227Z
190232024-04-17T17:22:17.230ZVerifying data now
190242024-04-17T17:22:21.787ZNo change after verify
190252024-04-17T17:22:21.787ZLoop: 88 Downstairs dump after verify (and repair):
190262024-04-17T17:22:21.802ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
190272024-04-17T17:22:21.802Z 0 000-019 170 170 170 299 299 299 F F F
190282024-04-17T17:22:21.807Z 1 020-039 176 176 176 313 313 313 F F F
190292024-04-17T17:22:21.807Z 2 040-059 175 175 175 307 307 307 F F F
190302024-04-17T17:22:21.807Z 3 060-079 175 175 175 307 307 307 F F F
190312024-04-17T17:22:21.807Z 4 080-099 174 174 174 304 304 304 F F F
190322024-04-17T17:22:21.807Z 5 100-119 176 176 176 313 313 313 F F F
190332024-04-17T17:22:21.808Z 6 120-139 170 170 170 299 299 299 F F F
190342024-04-17T17:22:21.808Z 7 140-159 174 174 174 305 305 305 F F F
190352024-04-17T17:22:21.808Z 8 160-179 174 174 174 306 306 306 F F F
190362024-04-17T17:22:21.808Z 9 180-199 176 176 176 309 309 309 F F F
190372024-04-17T17:22:21.808Z 10 200-219 172 172 172 300 300 300 F F F
190382024-04-17T17:22:21.808Z 11 220-239 172 172 172 300 300 300 F F F
190392024-04-17T17:22:21.808Z 12 240-259 172 172 172 302 302 302 F F F
190402024-04-17T17:22:21.808Z 13 260-279 176 176 176 313 313 313 F F F
190412024-04-17T17:22:21.808Z 14 280-299 175 175 175 307 307 307 F F F
190422024-04-17T17:22:21.808Z 15 300-319 175 175 175 307 307 307 F F F
190432024-04-17T17:22:21.808Z 16 320-339 175 175 175 307 307 307 F F F
190442024-04-17T17:22:21.808Z 17 340-359 175 175 175 307 307 307 F F F
190452024-04-17T17:22:21.808Z 18 360-379 176 176 176 311 311 311 F F F
190462024-04-17T17:22:21.808Z 19 380-399 177 177 177 314 314 314 F F F
190472024-04-17T17:22:21.808Z 20 400-419 177 177 177 314 314 314 F F F
190482024-04-17T17:22:21.808Z 21 420-439 176 176 176 308 308 308 F F F
190492024-04-17T17:22:21.808Z 22 440-459 176 176 176 308 308 308 F F F
190502024-04-17T17:22:21.808Z 23 460-479 166 166 166 291 291 291 F F F
190512024-04-17T17:22:21.808Z 24 480-499 177 177 177 314 314 314 F F F
190522024-04-17T17:22:21.808Z 25 500-519 176 176 176 313 313 313 F F F
190532024-04-17T17:22:21.808Z 26 520-539 177 177 177 314 314 314 F F F
190542024-04-17T17:22:21.808Z 27 540-559 170 170 170 299 299 299 F F F
190552024-04-17T17:22:21.809Z 28 560-579 174 174 174 305 305 305 F F F
190562024-04-17T17:22:21.809Z 29 580-599 163 163 163 288 288 288 F F F
190572024-04-17T17:22:21.809ZMax gen: 177, Max flush: 314
190582024-04-17T17:22:21.809Z
190592024-04-17T17:22:21.809ZBegin loop 89
190602024-04-17T17:22:21.809ZDownstairs to restart: 2
190612024-04-17T17:22:21.826Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.433814036Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883}
190622024-04-17T17:22:21.826Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.434065871Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883}
190632024-04-17T17:22:21.826Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.434086162Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883}
190642024-04-17T17:22:21.833Z{"msg":"Crucible 231ff345-d60f-4845-9930-9774ea524311 has session id: 133ce79a-3f31-42a0-ae5e-99b7fc7dc39d","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.434124915Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190652024-04-17T17:22:21.833Z{"msg":"Crucible runtime is spawned
190662024-04-17T17:22:21.834ZUpstairs opts: Upstairs UUID: 231ff345-d60f-4845-9930-9774ea524311, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.434230921Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190672024-04-17T17:22:21.834Z{"msg":"Crucible stats registered with UUID: 231ff345-d60f-4845-9930-9774ea524311","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.434260323Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190682024-04-17T17:22:21.834Z{"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.434293055Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883}
190692024-04-17T17:22:21.834Z{"msg":"231ff345-d60f-4845-9930-9774ea524311 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.434399331Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190702024-04-17T17:22:21.834Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.434429603Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"client":"0","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190712024-04-17T17:22:21.834Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.434455864Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"client":"0","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190722024-04-17T17:22:21.834Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.434488937Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"client":"1","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190732024-04-17T17:22:21.834Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.434516828Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"client":"1","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190742024-04-17T17:22:21.834Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.43454514Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"client":"2","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190752024-04-17T17:22:21.834Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.434568871Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"client":"2","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190762024-04-17T17:22:21.835Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.434589603Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"":"io task","client":"2","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190772024-04-17T17:22:21.835Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.434673988Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"":"io task","client":"0","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190782024-04-17T17:22:21.835Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.43471356Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"":"io task","client":"1","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190792024-04-17T17:22:21.835Z{"msg":"ds_connection connected from Ok(127.0.0.1:64964)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.434742162Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"":"io task","client":"0","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190802024-04-17T17:22:21.835Z{"msg":"ds_connection connected from Ok(127.0.0.1:47506)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.434767493Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"":"io task","client":"2","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190812024-04-17T17:22:21.835Z{"msg":"ds_connection connected from Ok(127.0.0.1:47993)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.434807866Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"":"io task","client":"1","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190822024-04-17T17:22:21.835Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.435627586Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"client":"1","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190832024-04-17T17:22:21.835Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.435669718Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"client":"1","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190842024-04-17T17:22:21.836Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.435690839Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"client":"0","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190852024-04-17T17:22:21.836Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.43570578Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"client":"0","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190862024-04-17T17:22:21.836Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.436126316Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"client":"2","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190872024-04-17T17:22:21.836Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.436175989Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"client":"2","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190882024-04-17T17:22:21.836Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.436205541Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"client":"0","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190892024-04-17T17:22:21.836Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.436414653Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"client":"1","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190902024-04-17T17:22:21.836Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.43669571Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"client":"0","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190912024-04-17T17:22:21.836Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.436737413Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"client":"0","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190922024-04-17T17:22:21.836Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.436761194Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190932024-04-17T17:22:21.836Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.436835629Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"client":"2","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190942024-04-17T17:22:21.837Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.436982558Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"client":"1","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190952024-04-17T17:22:21.837Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.437011969Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"client":"1","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190962024-04-17T17:22:21.837Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.4370264Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190972024-04-17T17:22:21.837Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.437301847Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"client":"2","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190982024-04-17T17:22:21.837Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.43734703Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"client":"2","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
190992024-04-17T17:22:21.837Z{"msg":"[0]R flush_numbers[0..12]: [299, 313, 307, 307, 304, 313, 299, 305, 306, 309, 300, 300]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.437383942Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"":"downstairs","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
191002024-04-17T17:22:21.837Z{"msg":"[0]R generation[0..12]: [170, 176, 175, 175, 174, 176, 170, 174, 174, 176, 172, 172]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.437420594Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"":"downstairs","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
191012024-04-17T17:22:21.837Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.437456546Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"":"downstairs","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
191022024-04-17T17:22:21.837Z{"msg":"[1]R flush_numbers[0..12]: [299, 313, 307, 307, 304, 313, 299, 305, 306, 309, 300, 300]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.437490558Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"":"downstairs","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
191032024-04-17T17:22:21.837ZWait for a query_work_queue command to finish before sending IO
191042024-04-17T17:22:21.838Z{"msg":"[1]R generation[0..12]: [170, 176, 175, 175, 174, 176, 170, 174, 174, 176, 172, 172]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.437529121Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"":"downstairs","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
191052024-04-17T17:22:21.838Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.437566623Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"":"downstairs","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
191062024-04-17T17:22:21.838Z{"msg":"[2]R flush_numbers[0..12]: [299, 313, 307, 307, 304, 313, 299, 305, 306, 309, 300, 300]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.437603815Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"":"downstairs","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
191072024-04-17T17:22:21.838Z{"msg":"[2]R generation[0..12]: [170, 176, 175, 175, 174, 176, 170, 174, 174, 176, 172, 172]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.437642588Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"":"downstairs","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
191082024-04-17T17:22:21.838Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.43768154Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"":"downstairs","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
191092024-04-17T17:22:21.838Z{"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.437717882Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"":"downstairs","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
191102024-04-17T17:22:21.838Z{"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.437753374Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"":"downstairs","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
191112024-04-17T17:22:21.838Z{"msg":"Next flush: 315","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.437790226Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"":"downstairs","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
191122024-04-17T17:22:21.839Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.437828199Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"":"downstairs","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
191132024-04-17T17:22:21.839Z{"msg":"No downstairs reconciliation required","v":0Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
191142024-04-17T17:22:21.839Z,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.437867221Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
191152024-04-17T17:22:21.839Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.437902553Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"client":"0","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
191162024-04-17T17:22:21.839Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.437934165Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"client":"1","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
191172024-04-17T17:22:21.839Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.437949206Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"client":"2","":"downstairs","session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
191182024-04-17T17:22:21.839Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.437969337Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
191192024-04-17T17:22:21.839Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.437981938Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
191202024-04-17T17:22:21.839Z{"msg":"231ff345-d60f-4845-9930-9774ea524311 is now active with session: 133ce79a-3f31-42a0-ae5e-99b7fc7dc39d","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.438001609Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
191212024-04-17T17:22:21.839Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.43801382Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883,"session_id":"133ce79a-3f31-42a0-ae5e-99b7fc7dc39d"}
191222024-04-17T17:22:21.840Z{"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:20.438033601Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883}
191232024-04-17T17:22:21.840ZLoading write count information from file "/tmp/test_repair_verify.data"
191242024-04-17T17:22:21.840ZRun Repair workload
191252024-04-17T17:22:21.840Z01/30 Flush
191262024-04-17T17:22:21.845Z02/30 Write block 166 len 28672 data: 17 18 16 14 13 16 14
191272024-04-17T17:22:21.846Z03/30 Write block 118 len 12288 data: 9 11 13
191282024-04-17T17:22:21.846Z04/30 Read block 21 len 16384
191292024-04-17T17:22:21.846Z05/30 Write block 538 len 28672 data: 9 14 15 16 20 18 18
191302024-04-17T17:22:21.846Z06/30 Read block 421 len 40960
191312024-04-17T17:22:21.846Z07/30 Write block 505 len 20480 data: 10 9 12 12 12
191322024-04-17T17:22:21.846Z08/30 Write block 134 len 36864 data: 13 12 15 15 13 13 11 8 8
191332024-04-17T17:22:21.846Z09/30 Read block 542 len 4096
191342024-04-17T17:22:21.846Z10/30 Write block 588 len 12288 data: 9 8 9
191352024-04-17T17:22:21.847Z11/30 Write block 291 len 16384 data: 20 17 20 18
191362024-04-17T17:22:21.847Z12/30 Write block 32 len 8192 data: 21 22
191372024-04-17T17:22:21.847Z13/30 Read block 298 len 24576
191382024-04-17T17:22:21.847Z14/30 Write block 127 len 4096 data: 14
191392024-04-17T17:22:21.847Z15/30 Flush
191402024-04-17T17:22:21.852Z16/30 Write block 60 len 24576 data: 19 18 15 17 19 18
191412024-04-17T17:22:21.852Z17/30 Flush
191422024-04-17T17:22:21.852Z18/30 Write block 165 len 24576 data: 13 18 19 17 15 14
191432024-04-17T17:22:21.852Z19/30 Read block 172 len 8192
191442024-04-17T17:22:21.852Z20/30 Write block 18 len 28672 data: 11 14 15 18 17 15 14
191452024-04-17T17:22:21.852Z21/30 Flush
191462024-04-17T17:22:21.857Z22/30 Write block 302 len 12288 data: 12 15 16
191472024-04-17T17:22:21.857Z23/30 Write block 27 len 28672 data: 9 10 15 14 16 22 23
191482024-04-17T17:22:21.857Z24/30 Read block 483 len 36864
191492024-04-17T17:22:21.857Z25/30 Read block 132 len 24576
191502024-04-17T17:22:21.862Z26/30 Read block 85 len 8192
191512024-04-17T17:22:21.862Z27/30 Read block 478 len 24576
191522024-04-17T17:22:21.862Z28/30 Read block 114 len 32768
191532024-04-17T17:22:21.863Z29/30 Write block 72 len 4096 data: 14
191542024-04-17T17:22:21.863Z30/30 Write block 567 len 28672 data: 11 13 11 11 11 13 14
191552024-04-17T17:22:21.863Z----------------------------------------------------------------
191562024-04-17T17:22:21.863Z Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:29
191572024-04-17T17:22:21.863ZStates: Active Active Active
191582024-04-17T17:22:21.864ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
191592024-04-17T17:22:21.864Z 2 Acked 1001 Write 7 Done Done Done false
191602024-04-17T17:22:21.864Z 3 Acked 1002 Write 3 Done Done Done false
191612024-04-17T17:22:21.864Z 4 Acked 1003 Read 4 Done Done Done false
191622024-04-17T17:22:21.864Z 5 Acked 1004 Write 7 Done Done Done false
191632024-04-17T17:22:21.864Z 6 Acked 1005 Read 10 Done Done Sent false
191642024-04-17T17:22:21.864Z 7 Acked 1006 Write 5 Done Done Sent false
191652024-04-17T17:22:21.864Z 8 Acked 1007 Write 9 Done Done Sent false
191662024-04-17T17:22:21.865Z 9 Acked 1008 Read 1 Done Done Sent false
191672024-04-17T17:22:21.865Z 10 Acked 1009 Write 3 Done Done Sent false
191682024-04-17T17:22:21.865Z 11 Acked 1010 Write 4 Done Done Sent false
191692024-04-17T17:22:21.865Z 12 Acked 1011 Write 2 Done Done Sent false
191702024-04-17T17:22:21.865Z 13 Acked 1012 Read 6 Done Done Sent false
191712024-04-17T17:22:21.865Z 14 Acked 1013 Write 1 Done Done Sent false
191722024-04-17T17:22:21.865Z 15 Acked 1014 Flush 0 Done Done Sent false
191732024-04-17T17:22:21.866Z 16 Acked 1015 Write 6 Done Done Sent false
191742024-04-17T17:22:21.866Z 17 Acked 1016 Flush 0 Done Done Sent false
191752024-04-17T17:22:21.866Z 18 Acked 1017 Write 6 Done Done Sent false
191762024-04-17T17:22:21.866Z 19 Acked 1018 Read 2 Done Done Sent false
191772024-04-17T17:22:21.866Z 20 Acked 1019 Write 7 Done Done Sent false
191782024-04-17T17:22:21.866Z 21 Acked 1020 Flush 0 Done Done Sent false
191792024-04-17T17:22:21.866Z 22 Acked 1021 Write 3 Done Done Sent false
191802024-04-17T17:22:21.867Z 23 Acked 1022 Write 7 Done Done Sent false
191812024-04-17T17:22:21.867Z 24 Acked 1023 Read 9 Done Done Sent false
191822024-04-17T17:22:21.867Z 25 Acked 1024 Read 6 Done Done Sent false
191832024-04-17T17:22:21.867Z 26 Acked 1025 Read 2 Done Done Sent false
191842024-04-17T17:22:21.867Z 27 Acked 1026 Read 6 Done Done Sent false
191852024-04-17T17:22:21.867Z 28 Acked 1027 Read 8 Done Done Sent false
191862024-04-17T17:22:21.867Z 29 Acked 1028 Write 1 Done Done Sent false
191872024-04-17T17:22:21.867Z 30 Acked 1029 Write 7 Sent Sent Sent false
191882024-04-17T17:22:21.868Z STATES DS:0 DS:1 DS:2 TOTAL
191892024-04-17T17:22:21.868Z New 0 0 0 0
191902024-04-17T17:22:21.868Z Sent 1 1 25 27
191912024-04-17T17:22:21.868Z Done 28 28 4 60
191922024-04-17T17:22:21.868Z Skipped 0 0 0 0
191932024-04-17T17:22:21.868Z Error 0 0 0 0
191942024-04-17T17:22:21.868ZLast Flush: 1020 1020 1000
191952024-04-17T17:22:21.868ZDownstairs last five completed: 1000
191962024-04-17T17:22:21.869ZUpstairs last five completed: 30 29 28 27 26
191972024-04-17T17:22:21.869Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:22:20.471134157Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1883}
191982024-04-17T17:22:21.869ZWrote out file "/tmp/test_repair_verify.data"
191992024-04-17T17:22:21.875Z
192002024-04-17T17:22:21.875ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
192012024-04-17T17:22:21.888ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
192022024-04-17T17:22:21.888Z 0 000-019 178 178 170 318 318 299 F F F <---
192032024-04-17T17:22:21.893Z 1 020-039 178 178 176 318 318 313 T T F <---
192042024-04-17T17:22:21.894Z 2 040-059 175 175 175 307 307 307 F F F
192052024-04-17T17:22:21.894Z 3 060-079 178 178 175 317 317 307 T T F <---
192062024-04-17T17:22:21.894Z 4 080-099 174 174 174 304 304 304 F F F
192072024-04-17T17:22:21.894Z 5 100-119 178 178 176 316 316 313 F F T <---
192082024-04-17T17:22:21.894Z 6 120-139 178 178 170 316 316 299 F F T <---
192092024-04-17T17:22:21.894Z 7 140-159 178 178 174 316 316 305 F F F <---
192102024-04-17T17:22:21.894Z 8 160-179 178 178 174 318 318 306 F F T <---
192112024-04-17T17:22:21.894Z 9 180-199 176 176 176 309 309 309 F F F
192122024-04-17T17:22:21.895Z 10 200-219 172 172 172 300 300 300 F F F
192132024-04-17T17:22:21.895Z 11 220-239 172 172 172 300 300 300 F F F
192142024-04-17T17:22:21.895Z 12 240-259 172 172 172 302 302 302 F F F
192152024-04-17T17:22:21.895Z 13 260-279 176 176 176 313 313 313 F F F
192162024-04-17T17:22:21.895Z 14 280-299 178 178 175 316 316 307 F F F <---
192172024-04-17T17:22:21.895Z 15 300-319 175 175 175 307 307 307 T T F <---
192182024-04-17T17:22:21.895Z 16 320-339 175 175 175 307 307 307 F F F
192192024-04-17T17:22:21.896Z 17 340-359 175 175 175 307 307 307 F F F
192202024-04-17T17:22:21.896Z 18 360-379 176 176 176 311 311 311 F F F
192212024-04-17T17:22:21.896Z 19 380-399 177 177 177 314 314 314 F F F
192222024-04-17T17:22:21.896Z 20 400-419 177 177 177 314 314 314 F F F
192232024-04-17T17:22:21.896Z 21 420-439 176 176 176 308 308 308 F F F
192242024-04-17T17:22:21.896Z 22 440-459 176 176 176 308 308 308 F F F
192252024-04-17T17:22:21.897Z 23 460-479 166 166 166 291 291 291 F F F
192262024-04-17T17:22:21.897Z 24 480-499 177 177 177 314 314 314 F F F
192272024-04-17T17:22:21.897Z 25 500-519 178 178 176 316 316 313 F F F <---
192282024-04-17T17:22:21.897Z 26 520-539 178 178 177 316 316 314 F F T <---
192292024-04-17T17:22:21.897Z 27 540-559 178 178 170 316 316 299 F F T <---
192302024-04-17T17:22:21.897Z 28 560-579 174 174 174 305 305 305 T T F <---
192312024-04-17T17:22:21.897Z 29 580-599 178 178 163 316 316 288 F F F <---
192322024-04-17T17:22:21.897ZMax gen: 178, Max flush: 318
192332024-04-17T17:22:21.898ZError: Difference in extent metadata found!
192342024-04-17T17:22:21.898ZA Difference in extent metadata is expected here
192352024-04-17T17:22:21.898ZOn loop 89
192362024-04-17T17:22:21.898Z
192372024-04-17T17:22:21.903ZVerifying data now
192382024-04-17T17:22:36.540ZNo change after verify
192392024-04-17T17:22:36.540ZLoop: 89 Downstairs dump after verify (and repair):
192402024-04-17T17:22:36.554ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
192412024-04-17T17:22:36.554Z 0 000-019 178 178 178 318 318 318 F F F
192422024-04-17T17:22:36.559Z 1 020-039 179 179 179 319 319 319 F F F
192432024-04-17T17:22:36.559Z 2 040-059 175 175 175 307 307 307 F F F
192442024-04-17T17:22:36.559Z 3 060-079 179 179 179 319 319 319 F F F
192452024-04-17T17:22:36.560Z 4 080-099 174 174 174 304 304 304 F F F
192462024-04-17T17:22:36.560Z 5 100-119 178 178 178 316 316 316 F F F
192472024-04-17T17:22:36.560Z 6 120-139 178 178 178 316 316 316 F F F
192482024-04-17T17:22:36.560Z 7 140-159 178 178 178 316 316 316 F F F
192492024-04-17T17:22:36.560Z 8 160-179 178 178 178 318 318 318 F F F
192502024-04-17T17:22:36.560Z 9 180-199 176 176 176 309 309 309 F F F
192512024-04-17T17:22:36.560Z 10 200-219 172 172 172 300 300 300 F F F
192522024-04-17T17:22:36.560Z 11 220-239 172 172 172 300 300 300 F F F
192532024-04-17T17:22:36.560Z 12 240-259 172 172 172 302 302 302 F F F
192542024-04-17T17:22:36.560Z 13 260-279 176 176 176 313 313 313 F F F
192552024-04-17T17:22:36.560Z 14 280-299 178 178 178 316 316 316 F F F
192562024-04-17T17:22:36.561Z 15 300-319 179 179 179 319 319 319 F F F
192572024-04-17T17:22:36.561Z 16 320-339 175 175 175 307 307 307 F F F
192582024-04-17T17:22:36.561Z 17 340-359 175 175 175 307 307 307 F F F
192592024-04-17T17:22:36.561Z 18 360-379 176 176 176 311 311 311 F F F
192602024-04-17T17:22:36.561Z 19 380-399 177 177 177 314 314 314 F F F
192612024-04-17T17:22:36.561Z 20 400-419 177 177 177 314 314 314 F F F
192622024-04-17T17:22:36.561Z 21 420-439 176 176 176 308 308 308 F F F
192632024-04-17T17:22:36.561Z 22 440-459 176 176 176 308 308 308 F F F
192642024-04-17T17:22:36.561Z 23 460-479 166 166 166 291 291 291 F F F
192652024-04-17T17:22:36.561Z 24 480-499 177 177 177 314 314 314 F F F
192662024-04-17T17:22:36.561Z 25 500-519 178 178 178 316 316 316 F F F
192672024-04-17T17:22:36.562Z 26 520-539 178 178 178 316 316 316 F F F
192682024-04-17T17:22:36.562Z 27 540-559 178 178 178 316 316 316 F F F
192692024-04-17T17:22:36.562Z 28 560-579 179 179 179 319 319 319 F F F
192702024-04-17T17:22:36.562Z 29 580-599 178 178 178 316 316 316 F F F
192712024-04-17T17:22:36.562ZMax gen: 179, Max flush: 319
192722024-04-17T17:22:36.562Z
192732024-04-17T17:22:36.562ZBegin loop 90
192742024-04-17T17:22:36.562ZDownstairs to restart: 2
192752024-04-17T17:22:36.578Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.186891928Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891}
192762024-04-17T17:22:36.578Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.187147373Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891}
192772024-04-17T17:22:36.578Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.187164344Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891}
192782024-04-17T17:22:36.578Z{"msg":"Crucible f30fd007-81af-4a4b-87ed-0749a1d423c3 has session id: 4bd3bdc1-da87-4127-9cad-d960c9a2cda2","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.187186406Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
192792024-04-17T17:22:36.587Z{"msg":"Upstairs opts: Upstairs UUID: f30fd007-81af-4a4b-87ed-0749a1d423c3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.187236618Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
192802024-04-17T17:22:36.587Z{"msg":"Crucible stats registered with UUID: f30fd007-81af-4a4b-87ed-0749a1d423c3","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.18726364Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
192812024-04-17T17:22:36.587ZCrucible runtime is spawned
192822024-04-17T17:22:36.587Z{"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.187398838Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891}
192832024-04-17T17:22:36.587Z{"msg":"f30fd007-81af-4a4b-87ed-0749a1d423c3 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.18742959Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
192842024-04-17T17:22:36.588Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.187468143Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"client":"0","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
192852024-04-17T17:22:36.588Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.187482593Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"client":"0","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
192862024-04-17T17:22:36.588Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.187496044Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"client":"1","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
192872024-04-17T17:22:36.588Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.187525446Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"client":"1","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
192882024-04-17T17:22:36.588Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.187546347Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"client":"2","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
192892024-04-17T17:22:36.588Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.187570189Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"client":"2","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
192902024-04-17T17:22:36.589Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.187602941Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"":"io task","client":"2","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
192912024-04-17T17:22:36.589Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.187636783Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"":"io task","client":"0","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
192922024-04-17T17:22:36.589Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.187673275Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"":"io task","client":"1","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
192932024-04-17T17:22:36.589Z{"msg":"ds_connection connected from Ok(127.0.0.1:62519)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.187710727Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"":"io task","client":"2","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
192942024-04-17T17:22:36.589Z{"msg":"ds_connection connected from Ok(127.0.0.1:45188)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.18774728Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"":"io task","client":"1","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
192952024-04-17T17:22:36.589Z{"msg":"ds_connection connected from Ok(127.0.0.1:54258)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.187856936Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"":"io task","client":"0","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
192962024-04-17T17:22:36.589Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.188548278Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"client":"0","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
192972024-04-17T17:22:36.589Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.188620172Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"client":"0","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
192982024-04-17T17:22:36.590Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.188661425Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"client":"1","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
192992024-04-17T17:22:36.590Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.188686666Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"client":"1","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193002024-04-17T17:22:36.590Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.189113552Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"client":"1","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193012024-04-17T17:22:36.590Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.189153375Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"client":"2","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193022024-04-17T17:22:36.590Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.18925046Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"client":"2","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193032024-04-17T17:22:36.590Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.189281802Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"client":"0","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193042024-04-17T17:22:36.590Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.189600302Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"client":"1","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193052024-04-17T17:22:36.590Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.189634754Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"client":"1","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193062024-04-17T17:22:36.591Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.189650325Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193072024-04-17T17:22:36.591Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.189688147Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"client":"0","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193082024-04-17T17:22:36.591Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.189701978Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"client":"0","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193092024-04-17T17:22:36.591Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.18972828Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193102024-04-17T17:22:36.591Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.189742581Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"client":"2","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193112024-04-17T17:22:36.591Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.19022875Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"client":"2","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193122024-04-17T17:22:36.591Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.190269732Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"client":"2","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193132024-04-17T17:22:36.592Z{"msg":"[0]R flush_numbers[0..12]: [318, 319, 307, 319, 304, 316, 316, 316, 318, 309, 300, 300]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.190285473Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"":"downstairs","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193142024-04-17T17:22:36.592Z{"msg":"[0]R generation[0..12]: [178, 179, 175, 179, 174, 178, 178, 178, 178, 176, 172, 172]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.190307105Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"":"downstairs","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193152024-04-17T17:22:36.592Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.190322176Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"":"downstairs","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193162024-04-17T17:22:36.592Z{"msg":"[1]R flush_numbers[0..12]: [318, 319, 307, 319, 304, 316, 316, 316, 318, 309, 300, 300]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.190344937Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"":"downstairs","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193172024-04-17T17:22:36.592Z{"msg":"[1]R generation[0..12]: [178, 179, 175, 179, 174, 178, 178, 178, 178, 176, 172, 172]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.190359098Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"":"downstairs","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193182024-04-17T17:22:36.592Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.190385049Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"":"downstairs","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193192024-04-17T17:22:36.592Z{"msg":"[2]R flush_numbers[0..12]: [318, 319, 307, 319, 304, 316, 316, 316, 318, 309, 300, 300]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
193202024-04-17T17:22:36.592Z,"time":"2024-04-17T17:22:35.19039932Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"":"downstairs","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193212024-04-17T17:22:36.593Z{"msg":"[2]R generation[0..12]: [178, 179, 175, 179, 174, 178, 178, 178, 178, 176, 172, 172]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.190414131Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"":"downstairs","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193222024-04-17T17:22:36.593Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.190435803Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"":"downstairs","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193232024-04-17T17:22:36.593Z{"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.190449093Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"":"downstairs","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193242024-04-17T17:22:36.593Z{"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.190462664Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"":"downstairs","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193252024-04-17T17:22:36.593Z{"msg":"Next flush: 320","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.190475905Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"":"downstairs","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193262024-04-17T17:22:36.593Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.190488966Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"":"downstairs","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193272024-04-17T17:22:36.593Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.190502467Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193282024-04-17T17:22:36.594Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.190514177Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"client":"0","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193292024-04-17T17:22:36.594Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.190527118Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"client":"1","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193302024-04-17T17:22:36.594Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.190547169Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"client":"2","":"downstairs","session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193312024-04-17T17:22:36.594Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.1905605Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193322024-04-17T17:22:36.594Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.190581421Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193332024-04-17T17:22:36.594Z{"msg":"f30fd007-81af-4a4b-87ed-0749a1d423c3 is now active with session: 4bd3bdc1-da87-4127-9cad-d960c9a2cda2","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.190594712Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193342024-04-17T17:22:36.595Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.190606973Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891,"session_id":"4bd3bdc1-da87-4127-9cad-d960c9a2cda2"}
193352024-04-17T17:22:36.595Z{"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:35.190626124Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891}
193362024-04-17T17:22:36.595ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
193372024-04-17T17:22:36.595ZLoading write count information from file "/tmp/test_repair_verify.data"
193382024-04-17T17:22:36.595ZRun Repair workload
193392024-04-17T17:22:36.595Z01/30 Write block 68 len 12288 data: 16 14 14
193402024-04-17T17:22:36.595Z02/30 Write block 374 len 20480 data: 11 11 9 9 9
193412024-04-17T17:22:36.596Z03/30 Read block 568 len 20480
193422024-04-17T17:22:36.601Z04/30 Flush
193432024-04-17T17:22:36.601Z05/30 Write block 299 len 12288 data: 12 11 12
193442024-04-17T17:22:36.601Z06/30 Write block 402 len 40960 data: 10 11 12 13 13 11 8 10 10 12
193452024-04-17T17:22:36.601Z07/30 Write block 8 len 40960 data: 13 12 12 10 12 13 14 11 9 12
193462024-04-17T17:22:36.601Z08/30 Write block 426 len 28672 data: 10 11 10 12 12 13 17
193472024-04-17T17:22:36.601Z09/30 Read block 276 len 8192
193482024-04-17T17:22:36.601Z10/30 Write block 562 len 8192 data: 13 13
193492024-04-17T17:22:36.601Z11/30 Read block 521 len 16384
193502024-04-17T17:22:36.602Z12/30 Read block 40 len 12288
193512024-04-17T17:22:36.602Z13/30 Write block 154 len 12288 data: 13 13 14
193522024-04-17T17:22:36.602Z14/30 Write block 437 len 28672 data: 18 18 20 16 14 15 15
193532024-04-17T17:22:36.602Z15/30 Write block 416 len 12288 data: 17 15 12
193542024-04-17T17:22:36.602Z16/30 Read block 424 len 40960
193552024-04-17T17:22:36.602Z17/30 Flush
193562024-04-17T17:22:36.606Z18/30 Write block 205 len 40960 data: 22 21 24 16 17 16 16 16 17 14
193572024-04-17T17:22:36.606Z19/30 Read block 454 len 24576
193582024-04-17T17:22:36.606Z20/30 Read block 494 len 28672
193592024-04-17T17:22:36.611Z21/30 Write block 225 len 32768 data: 13 14 16 15 16 15 13 14
193602024-04-17T17:22:36.611Z22/30 Read block 41 len 32768
193612024-04-17T17:22:36.611Z23/30 Write block 272 len 36864 data: 18 22 19 16 15 9 10 11 12
193622024-04-17T17:22:36.612Z24/30 Read block 327 len 36864
193632024-04-17T17:22:36.617Z25/30 Write block 446 len 36864 data: 11 9 10 11 10 8 9 7 8
193642024-04-17T17:22:36.617Z26/30 Read block 400 len 32768
193652024-04-17T17:22:36.618Z27/30 Write block 145 len 4096 data: 14
193662024-04-17T17:22:36.618Z28/30 Write block 342 len 24576 data: 16 18 19 20 19 17
193672024-04-17T17:22:36.618Z29/30 Read block 352 len 4096
193682024-04-17T17:22:36.618Z30/30 Read block 85 len 20480
193692024-04-17T17:22:36.618Z----------------------------------------------------------------
193702024-04-17T17:22:36.618Z Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:26
193712024-04-17T17:22:36.618ZStates: Active Active Active
193722024-04-17T17:22:36.618ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
193732024-04-17T17:22:36.618Z 5 Acked 1004 Write 3 Done Done Done false
193742024-04-17T17:22:36.618Z 6 Acked 1005 Write 10 Done Done Sent false
193752024-04-17T17:22:36.619Z 7 Acked 1006 Write 10 Done Done Sent false
193762024-04-17T17:22:36.619Z 8 Acked 1007 Write 7 Done Done Sent false
193772024-04-17T17:22:36.619Z 9 Acked 1008 Read 2 Done Done Sent false
193782024-04-17T17:22:36.619Z 10 Acked 1009 Write 2 Done Done Sent false
193792024-04-17T17:22:36.619Z 11 Acked 1010 Read 4 Done Done Sent false
193802024-04-17T17:22:36.619Z 12 Acked 1011 Read 3 Done Done Sent false
193812024-04-17T17:22:36.619Z 13 Acked 1012 Write 3 Done Done Sent false
193822024-04-17T17:22:36.619Z 14 Acked 1013 Write 7 Done Done Sent false
193832024-04-17T17:22:36.619Z 15 Acked 1014 Write 3 Done Done Sent false
193842024-04-17T17:22:36.619Z 16 Acked 1015 Read 10 Done Done Sent false
193852024-04-17T17:22:36.619Z 17 Acked 1016 Flush 0 Done Done Sent false
193862024-04-17T17:22:36.620Z 18 Acked 1017 Write 10 Done Done Sent false
193872024-04-17T17:22:36.620Z 19 Acked 1018 Read 6 Done Done Sent false
193882024-04-17T17:22:36.620Z 20 Acked 1019 Read 7 Done Done Sent false
193892024-04-17T17:22:36.620Z 21 Acked 1020 Write 8 Done Done Sent false
193902024-04-17T17:22:36.620Z 22 Acked 1021 Read 8 Done Done Sent false
193912024-04-17T17:22:36.620Z 23 Acked 1022 Write 9 Done Done Sent false
193922024-04-17T17:22:36.620Z 24 Acked 1023 Read 9 Done Done Sent false
193932024-04-17T17:22:36.620Z 25 Acked 1024 Write 9 Done Done Sent false
193942024-04-17T17:22:36.620Z 26 Acked 1025 Read 8 Done Done Sent false
193952024-04-17T17:22:36.620Z 27 Acked 1026 Write 1 Done Done Sent false
193962024-04-17T17:22:36.620Z 28 Acked 1027 Write 6 Done Done Sent false
193972024-04-17T17:22:36.621Z 29 Acked 1028 Read 1 Done Done Sent false
193982024-04-17T17:22:36.621Z 30 Acked 1029 Read 5 Done Done Sent false
193992024-04-17T17:22:36.621Z STATES DS:0 DS:1 DS:2 TOTAL
194002024-04-17T17:22:36.621Z New 0 0 0 0
194012024-04-17T17:22:36.621Z Sent 0 0 25 25
194022024-04-17T17:22:36.621Z Done 26 26 1 53
194032024-04-17T17:22:36.621Z Skipped 0 0 0 0
194042024-04-17T17:22:36.621Z Error 0 0 0 0
194052024-04-17T17:22:36.621ZLast Flush: 1016 1016 1003
194062024-04-17T17:22:36.621ZDownstairs last five completed: 1003 1002 1001 1000
194072024-04-17T17:22:36.622ZUpstairs last five completed: 30 29 28 27 26
194082024-04-17T17:22:36.622Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:22:35.226546451Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1891}
194092024-04-17T17:22:36.626ZWrote out file "/tmp/test_repair_verify.data"
194102024-04-17T17:22:36.626Z
194112024-04-17T17:22:36.626ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
194122024-04-17T17:22:36.644ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
194132024-04-17T17:22:36.644Z 0 000-019 180 180 178 321 321 318 F F F <---
194142024-04-17T17:22:36.644Z 1 020-039 179 179 179 319 319 319 F F F
194152024-04-17T17:22:36.644Z 2 040-059 175 175 175 307 307 307 F F F
194162024-04-17T17:22:36.644Z 3 060-079 180 180 180 320 320 320 F F F
194172024-04-17T17:22:36.645Z 4 080-099 174 174 174 304 304 304 F F F
194182024-04-17T17:22:36.645Z 5 100-119 178 178 178 316 316 316 F F F
194192024-04-17T17:22:36.645Z 6 120-139 178 178 178 316 316 316 F F F
194202024-04-17T17:22:36.645Z 7 140-159 180 180 178 321 321 316 T T F <---
194212024-04-17T17:22:36.645Z 8 160-179 178 178 178 318 318 318 F F F
194222024-04-17T17:22:36.650Z 9 180-199 176 176 176 309 309 309 F F F
194232024-04-17T17:22:36.651Z 10 200-219 172 172 172 300 300 300 T T F <---
194242024-04-17T17:22:36.651Z 11 220-239 172 172 172 300 300 300 T T F <---
194252024-04-17T17:22:36.651Z 12 240-259 172 172 172 302 302 302 F F F
194262024-04-17T17:22:36.651Z 13 260-279 176 176 176 313 313 313 T T F <---
194272024-04-17T17:22:36.651Z 14 280-299 180 180 178 321 321 316 T T T <---
194282024-04-17T17:22:36.651Z 15 300-319 180 180 179 321 321 319 F F T <---
194292024-04-17T17:22:36.651Z 16 320-339 175 175 175 307 307 307 F F F
194302024-04-17T17:22:36.652Z 17 340-359 175 175 175 307 307 307 T T F <---
194312024-04-17T17:22:36.652Z 18 360-379 180 180 180 320 320 320 F F F
194322024-04-17T17:22:36.652Z 19 380-399 177 177 177 314 314 314 F F F
194332024-04-17T17:22:36.652Z 20 400-419 180 180 177 321 321 314 F F F <---
194342024-04-17T17:22:36.652Z 21 420-439 180 180 176 321 321 308 F F F <---
194352024-04-17T17:22:36.652Z 22 440-459 180 180 176 321 321 308 T T F <---
194362024-04-17T17:22:36.652Z 23 460-479 166 166 166 291 291 291 F F F
194372024-04-17T17:22:36.652Z 24 480-499 177 177 177 314 314 314 F F F
194382024-04-17T17:22:36.652Z 25 500-519 178 178 178 316 316 316 F F F
194392024-04-17T17:22:36.653Z 26 520-539 178 178 178 316 316 316 F F F
194402024-04-17T17:22:36.653Z 27 540-559 178 178 178 316 316 316 F F F
194412024-04-17T17:22:36.653Z 28 560-579 180 180 179 321 321 319 F F F <---
194422024-04-17T17:22:36.653Z 29 580-599 178 178 178 316 316 316 F F F
194432024-04-17T17:22:36.653ZMax gen: 180, Max flush: 321
194442024-04-17T17:22:36.653ZError: Difference in extent metadata found!
194452024-04-17T17:22:36.653ZA Difference in extent metadata is expected here
194462024-04-17T17:22:36.653ZOn loop 90
194472024-04-17T17:22:36.653Z
194482024-04-17T17:22:36.658ZVerifying data now
194492024-04-17T17:22:51.227ZNo change after verify
194502024-04-17T17:22:51.227ZLoop: 90 Downstairs dump after verify (and repair):
194512024-04-17T17:22:51.238ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
194522024-04-17T17:22:51.238Z 0 000-019 180 180 180 321 321 321 F F F
194532024-04-17T17:22:51.242Z 1 020-039 179 179 179 319 319 319 F F F
194542024-04-17T17:22:51.242Z 2 040-059 175 175 175 307 307 307 F F F
194552024-04-17T17:22:51.242Z 3 060-079 180 180 180 320 320 320 F F F
194562024-04-17T17:22:51.242Z 4 080-099 174 174 174 304 304 304 F F F
194572024-04-17T17:22:51.242Z 5 100-119 178 178 178 316 316 316 F F F
194582024-04-17T17:22:51.242Z 6 120-139 178 178 178 316 316 316 F F F
194592024-04-17T17:22:51.242Z 7 140-159 181 181 181 322 322 322 F F F
194602024-04-17T17:22:51.242Z 8 160-179 178 178 178 318 318 318 F F F
194612024-04-17T17:22:51.242Z 9 180-199 176 176 176 309 309 309 F F F
194622024-04-17T17:22:51.242Z 10 200-219 181 181 181 322 322 322 F F F
194632024-04-17T17:22:51.242Z 11 220-239 181 181 181 322 322 322 F F F
194642024-04-17T17:22:51.242Z 12 240-259 172 172 172 302 302 302 F F F
194652024-04-17T17:22:51.242Z 13 260-279 181 181 181 322 322 322 F F F
194662024-04-17T17:22:51.242Z 14 280-299 181 181 181 322 322 322 F F F
194672024-04-17T17:22:51.242Z 15 300-319 180 180 180 321 321 321 F F F
194682024-04-17T17:22:51.242Z 16 320-339 175 175 175 307 307 307 F F F
194692024-04-17T17:22:51.242Z 17 340-359 181 181 181 322 322 322 F F F
194702024-04-17T17:22:51.242Z 18 360-379 180 180 180 320 320 320 F F F
194712024-04-17T17:22:51.242Z 19 380-399 177 177 177 314 314 314 F F F
194722024-04-17T17:22:51.242Z 20 400-419 180 180 180 321 321 321 F F F
194732024-04-17T17:22:51.242Z 21 420-439 180 180 180 321 321 321 F F F
194742024-04-17T17:22:51.243Z 22 440-459 181 181 181 322 322 322 F F F
194752024-04-17T17:22:51.243Z 23 460-479 166 166 166 291 291 291 F F F
194762024-04-17T17:22:51.243Z 24 480-499 177 177 177 314 314 314 F F F
194772024-04-17T17:22:51.243Z 25 500-519 178 178 178 316 316 316 F F F
194782024-04-17T17:22:51.243Z 26 520-539 178 178 178 316 316 316 F F F
194792024-04-17T17:22:51.243Z 27 540-559 178 178 178 316 316 316 F F F
194802024-04-17T17:22:51.243Z 28 560-579 180 180 180 321 321 321 F F F
194812024-04-17T17:22:51.243Z 29 580-599 178 178 178 316 316 316 F F F
194822024-04-17T17:22:51.243ZMax gen: 181, Max flush: 322
194832024-04-17T17:22:51.246Z
194842024-04-17T17:22:51.246ZBegin loop 91
194852024-04-17T17:22:51.247ZDownstairs to restart: 2
194862024-04-17T17:22:51.277Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.871840009Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899}
194872024-04-17T17:22:51.277Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.872173939Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":Crucible runtime is spawned
194882024-04-17T17:22:51.277Z1899}
194892024-04-17T17:22:51.277Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.872228862Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899}
194902024-04-17T17:22:51.277Z{"msg":"Crucible e574ad1e-5d90-4a8a-bfa3-958df73a8a0f has session id: 815ab114-ac8e-44b9-b56f-df2bb04a51a6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.872298877Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
194912024-04-17T17:22:51.277Z{"msg":"Upstairs opts: Upstairs UUID: e574ad1e-5d90-4a8a-bfa3-958df73a8a0f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.87235043Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
194922024-04-17T17:22:51.277Z{"msg":"Crucible stats registered with UUID: e574ad1e-5d90-4a8a-bfa3-958df73a8a0f","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.872380351Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
194932024-04-17T17:22:51.277Z{"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.872398313Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899}
194942024-04-17T17:22:51.277Z{"msg":"e574ad1e-5d90-4a8a-bfa3-958df73a8a0f active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.872413384Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
194952024-04-17T17:22:51.277Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.872428874Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"client":"0","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
194962024-04-17T17:22:51.277Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.872470687Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"client":"0","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
194972024-04-17T17:22:51.277Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.872528311Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"client":"1","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
194982024-04-17T17:22:51.277Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.872567463Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"client":"1","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
194992024-04-17T17:22:51.277Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.872595844Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"client":"2","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195002024-04-17T17:22:51.277Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.872634457Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"client":"2","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195012024-04-17T17:22:51.277Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.872765035Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"":"io task","client":"0","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195022024-04-17T17:22:51.277Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.872809208Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"":"io task","client":"2","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195032024-04-17T17:22:51.277Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.872843119Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"":"io task","client":"1","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195042024-04-17T17:22:51.277Z{"msg":"ds_connection connected from Ok(127.0.0.1:63155)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.872871161Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"":"io task","client":"0","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195052024-04-17T17:22:51.278Z{"msg":"ds_connection connected from Ok(127.0.0.1:56460)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.872894653Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"":"io task","client":"1","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195062024-04-17T17:22:51.278Z{"msg":"ds_connection connected from Ok(127.0.0.1:63343)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.872909754Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"":"io task","client":"2","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195072024-04-17T17:22:51.278Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.873468317Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"client":"0","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195082024-04-17T17:22:51.278Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.87350719Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"client":"0","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195092024-04-17T17:22:51.278Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.873605176Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"client":"1","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195102024-04-17T17:22:51.278Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.873636258Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"client":"1","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195112024-04-17T17:22:51.278Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.873660809Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"client":"2","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195122024-04-17T17:22:51.278Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.873686261Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"client":"2","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195132024-04-17T17:22:51.278Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.874118107Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"client":"2","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195142024-04-17T17:22:51.278Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.874160369Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"client":"1","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195152024-04-17T17:22:51.278Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.874255955Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"client":"0","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195162024-04-17T17:22:51.278Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.874706043Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"client":"1","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195172024-04-17T17:22:51.278Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.874757886Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"client":"1","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195182024-04-17T17:22:51.278Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.874788348Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195192024-04-17T17:22:51.278Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.87483611Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"client":"0","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195202024-04-17T17:22:51.278Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.874869402Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"client":"0","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195212024-04-17T17:22:51.278Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.874890774Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195222024-04-17T17:22:51.278Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.874918245Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"client":"2","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195232024-04-17T17:22:51.278Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.874938557Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"client":"2","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195242024-04-17T17:22:51.278Z{"msg":"[0]R flush_numbers[0..12]: [321, 319, 307, 320, 304, 316, 316, 322, 318, 309, 322, 322]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.874966998Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"":"downstairs","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195252024-04-17T17:22:51.278Z{"msg":"[0]R generation[0..12]: [180, 179, 175, 180, 174, 178, 178, 181, 178, 176, 181, 181]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.8749977Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"":"downstairs","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195262024-04-17T17:22:51.279Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.875032212Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"":"downstairs","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195272024-04-17T17:22:51.279Z{"msg":"[1]R flush_numbers[0..12]: [321, 319, 307, 320, 304, 316, 316, 322, 318, 309, 322, 322]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.875053973Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQWait for a query_work_queue command to finish before sending IO
195282024-04-17T17:22:51.279Z","pid":1899,"":"downstairs","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195292024-04-17T17:22:51.279Z{"msg":"[1]R generation[0..12]: [180, 179, 175, 180, 174, 178, 178, 181, 178, 176, 181, 181]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.875079575Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"":"downstairs","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195302024-04-17T17:22:51.279Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.875093466Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"":"downstairs","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195312024-04-17T17:22:51.279Z{"msg":"[2]R flush_numbers[0..12]: [321, 319, 307, 320, 304, 316, 316, 322, 318, 309, 322, 322]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.875114897Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"":"downstairs","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195322024-04-17T17:22:51.279Z{"msg":"[2]R generation[0..12]: [180, 179, 175, 180, 174, 178, 178, 181, 178, 176, 181, 181]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.875129198Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"":"downstairs","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195332024-04-17T17:22:51.279Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.875149749Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"":"downstairs","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195342024-04-17T17:22:51.279Z{"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.87516263Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"":"downstairs","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195352024-04-17T17:22:51.279Z{"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.875175631Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"":"downstairs","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195362024-04-17T17:22:51.279Z{"msg":"Next flush: 323","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.875188212Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"":"downstairs","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195372024-04-17T17:22:51.279Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.875201353Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"":"downstairs","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195382024-04-17T17:22:51.279Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.875221144Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195392024-04-17T17:22:51.279Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.875233315Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"client":"0","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195402024-04-17T17:22:51.279Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.875252826Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"client":"1","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195412024-04-17T17:22:51.279Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.875266026Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"client":"2","":"downstairs","session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195422024-04-17T17:22:51.279Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.875285678Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195432024-04-17T17:22:51.279Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.875297908Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195442024-04-17T17:22:51.279Z{"msg":"e574ad1e-5d90-4a8a-bfa3-958df73a8a0f is now active with session: 815ab114-ac8e-44b9-b56f-df2bb04a51a6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.87531684Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195452024-04-17T17:22:51.279Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.87532902Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899,"session_id":"815ab114-ac8e-44b9-b56f-df2bb04a51a6"}
195462024-04-17T17:22:51.279Z{"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:22:49.875348321Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899}
195472024-04-17T17:22:51.279ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
195482024-04-17T17:22:51.279ZLoading write count information from file "/tmp/test_repair_verify.data"
195492024-04-17T17:22:51.280ZRun Repair workload
195502024-04-17T17:22:51.280Z01/30 Write block 312 len 12288 data: 15 17 19
195512024-04-17T17:22:51.280Z02/30 Write block 211 len 40960 data: 17 17 18 15 15 14 11 12 12 14
195522024-04-17T17:22:51.280Z03/30 Read block 17 len 12288
195532024-04-17T17:22:51.280Z04/30 Read block 563 len 28672
195542024-04-17T17:22:51.280Z05/30 Write block 552 len 20480 data: 17 13 13 14 11
195552024-04-17T17:22:51.280Z06/30 Read block 458 len 4096
195562024-04-17T17:22:51.280Z07/30 Write block 314 len 36864 data: 20 19 17 17 18 16 14 12 7
195572024-04-17T17:22:51.285Z08/30 Write block 545 len 8192 data: 14 15
195582024-04-17T17:22:51.285Z09/30 Read block 405 len 4096
195592024-04-17T17:22:51.285Z10/30 Read block 410 len 16384
195602024-04-17T17:22:51.285Z11/30 Write block 312 len 20480 data: 16 18 21 20 18
195612024-04-17T17:22:51.285Z12/30 Read block 426 len 24576
195622024-04-17T17:22:51.285Z13/30 Write block 546 len 40960 data: 16 14 16 18 14 15 18 14 14 15
195632024-04-17T17:22:51.285Z14/30 Flush
195642024-04-17T17:22:51.285Z15/30 Flush
195652024-04-17T17:22:51.285Z16/30 Write block 513 len 12288 data: 8 13 11
195662024-04-17T17:22:51.289Z17/30 Write block 6 len 32768 data: 12 13 14 13 13 11 13 14
195672024-04-17T17:22:51.289Z18/30 Write block 422 len 16384 data: 10 9 11 10
195682024-04-17T17:22:51.289Z19/30 Read block 320 len 12288
195692024-04-17T17:22:51.289Z20/30 Read block 300 len 12288
195702024-04-17T17:22:51.289Z21/30 Read block 559 len 28672
195712024-04-17T17:22:51.293Z22/30 Read block 499 len 40960
195722024-04-17T17:22:51.293Z23/30 Write block 39 len 16384 data: 14 15 16 15
195732024-04-17T17:22:51.293Z24/30 Read block 352 len 40960
195742024-04-17T17:22:51.293Z25/30 Read block 542 len 24576
195752024-04-17T17:22:51.293Z26/30 Write block 104 len 8192 data: 12 13
195762024-04-17T17:22:51.298Z27/30 Write block 546 len 28672 data: 17 15 17 19 15 16 19
195772024-04-17T17:22:51.298Z28/30 Read block 543 len 12288
195782024-04-17T17:22:51.298Z29/30 Write block 498 len 36864 data: 15 16 16 17 15 15 13 11 10
195792024-04-17T17:22:51.298Z30/30 Write block 559 len 36864 data: 12 13 13 14 14 15 10 10 12
195802024-04-17T17:22:51.298Z----------------------------------------------------------------
195812024-04-17T17:22:51.298Z Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30
195822024-04-17T17:22:51.298ZStates: Active Active Active
195832024-04-17T17:22:51.298ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
195842024-04-17T17:22:51.298Z 1 Acked 1000 Write 3 Done Done Done false
195852024-04-17T17:22:51.298Z 2 Acked 1001 Write 10 Done Done Done false
195862024-04-17T17:22:51.298Z 3 Acked 1002 Read 3 Done Done Done false
195872024-04-17T17:22:51.298Z 4 Acked 1003 Read 7 Done Done Done false
195882024-04-17T17:22:51.298Z 5 Acked 1004 Write 5 Done Done Done false
195892024-04-17T17:22:51.298Z 6 Acked 1005 Read 1 Done Done Sent false
195902024-04-17T17:22:51.298Z 7 Acked 1006 Write 9 Done Done Sent false
195912024-04-17T17:22:51.303Z 8 Acked 1007 Write 2 Done Done Sent false
195922024-04-17T17:22:51.303Z 9 Acked 1008 Read 1 Done Done Sent false
195932024-04-17T17:22:51.303Z 10 Acked 1009 Read 4 Done Done Sent false
195942024-04-17T17:22:51.303Z 11 Acked 1010 Write 5 Done Done Sent false
195952024-04-17T17:22:51.303Z 12 Acked 1011 Read 6 Done Done Sent false
195962024-04-17T17:22:51.303Z 13 Acked 1012 Write 10 Done Done Sent false
195972024-04-17T17:22:51.303Z 14 Acked 1013 Flush 0 Done Done Sent false
195982024-04-17T17:22:51.303Z 15 Acked 1014 Flush 0 Done Done Sent false
195992024-04-17T17:22:51.303Z 16 Acked 1015 Write 3 Done Done Sent false
196002024-04-17T17:22:51.303Z 17 Acked 1016 Write 8 Done Done Sent false
196012024-04-17T17:22:51.303Z 18 Acked 1017 Write 4 Done Done Sent false
196022024-04-17T17:22:51.303Z 19 Acked 1018 Read 3 Done Done Sent false
196032024-04-17T17:22:51.303Z 20 Acked 1019 Read 3 Done Done Sent false
196042024-04-17T17:22:51.303Z 21 Acked 1020 Read 7 Done Done Sent false
196052024-04-17T17:22:51.303Z 22 Acked 1021 Read 10 Done Done Sent false
196062024-04-17T17:22:51.303Z 23 Acked 1022 Write 4 Done Done Sent false
196072024-04-17T17:22:51.303Z 24 Acked 1023 Read 10 Done Done Sent false
196082024-04-17T17:22:51.303Z 25 Acked 1024 Read 6 Done Done Sent false
196092024-04-17T17:22:51.303Z 26 Acked 1025 Write 2 Done Done Sent false
196102024-04-17T17:22:51.304Z 27 Acked 1026 Write 7 Done Done Sent false
196112024-04-17T17:22:51.304Z 28 Acked 1027 Read 3 Done Done Sent false
196122024-04-17T17:22:51.304Z 29 Acked 1028 Write 9 Done Done Sent false
196132024-04-17T17:22:51.304Z 30 Acked 1029 Write 9 Sent Sent Sent false
196142024-04-17T17:22:51.304Z STATES DS:0 DS:1 DS:2 TOTAL
196152024-04-17T17:22:51.304Z New 0 0 0 0
196162024-04-17T17:22:51.304Z Sent 1 1 25 27
196172024-04-17T17:22:51.304Z Done 29 29 5 63
196182024-04-17T17:22:51.304Z Skipped 0 0 0 0
196192024-04-17T17:22:51.304Z Error 0 0 0 0
196202024-04-17T17:22:51.304ZLast Flush: 1014 1014 0
196212024-04-17T17:22:51.304ZDownstairs last five completed:
196222024-04-17T17:22:51.304ZUpstairs last five completed: 30 29 28 27 26
196232024-04-17T17:22:51.304Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:22:49.907865332Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1899}
196242024-04-17T17:22:51.304ZWrote out file "/tmp/test_repair_verify.data"
196252024-04-17T17:22:51.304Z
196262024-04-17T17:22:51.312ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
196272024-04-17T17:22:51.323ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
196282024-04-17T17:22:51.324Z 0 000-019 180 180 180 321 321 321 T T F <---
196292024-04-17T17:22:51.324Z 1 020-039 179 179 179 319 319 319 T T F <---
196302024-04-17T17:22:51.324Z 2 040-059 175 175 175 307 307 307 T T F <---
196312024-04-17T17:22:51.324Z 3 060-079 180 180 180 320 320 320 F F F
196322024-04-17T17:22:51.324Z 4 080-099 174 174 174 304 304 304 F F F
196332024-04-17T17:22:51.324Z 5 100-119 178 178 178 316 316 316 T T F <---
196342024-04-17T17:22:51.324Z 6 120-139 178 178 178 316 316 316 F F F
196352024-04-17T17:22:51.324Z 7 140-159 181 181 181 322 322 322 F F F
196362024-04-17T17:22:51.324Z 8 160-179 178 178 178 318 318 318 F F F
196372024-04-17T17:22:51.324Z 9 180-199 176 176 176 309 309 309 F F F
196382024-04-17T17:22:51.324Z 10 200-219 182 182 181 323 323 322 F F T <---
196392024-04-17T17:22:51.328Z 11 220-239 182 182 181 323 323 322 F F T <---
196402024-04-17T17:22:51.328Z 12 240-259 172 172 172 302 302 302 F F F
196412024-04-17T17:22:51.328Z 13 260-279 181 181 181 322 322 322 F F F
196422024-04-17T17:22:51.328Z 14 280-299 181 181 181 322 322 322 F F F
196432024-04-17T17:22:51.328Z 15 300-319 182 182 180 323 323 321 F F T <---
196442024-04-17T17:22:51.328Z 16 320-339 182 182 175 323 323 307 F F F <---
196452024-04-17T17:22:51.328Z 17 340-359 181 181 181 322 322 322 F F F
196462024-04-17T17:22:51.328Z 18 360-379 180 180 180 320 320 320 F F F
196472024-04-17T17:22:51.328Z 19 380-399 177 177 177 314 314 314 F F F
196482024-04-17T17:22:51.328Z 20 400-419 180 180 180 321 321 321 F F F
196492024-04-17T17:22:51.328Z 21 420-439 180 180 180 321 321 321 T T F <---
196502024-04-17T17:22:51.328Z 22 440-459 181 181 181 322 322 322 F F F
196512024-04-17T17:22:51.328Z 23 460-479 166 166 166 291 291 291 F F F
196522024-04-17T17:22:51.328Z 24 480-499 177 177 177 314 314 314 T T F <---
196532024-04-17T17:22:51.328Z 25 500-519 178 178 178 316 316 316 T T F <---
196542024-04-17T17:22:51.328Z 26 520-539 178 178 178 316 316 316 F F F
196552024-04-17T17:22:51.328Z 27 540-559 182 182 178 323 323 316 T T T <---
196562024-04-17T17:22:51.328Z 28 560-579 180 180 180 321 321 321 T T F <---
196572024-04-17T17:22:51.329Z 29 580-599 178 178 178 316 316 316 F F F
196582024-04-17T17:22:51.329ZMax gen: 182, Max flush: 323
196592024-04-17T17:22:51.329ZError: Difference in extent metadata found!
196602024-04-17T17:22:51.329ZA Difference in extent metadata is expected here
196612024-04-17T17:22:51.329ZOn loop 91
196622024-04-17T17:22:51.329Z
196632024-04-17T17:22:51.333ZVerifying data now
196642024-04-17T17:23:05.951ZNo change after verify
196652024-04-17T17:23:05.951ZLoop: 91 Downstairs dump after verify (and repair):
196662024-04-17T17:23:05.965ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
196672024-04-17T17:23:05.965Z 0 000-019 183 183 183 324 324 324 F F F
196682024-04-17T17:23:05.965Z 1 020-039 183 183 183 324 324 324 F F F
196692024-04-17T17:23:05.965Z 2 040-059 183 183 183 324 324 324 F F F
196702024-04-17T17:23:05.965Z 3 060-079 180 180 180 320 320 320 F F F
196712024-04-17T17:23:05.965Z 4 080-099 174 174 174 304 304 304 F F F
196722024-04-17T17:23:05.965Z 5 100-119 183 183 183 324 324 324 F F F
196732024-04-17T17:23:05.965Z 6 120-139 178 178 178 316 316 316 F F F
196742024-04-17T17:23:05.972Z 7 140-159 181 181 181 322 322 322 F F F
196752024-04-17T17:23:05.973Z 8 160-179 178 178 178 318 318 318 F F F
196762024-04-17T17:23:05.973Z 9 180-199 176 176 176 309 309 309 F F F
196772024-04-17T17:23:05.973Z 10 200-219 182 182 182 323 323 323 F F F
196782024-04-17T17:23:05.973Z 11 220-239 182 182 182 323 323 323 F F F
196792024-04-17T17:23:05.973Z 12 240-259 172 172 172 302 302 302 F F F
196802024-04-17T17:23:05.973Z 13 260-279 181 181 181 322 322 322 F F F
196812024-04-17T17:23:05.973Z 14 280-299 181 181 181 322 322 322 F F F
196822024-04-17T17:23:05.973Z 15 300-319 182 182 182 323 323 323 F F F
196832024-04-17T17:23:05.973Z 16 320-339 182 182 182 323 323 323 F F F
196842024-04-17T17:23:05.973Z 17 340-359 181 181 181 322 322 322 F F F
196852024-04-17T17:23:05.973Z 18 360-379 180 180 180 320 320 320 F F F
196862024-04-17T17:23:05.973Z 19 380-399 177 177 177 314 314 314 F F F
196872024-04-17T17:23:05.973Z 20 400-419 180 180 180 321 321 321 F F F
196882024-04-17T17:23:05.973Z 21 420-439 183 183 183 324 324 324 F F F
196892024-04-17T17:23:05.973Z 22 440-459 181 181 181 322 322 322 F F F
196902024-04-17T17:23:05.973Z 23 460-479 166 166 166 291 291 291 F F F
196912024-04-17T17:23:05.973Z 24 480-499 183 183 183 324 324 324 F F F
196922024-04-17T17:23:05.973Z 25 500-519 183 183 183 324 324 324 F F F
196932024-04-17T17:23:05.973Z 26 520-539 178 178 178 316 316 316 F F F
196942024-04-17T17:23:05.973Z 27 540-559 183 183 183 324 324 324 F F F
196952024-04-17T17:23:05.973Z 28 560-579 183 183 183 324 324 324 F F F
196962024-04-17T17:23:05.973Z 29 580-599 178 178 178 316 316 316 F F F
196972024-04-17T17:23:05.974ZMax gen: 183, Max flush: 324
196982024-04-17T17:23:05.974Z
196992024-04-17T17:23:05.974ZBegin loop 92
197002024-04-17T17:23:05.974ZDownstairs to restart: 2
197012024-04-17T17:23:05.987Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.597251013Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907}
197022024-04-17T17:23:05.987Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.59753563Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907}
197032024-04-17T17:23:05.996Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30Crucible runtime is spawned
197042024-04-17T17:23:05.996Z,"time":"2024-04-17T17:23:04.597574472Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907}
197052024-04-17T17:23:05.996Z{"msg":"Crucible ccf8c5e7-2a26-4911-a4d9-df81548e8338 has session id: 52c2e379-9ac3-40c3-b5a0-40d3d5937ea8","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.597606674Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197062024-04-17T17:23:05.996Z{"msg":"Upstairs opts: Upstairs UUID: ccf8c5e7-2a26-4911-a4d9-df81548e8338, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.59769708Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197072024-04-17T17:23:05.996Z{"msg":"Crucible stats registered with UUID: ccf8c5e7-2a26-4911-a4d9-df81548e8338","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.597736912Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197082024-04-17T17:23:05.996Z{"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.597761274Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907}
197092024-04-17T17:23:05.996Z{"msg":"ccf8c5e7-2a26-4911-a4d9-df81548e8338 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.597794046Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197102024-04-17T17:23:05.996Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.597827988Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"client":"0","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197112024-04-17T17:23:05.996Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.59786394Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"client":"0","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197122024-04-17T17:23:05.996Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.597902682Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"client":"1","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197132024-04-17T17:23:05.996Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.597940134Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"client":"1","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197142024-04-17T17:23:05.996Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.597976777Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"client":"2","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197152024-04-17T17:23:05.996Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.598002808Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"client":"2","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197162024-04-17T17:23:05.996Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.598038971Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"":"io task","client":"0","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197172024-04-17T17:23:05.996Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.598080103Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"":"io task","client":"2","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197182024-04-17T17:23:05.996Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.598102574Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"":"io task","client":"1","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197192024-04-17T17:23:05.996Z{"msg":"ds_connection connected from Ok(127.0.0.1:48913)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.598133916Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"":"io task","client":"0","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197202024-04-17T17:23:05.996Z{"msg":"ds_connection connected from Ok(127.0.0.1:51692)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.598158038Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"":"io task","client":"2","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197212024-04-17T17:23:05.997Z{"msg":"ds_connection connected from Ok(127.0.0.1:53303)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.5982005Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"":"io task","client":"1","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197222024-04-17T17:23:05.997Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.598871681Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"client":"0","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197232024-04-17T17:23:05.997Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.598909613Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"client":"0","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197242024-04-17T17:23:05.997Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.598926044Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"client":"1","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197252024-04-17T17:23:05.997Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.598949486Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"client":"1","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197262024-04-17T17:23:05.997Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.599075243Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"client":"2","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197272024-04-17T17:23:05.997Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.599127777Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"client":"2","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197282024-04-17T17:23:05.997Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.599570303Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"client":"1","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197292024-04-17T17:23:05.997Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.599620866Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"client":"2","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197302024-04-17T17:23:05.997Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.599695381Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"client":"0","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197312024-04-17T17:23:05.997Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.600131557Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"client":"1","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197322024-04-17T17:23:05.997Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.60017874Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"client":"1","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197332024-04-17T17:23:05.997Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.600196961Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197342024-04-17T17:23:05.997Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.600263075Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"client":"2","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197352024-04-17T17:23:05.997Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.600289637Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"client":"2","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197362024-04-17T17:23:05.997Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.600304288Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197372024-04-17T17:23:05.997Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.600316858Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"client":"0","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197382024-04-17T17:23:05.997Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.60033742Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"client":"0","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197392024-04-17T17:23:05.997Z{"msg":"[0]R flush_numbers[0..12]: [324, 324, 324, 320, 304, 324, 316, 322, 318, 309, 323, 323]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.600351451Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"":"downstairs","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197402024-04-17T17:23:05.997Z{"msg":"[0]R generation[0..12]: [183, 183, 183, 180, 174, 183, 178, 181, 178, 176, 182, 182]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.600365661Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"":"downstairs","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197412024-04-17T17:23:05.997Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.600387563Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"":"downstairs","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197422024-04-17T17:23:05.997Z{"msg":"[1]R flush_numbers[0..12]: [324, 324, 324, 320, 304, 324, 316, 322, 318, 309, 323, 323]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.600401724Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"":"downstairs","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197432024-04-17T17:23:05.998Z{"msg":"[1]R generation[0..12]: [183, 183, 183, 180, 174, 183, 178, 181, 178, 176, 182, 182]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.600415944Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"":"downstairs","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197442024-04-17T17:23:05.998Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.600429936Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"":"downstairs","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197452024-04-17T17:23:05.998Z{"msg":"[2]R flush_numbers[0..12]: [324, 324, 324, 320, 304, 324, 316, 322, 318, 309, 323, 323]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.600450647Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"":"downstairs","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197462024-04-17T17:23:05.998Z{"msg":"[2]R generation[0..12]: [183, 183, 183, 180, 174, 183, 178, 181, 178, 176, 182, 182]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.600464407Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"":"downstairs","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197472024-04-17T17:23:05.998Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.600478358Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"":"downstairs","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197482024-04-17T17:23:05.998Z{"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
197492024-04-17T17:23:05.998Z2024-04-17T17:23:04.60050312Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"":"downstairs","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197502024-04-17T17:23:05.998Z{"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.600520881Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"":"downstairs","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197512024-04-17T17:23:05.998Z{"msg":"Next flush: 325","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.600543122Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"":"downstairs","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197522024-04-17T17:23:05.998Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.600556333Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"":"downstairs","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197532024-04-17T17:23:05.998Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.600569134Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197542024-04-17T17:23:05.998Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.600587765Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"client":"0","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197552024-04-17T17:23:05.998Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.600601196Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"client":"1","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197562024-04-17T17:23:05.998Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.600613876Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"client":"2","":"downstairs","session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197572024-04-17T17:23:05.998Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.600627367Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197582024-04-17T17:23:05.998Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.600639148Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197592024-04-17T17:23:05.998Z{"msg":"ccf8c5e7-2a26-4911-a4d9-df81548e8338 is now active with session: 52c2e379-9ac3-40c3-b5a0-40d3d5937ea8","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.600651789Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197602024-04-17T17:23:05.998Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.60067048Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907,"session_id":"52c2e379-9ac3-40c3-b5a0-40d3d5937ea8"}
197612024-04-17T17:23:05.998Z{"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:04.600682671Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907}
197622024-04-17T17:23:05.998ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
197632024-04-17T17:23:06.003ZLoading write count information from file "/tmp/test_repair_verify.data"
197642024-04-17T17:23:06.003ZRun Repair workload
197652024-04-17T17:23:06.003Z01/30 Write block 307 len 24576 data: 15 16 15 16 14 17
197662024-04-17T17:23:06.003Z02/30 Read block 365 len 32768
197672024-04-17T17:23:06.003Z03/30 Read block 554 len 40960
197682024-04-17T17:23:06.003Z04/30 Read block 79 len 8192
197692024-04-17T17:23:06.003Z05/30 Write block 146 len 24576 data: 14 12 12 13 12 13
197702024-04-17T17:23:06.003Z06/30 Read block 125 len 20480
197712024-04-17T17:23:06.003Z07/30 Read block 184 len 20480
197722024-04-17T17:23:06.003Z08/30 Read block 440 len 28672
197732024-04-17T17:23:06.003Z09/30 Write block 67 len 32768 data: 17 17 15 15 14 15 11 14
197742024-04-17T17:23:06.003Z10/30 Read block 340 len 32768
197752024-04-17T17:23:06.003Z11/30 Read block 478 len 32768
197762024-04-17T17:23:06.003Z12/30 Flush
197772024-04-17T17:23:06.008Z13/30 Write block 491 len 12288 data: 12 13 11
197782024-04-17T17:23:06.008Z14/30 Read block 468 len 28672
197792024-04-17T17:23:06.008Z15/30 Read block 99 len 28672
197802024-04-17T17:23:06.008Z16/30 Write block 449 len 16384 data: 12 11 9 10
197812024-04-17T17:23:06.008Z17/30 Read block 394 len 40960
197822024-04-17T17:23:06.012Z18/30 Flush
197832024-04-17T17:23:06.012Z19/30 Write block 513 len 16384 data: 9 14 12 10
197842024-04-17T17:23:06.012Z20/30 Read block 418 len 8192
197852024-04-17T17:23:06.012Z21/30 Write block 493 len 12288 data: 12 13 14
197862024-04-17T17:23:06.012Z22/30 Read block 492 len 4096
197872024-04-17T17:23:06.017Z23/30 Write block 371 len 16384 data: 16 13 14 12
197882024-04-17T17:23:06.017Z24/30 Write block 442 len 16384 data: 16 16 14 10
197892024-04-17T17:23:06.017Z25/30 Read block 529 len 12288
197902024-04-17T17:23:06.017Z26/30 Write block 542 len 24576 data: 21 19 19 15 18 16
197912024-04-17T17:23:06.017Z27/30 Read block 166 len 40960
197922024-04-17T17:23:06.022Z28/30 Write block 552 len 40960 data: 20 15 15 16 12 8 10 13 14 14
197932024-04-17T17:23:06.022Z29/30 Read block 258 len 12288
197942024-04-17T17:23:06.022Z30/30 Read block 352 len 8192
197952024-04-17T17:23:06.022Z----------------------------------------------------------------
197962024-04-17T17:23:06.022Z Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:30
197972024-04-17T17:23:06.022ZStates: Active Active Active
197982024-04-17T17:23:06.022ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
197992024-04-17T17:23:06.022Z 1 Acked 1000 Write 6 Done Done Done false
198002024-04-17T17:23:06.022Z 2 Acked 1001 Read 8 Done Done Done false
198012024-04-17T17:23:06.022Z 3 Acked 1002 Read 10 Done Done Done false
198022024-04-17T17:23:06.022Z 4 Acked 1003 Read 2 Done Done Sent false
198032024-04-17T17:23:06.022Z 5 Acked 1004 Write 6 Done Done Sent false
198042024-04-17T17:23:06.022Z 6 Acked 1005 Read 5 Done Done Sent false
198052024-04-17T17:23:06.022Z 7 Acked 1006 Read 5 Done Done Sent false
198062024-04-17T17:23:06.022Z 8 Acked 1007 Read 7 Done Done Sent false
198072024-04-17T17:23:06.022Z 9 Acked 1008 Write 8 Done Done Sent false
198082024-04-17T17:23:06.023Z 10 Acked 1009 Read 8 Done Done Sent false
198092024-04-17T17:23:06.023Z 11 Acked 1010 Read 8 Done Done Sent false
198102024-04-17T17:23:06.023Z 12 Acked 1011 Flush 0 Done Done Sent false
198112024-04-17T17:23:06.023Z 13 Acked 1012 Write 3 Done Done Sent false
198122024-04-17T17:23:06.023Z 14 Acked 1013 Read 7 Done Done Sent false
198132024-04-17T17:23:06.023Z 15 Acked 1014 Read 7 Done Done Sent false
198142024-04-17T17:23:06.023Z 16 Acked 1015 Write 4 Done Done Sent false
198152024-04-17T17:23:06.023Z 17 Acked 1016 Read 10 Done Done Sent false
198162024-04-17T17:23:06.023Z 18 Acked 1017 Flush 0 Done Done Sent false
198172024-04-17T17:23:06.023Z 19 Acked 1018 Write 4 Done Done Sent false
198182024-04-17T17:23:06.023Z 20 Acked 1019 Read 2 Done Done Sent false
198192024-04-17T17:23:06.023Z 21 Acked 1020 Write 3 Done Done Sent false
198202024-04-17T17:23:06.023Z 22 Acked 1021 Read 1 Done Done Sent false
198212024-04-17T17:23:06.023Z 23 Acked 1022 Write 4 Done Done Sent false
198222024-04-17T17:23:06.023Z 24 Acked 1023 Write 4 Done Done Sent false
198232024-04-17T17:23:06.023Z 25 Acked 1024 Read 3 Done Done Sent false
198242024-04-17T17:23:06.023Z 26 Acked 1025 Write 6 Done Done Sent false
198252024-04-17T17:23:06.023Z 27 Acked 1026 Read 10 Done Done Sent false
198262024-04-17T17:23:06.023Z 28 Acked 1027 Write 10 Done Done Sent false
198272024-04-17T17:23:06.023Z 29 Acked 1028 Read 3 Done Done Sent false
198282024-04-17T17:23:06.023Z 30 Acked 1029 Read 2 Done Done Sent false
198292024-04-17T17:23:06.023Z STATES DS:0 DS:1 DS:2 TOTAL
198302024-04-17T17:23:06.023Z New 0 0 0 0
198312024-04-17T17:23:06.023Z Sent 0 0 27 27
198322024-04-17T17:23:06.024Z Done 30 30 3 63
198332024-04-17T17:23:06.024Z Skipped 0 0 0 0
198342024-04-17T17:23:06.024Z Error 0 0 0 0
198352024-04-17T17:23:06.024ZLast Flush: 1017 1017 0
198362024-04-17T17:23:06.024ZDownstairs last five completed:
198372024-04-17T17:23:06.024ZUpstairs last five completed: 30 29 28 27 26
198382024-04-17T17:23:06.024Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:23:04.630339348Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1907}
198392024-04-17T17:23:06.024ZWrote out file "/tmp/test_repair_verify.data"
198402024-04-17T17:23:06.027Z
198412024-04-17T17:23:06.032ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
198422024-04-17T17:23:06.050ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
198432024-04-17T17:23:06.050Z 0 000-019 183 183 183 324 324 324 F F F
198442024-04-17T17:23:06.057Z 1 020-039 183 183 183 324 324 324 F F F
198452024-04-17T17:23:06.057Z 2 040-059 183 183 183 324 324 324 F F F
198462024-04-17T17:23:06.057Z 3 060-079 184 184 180 325 325 320 F F F <---
198472024-04-17T17:23:06.057Z 4 080-099 174 174 174 304 304 304 F F F
198482024-04-17T17:23:06.057Z 5 100-119 183 183 183 324 324 324 F F F
198492024-04-17T17:23:06.057Z 6 120-139 178 178 178 316 316 316 F F F
198502024-04-17T17:23:06.057Z 7 140-159 184 184 181 325 325 322 F F F <---
198512024-04-17T17:23:06.057Z 8 160-179 178 178 178 318 318 318 F F F
198522024-04-17T17:23:06.057Z 9 180-199 176 176 176 309 309 309 F F F
198532024-04-17T17:23:06.057Z 10 200-219 182 182 182 323 323 323 F F F
198542024-04-17T17:23:06.057Z 11 220-239 182 182 182 323 323 323 F F F
198552024-04-17T17:23:06.058Z 12 240-259 172 172 172 302 302 302 F F F
198562024-04-17T17:23:06.058Z 13 260-279 181 181 181 322 322 322 F F F
198572024-04-17T17:23:06.058Z 14 280-299 181 181 181 322 322 322 F F F
198582024-04-17T17:23:06.058Z 15 300-319 184 184 182 325 325 323 F F T <---
198592024-04-17T17:23:06.058Z 16 320-339 182 182 182 323 323 323 F F F
198602024-04-17T17:23:06.058Z 17 340-359 181 181 181 322 322 322 F F F
198612024-04-17T17:23:06.058Z 18 360-379 180 180 180 320 320 320 T T F <---
198622024-04-17T17:23:06.058Z 19 380-399 177 177 177 314 314 314 F F F
198632024-04-17T17:23:06.058Z 20 400-419 180 180 180 321 321 321 F F F
198642024-04-17T17:23:06.058Z 21 420-439 183 183 183 324 324 324 F F F
198652024-04-17T17:23:06.058Z 22 440-459 184 184 181 326 326 322 T T F <---
198662024-04-17T17:23:06.058Z 23 460-479 166 166 166 291 291 291 F F F
198672024-04-17T17:23:06.058Z 24 480-499 184 184 183 326 326 324 T T F <---
198682024-04-17T17:23:06.058Z 25 500-519 183 183 183 324 324 324 T T F <---
198692024-04-17T17:23:06.058Z 26 520-539 178 178 178 316 316 316 F F F
198702024-04-17T17:23:06.058Z 27 540-559 183 183 183 324 324 324 T T F <---
198712024-04-17T17:23:06.058Z 28 560-579 183 183 183 324 324 324 T T F <---
198722024-04-17T17:23:06.058Z 29 580-599 178 178 178 316 316 316 F F F
198732024-04-17T17:23:06.058ZMax gen: 184, Max flush: 326
198742024-04-17T17:23:06.058ZError: Difference in extent metadata found!
198752024-04-17T17:23:06.058ZA Difference in extent metadata is expected here
198762024-04-17T17:23:06.058ZOn loop 92
198772024-04-17T17:23:06.058Z
198782024-04-17T17:23:06.059ZVerifying data now
198792024-04-17T17:23:10.505ZNo change after verify
198802024-04-17T17:23:10.506ZLoop: 92 Downstairs dump after verify (and repair):
198812024-04-17T17:23:10.523ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
198822024-04-17T17:23:10.523Z 0 000-019 183 183 183 324 324 324 F F F
198832024-04-17T17:23:10.527Z 1 020-039 183 183 183 324 324 324 F F F
198842024-04-17T17:23:10.528Z 2 040-059 183 183 183 324 324 324 F F F
198852024-04-17T17:23:10.528Z 3 060-079 184 184 184 325 325 325 F F F
198862024-04-17T17:23:10.528Z 4 080-099 174 174 174 304 304 304 F F F
198872024-04-17T17:23:10.528Z 5 100-119 183 183 183 324 324 324 F F F
198882024-04-17T17:23:10.528Z 6 120-139 178 178 178 316 316 316 F F F
198892024-04-17T17:23:10.528Z 7 140-159 184 184 184 325 325 325 F F F
198902024-04-17T17:23:10.528Z 8 160-179 178 178 178 318 318 318 F F F
198912024-04-17T17:23:10.528Z 9 180-199 176 176 176 309 309 309 F F F
198922024-04-17T17:23:10.528Z 10 200-219 182 182 182 323 323 323 F F F
198932024-04-17T17:23:10.528Z 11 220-239 182 182 182 323 323 323 F F F
198942024-04-17T17:23:10.528Z 12 240-259 172 172 172 302 302 302 F F F
198952024-04-17T17:23:10.528Z 13 260-279 181 181 181 322 322 322 F F F
198962024-04-17T17:23:10.528Z 14 280-299 181 181 181 322 322 322 F F F
198972024-04-17T17:23:10.528Z 15 300-319 184 184 184 325 325 325 F F F
198982024-04-17T17:23:10.528Z 16 320-339 182 182 182 323 323 323 F F F
198992024-04-17T17:23:10.528Z 17 340-359 181 181 181 322 322 322 F F F
199002024-04-17T17:23:10.528Z 18 360-379 185 185 185 327 327 327 F F F
199012024-04-17T17:23:10.528Z 19 380-399 177 177 177 314 314 314 F F F
199022024-04-17T17:23:10.528Z 20 400-419 180 180 180 321 321 321 F F F
199032024-04-17T17:23:10.528Z 21 420-439 183 183 183 324 324 324 F F F
199042024-04-17T17:23:10.528Z 22 440-459 185 185 185 327 327 327 F F F
199052024-04-17T17:23:10.528Z 23 460-479 166 166 166 291 291 291 F F F
199062024-04-17T17:23:10.528Z 24 480-499 185 185 185 327 327 327 F F F
199072024-04-17T17:23:10.528Z 25 500-519 185 185 185 327 327 327 F F F
199082024-04-17T17:23:10.529Z 26 520-539 178 178 178 316 316 316 F F F
199092024-04-17T17:23:10.529Z 27 540-559 185 185 185 327 327 327 F F F
199102024-04-17T17:23:10.529Z 28 560-579 185 185 185 327 327 327 F F F
199112024-04-17T17:23:10.529Z 29 580-599 178 178 178 316 316 316 F F F
199122024-04-17T17:23:10.529ZMax gen: 185, Max flush: 327
199132024-04-17T17:23:10.529Z
199142024-04-17T17:23:10.529ZBegin loop 93
199152024-04-17T17:23:10.529ZDownstairs to restart: 0
199162024-04-17T17:23:10.544Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.154015066Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915}
199172024-04-17T17:23:10.550Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.154325615Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915}
199182024-04-17T17:23:10.550ZCrucible runtime is spawned
199192024-04-17T17:23:10.550Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.15440517Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915}
199202024-04-17T17:23:10.550Z{"msg":"Crucible 4f4e365d-e8d0-46c5-859e-5b53aac7cd27 has session id: 3e07f7a4-255b-4dd4-b208-14bcdd5e8173","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.154448863Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199212024-04-17T17:23:10.550Z{"msg":"Upstairs opts: Upstairs UUID: 4f4e365d-e8d0-46c5-859e-5b53aac7cd27, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.154498616Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199222024-04-17T17:23:10.551Z{"msg":"Crucible stats registered with UUID: 4f4e365d-e8d0-46c5-859e-5b53aac7cd27","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.154542938Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199232024-04-17T17:23:10.551Z{"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.154596442Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915}
199242024-04-17T17:23:10.551Z{"msg":"4f4e365d-e8d0-46c5-859e-5b53aac7cd27 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.154622483Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199252024-04-17T17:23:10.551Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.154670766Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"client":"0","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199262024-04-17T17:23:10.551Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.15472697Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"client":"0","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199272024-04-17T17:23:10.551Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.154760681Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"client":"1","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199282024-04-17T17:23:10.551Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.154812695Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"client":"1","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199292024-04-17T17:23:10.551Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.154844026Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"client":"2","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199302024-04-17T17:23:10.551Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.154883899Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"client":"2","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199312024-04-17T17:23:10.551Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.154910521Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"":"io task","client":"0","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199322024-04-17T17:23:10.551Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.154939092Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"":"io task","client":"2","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199332024-04-17T17:23:10.551Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.154962754Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"":"io task","client":"1","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199342024-04-17T17:23:10.551Z{"msg":"ds_connection connected from Ok(127.0.0.1:64642)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.155010627Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"":"io task","client":"1","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199352024-04-17T17:23:10.551Z{"msg":"ds_connection connected from Ok(127.0.0.1:65003)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.155053009Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"":"io task","client":"2","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199362024-04-17T17:23:10.551Z{"msg":"ds_connection connected from Ok(127.0.0.1:64069)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.155091561Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"":"io task","client":"0","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199372024-04-17T17:23:10.551Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.155790634Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"client":"1","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199382024-04-17T17:23:10.551Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.155839697Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"client":"1","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199392024-04-17T17:23:10.551Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.155870239Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"client":"2","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199402024-04-17T17:23:10.551Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.155910641Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"client":"2","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199412024-04-17T17:23:10.551Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.155937753Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"client":"0","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199422024-04-17T17:23:10.551Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.155958484Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"client":"0","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199432024-04-17T17:23:10.551Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.156346008Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"client":"1","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199442024-04-17T17:23:10.551Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.156397971Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"client":"2","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199452024-04-17T17:23:10.552Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.156441794Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"client":"0","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199462024-04-17T17:23:10.552Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.156975296Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"client":"1","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199472024-04-17T17:23:10.552Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.157028939Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"client":"1","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199482024-04-17T17:23:10.552Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.157058611Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199492024-04-17T17:23:10.552Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.157086993Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"client":"2","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199502024-04-17T17:23:10.552Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.157113644Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"client":"2","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199512024-04-17T17:23:10.552Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.157133635Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199522024-04-17T17:23:10.552Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.157157527Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"client":"0","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199532024-04-17T17:23:10.552Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.157172108Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"client":"0","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199542024-04-17T17:23:10.552Z{"msg":"[0]R flush_numbers[0..12]: [324, 324, 324, 325, 304, 324, 316, 325, 318, 309, 323, 323]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.157192829Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"":"downstairs","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199552024-04-17T17:23:10.552Z{"msg":"[0]R generation[0..12]: [183, 183, 183, 184, 174, 183, 178, 184, 178, 176, 182, 182]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.15720764Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"":"downstairs","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199562024-04-17T17:23:10.552Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.157228711Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"":"downstairs","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199572024-04-17T17:23:10.552Z{"msg":"[1]R flush_numbers[0..12]: [324, 324, 324, 325, 304, 324, 316, 325, 318, 309, 323, 323]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.157242942Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"":"downstairs","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199582024-04-17T17:23:10.552Z{"msg":"[1]R generation[0..12]: [183, 183, 183, 184, 174, 183, 178, 184, 178, 176, 182, 182]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.157263123Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"":"downstairs","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199592024-04-17T17:23:10.552Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.157277814Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"":"downstairs","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199602024-04-17T17:23:10.552Z{"msg":"[2]R flush_numbers[0..12]: [324, 324, 324, 325, 304, 324, 316, 325, 318, 309, 323, 323]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.157297965Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"":"downstairs","Wait for a query_work_queue command to finish before sending IO
199612024-04-17T17:23:10.552Z":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199622024-04-17T17:23:10.552Z{"msg":"[2]R generation[0..12]: [183, 183, 183, 184, 174, 183, 178, 184, 178, 176, 182, 182]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.157314846Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"":"downstairs","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199632024-04-17T17:23:10.552Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.157335197Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"":"downstairs","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199642024-04-17T17:23:10.552Z{"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.157348148Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"":"downstairs","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199652024-04-17T17:23:10.552Z{"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.157367519Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"":"downstairs","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199662024-04-17T17:23:10.552Z{"msg":"Next flush: 328","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.15738049Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"":"downstairs","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199672024-04-17T17:23:10.552Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.157408792Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"":"downstairs","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199682024-04-17T17:23:10.553Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.157428083Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199692024-04-17T17:23:10.553Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.157450504Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"client":"0","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199702024-04-17T17:23:10.553Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.157463905Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"client":"1","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199712024-04-17T17:23:10.553Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.157488777Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"client":"2","":"downstairs","session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199722024-04-17T17:23:10.553Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.157503908Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199732024-04-17T17:23:10.553Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.157527519Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199742024-04-17T17:23:10.553Z{"msg":"4f4e365d-e8d0-46c5-859e-5b53aac7cd27 is now active with session: 3e07f7a4-255b-4dd4-b208-14bcdd5e8173","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.15754265Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199752024-04-17T17:23:10.553Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.157564782Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915,"session_id":"3e07f7a4-255b-4dd4-b208-14bcdd5e8173"}
199762024-04-17T17:23:10.553Z{"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:09.157577682Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915}
199772024-04-17T17:23:10.553ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
199782024-04-17T17:23:10.553ZLoading write count information from file "/tmp/test_repair_verify.data"
199792024-04-17T17:23:10.553ZRun Repair workload
199802024-04-17T17:23:10.557Z01/30 Write block 44 len 8192 data: 11 14
199812024-04-17T17:23:10.557Z02/30 Read block 549 len 4096
199822024-04-17T17:23:10.557Z03/30 Read block 423 len 36864
199832024-04-17T17:23:10.557Z04/30 Write block 71 len 36864 data: 15 16 12 15 16 15 13 12 12
199842024-04-17T17:23:10.557Z05/30 Write block 198 len 40960 data: 13 10 11 15 19 20 22 23 22 25
199852024-04-17T17:23:10.557Z06/30 Write block 169 len 32768 data: 16 15 17 15 15 12 14 14
199862024-04-17T17:23:10.557Z07/30 Write block 567 len 16384 data: 13 14 12 12
199872024-04-17T17:23:10.558Z08/30 Write block 403 len 8192 data: 12 13
199882024-04-17T17:23:10.558Z09/30 Read block 526 len 40960
199892024-04-17T17:23:10.558Z10/30 Read block 566 len 20480
199902024-04-17T17:23:10.561Z11/30 Read block 22 len 36864
199912024-04-17T17:23:10.561Z12/30 Write block 230 len 36864 data: 16 14 15 14 15 20 16 13 11
199922024-04-17T17:23:10.561Z13/30 Read block 136 len 32768
199932024-04-17T17:23:10.561Z14/30 Flush
199942024-04-17T17:23:10.565Z15/30 Write block 377 len 4096 data: 10
199952024-04-17T17:23:10.565Z16/30 Flush
199962024-04-17T17:23:10.568Z17/30 Write block 152 len 24576 data: 11 11 14 14 15 15
199972024-04-17T17:23:10.569Z18/30 Write block 39 len 36864 data: 15 16 17 16 14 12 15 13 12
199982024-04-17T17:23:10.572Z19/30 Write block 63 len 36864 data: 18 20 19 17 18 18 16 16 16
199992024-04-17T17:23:10.572Z20/30 Flush
200002024-04-17T17:23:10.575Z21/30 Write block 34 len 16384 data: 26 21 18 16
200012024-04-17T17:23:10.575Z22/30 Write block 495 len 32768 data: 15 13 13 16 17 17 18 16
200022024-04-17T17:23:10.575Z23/30 Read block 343 len 24576
200032024-04-17T17:23:10.579Z24/30 Read block 197 len 24576
200042024-04-17T17:23:10.579Z25/30 Read block 47 len 32768
200052024-04-17T17:23:10.579Z26/30 Flush
200062024-04-17T17:23:10.582Z27/30 Write block 144 len 4096 data: 12
200072024-04-17T17:23:10.582Z28/30 Read block 372 len 20480
200082024-04-17T17:23:10.582Z29/30 Write block 16 len 36864 data: 10 13 12 15 16 19 18 16 15
200092024-04-17T17:23:10.583Z30/30 Read block 262 len 20480
200102024-04-17T17:23:10.587Z----------------------------------------------------------------
200112024-04-17T17:23:10.587Z Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:30
200122024-04-17T17:23:10.587ZStates: Active Active Active
200132024-04-17T17:23:10.587ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
200142024-04-17T17:23:10.587Z 1 Acked 1000 Write 2 Sent Done Done false
200152024-04-17T17:23:10.587Z 2 Acked 1001 Read 1 Sent Done Done false
200162024-04-17T17:23:10.587Z 3 Acked 1002 Read 9 Sent Done Done false
200172024-04-17T17:23:10.587Z 4 Acked 1003 Write 9 Sent Done Done false
200182024-04-17T17:23:10.587Z 5 Acked 1004 Write 10 Sent Done Done false
200192024-04-17T17:23:10.587Z 6 Acked 1005 Write 8 Sent Done Done false
200202024-04-17T17:23:10.587Z 7 Acked 1006 Write 4 Sent Done Done false
200212024-04-17T17:23:10.587Z 8 Acked 1007 Write 2 Sent Done Done false
200222024-04-17T17:23:10.587Z 9 Acked 1008 Read 10 Sent Done Done false
200232024-04-17T17:23:10.587Z 10 Acked 1009 Read 5 Sent Done Done false
200242024-04-17T17:23:10.587Z 11 Acked 1010 Read 9 Sent Done Done false
200252024-04-17T17:23:10.587Z 12 Acked 1011 Write 9 Sent Done Done false
200262024-04-17T17:23:10.587Z 13 Acked 1012 Read 8 Sent Done Done false
200272024-04-17T17:23:10.587Z 14 Acked 1013 Flush 0 Sent Done Done false
200282024-04-17T17:23:10.587Z 15 Acked 1014 Write 1 Sent Done Done false
200292024-04-17T17:23:10.587Z 16 Acked 1015 Flush 0 Sent Done Done false
200302024-04-17T17:23:10.587Z 17 Acked 1016 Write 6 Sent Done Done false
200312024-04-17T17:23:10.588Z 18 Acked 1017 Write 9 Sent Done Done false
200322024-04-17T17:23:10.588Z 19 Acked 1018 Write 9 Sent Done Done false
200332024-04-17T17:23:10.588Z 20 Acked 1019 Flush 0 Sent Done Done false
200342024-04-17T17:23:10.588Z 21 Acked 1020 Write 4 Sent Done Done false
200352024-04-17T17:23:10.588Z 22 Acked 1021 Write 8 Sent Done Done false
200362024-04-17T17:23:10.588Z 23 Acked 1022 Read 6 Sent Done Done false
200372024-04-17T17:23:10.588Z 24 Acked 1023 Read 6 Sent Done Done false
200382024-04-17T17:23:10.588Z 25 Acked 1024 Read 8 Sent Done Done false
200392024-04-17T17:23:10.588Z 26 Acked 1025 Flush 0 Sent Done Done false
200402024-04-17T17:23:10.588Z 27 Acked 1026 Write 1 Sent Done Done false
200412024-04-17T17:23:10.588Z 28 Acked 1027 Read 5 Sent Done Done false
200422024-04-17T17:23:10.588Z 29 Acked 1028 Write 9 Sent Done Done false
200432024-04-17T17:23:10.588Z 30 Acked 1029 Read 5 Sent Done Sent false
200442024-04-17T17:23:10.588Z STATES DS:0 DS:1 DS:2 TOTAL
200452024-04-17T17:23:10.588Z New 0 0 0 0
200462024-04-17T17:23:10.588Z Sent 30 0 1 31
200472024-04-17T17:23:10.588Z Done 0 30 29 59
200482024-04-17T17:23:10.588Z Skipped 0 0 0 0
200492024-04-17T17:23:10.588Z Error 0 0 0 0
200502024-04-17T17:23:10.588ZLast Flush: 0 1025 1025
200512024-04-17T17:23:10.588ZDownstairs last five completed:
200522024-04-17T17:23:10.588ZUpstairs last five completed: 30 29 28 27 26
200532024-04-17T17:23:10.588Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:23:09.194112977Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1915}
200542024-04-17T17:23:10.588ZWrote out file "/tmp/test_repair_verify.data"
200552024-04-17T17:23:10.592Z
200562024-04-17T17:23:10.592ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
200572024-04-17T17:23:10.608ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
200582024-04-17T17:23:10.608Z 0 000-019 183 183 183 324 324 324 F T T <---
200592024-04-17T17:23:10.608Z 1 020-039 183 186 186 324 331 331 F T T <---
200602024-04-17T17:23:10.608Z 2 040-059 183 186 186 324 330 330 F F F <---
200612024-04-17T17:23:10.608Z 3 060-079 184 186 186 325 330 330 F F F <---
200622024-04-17T17:23:10.608Z 4 080-099 174 174 174 304 304 304 F F F
200632024-04-17T17:23:10.612Z 5 100-119 183 183 183 324 324 324 F F F
200642024-04-17T17:23:10.612Z 6 120-139 178 178 178 316 316 316 F F F
200652024-04-17T17:23:10.612Z 7 140-159 184 186 186 325 330 330 F T T <---
200662024-04-17T17:23:10.612Z 8 160-179 178 186 186 318 328 328 F F F <---
200672024-04-17T17:23:10.612Z 9 180-199 176 186 186 309 328 328 F F F <---
200682024-04-17T17:23:10.612Z 10 200-219 182 186 186 323 328 328 F F F <---
200692024-04-17T17:23:10.612Z 11 220-239 182 186 186 323 328 328 F F F <---
200702024-04-17T17:23:10.612Z 12 240-259 172 172 172 302 302 302 F F F
200712024-04-17T17:23:10.612Z 13 260-279 181 181 181 322 322 322 F F F
200722024-04-17T17:23:10.612Z 14 280-299 181 181 181 322 322 322 F F F
200732024-04-17T17:23:10.612Z 15 300-319 184 184 184 325 325 325 F F F
200742024-04-17T17:23:10.612Z 16 320-339 182 182 182 323 323 323 F F F
200752024-04-17T17:23:10.612Z 17 340-359 181 181 181 322 322 322 F F F
200762024-04-17T17:23:10.612Z 18 360-379 185 186 186 327 329 329 F F F <---
200772024-04-17T17:23:10.612Z 19 380-399 177 177 177 314 314 314 F F F
200782024-04-17T17:23:10.612Z 20 400-419 180 186 186 321 328 328 F F F <---
200792024-04-17T17:23:10.613Z 21 420-439 183 183 183 324 324 324 F F F
200802024-04-17T17:23:10.613Z 22 440-459 185 185 185 327 327 327 F F F
200812024-04-17T17:23:10.613Z 23 460-479 166 166 166 291 291 291 F F F
200822024-04-17T17:23:10.613Z 24 480-499 185 186 186 327 331 331 F F F <---
200832024-04-17T17:23:10.613Z 25 500-519 185 186 186 327 331 331 F F F <---
200842024-04-17T17:23:10.613Z 26 520-539 178 178 178 316 316 316 F F F
200852024-04-17T17:23:10.613Z 27 540-559 185 185 185 327 327 327 F F F
200862024-04-17T17:23:10.613Z 28 560-579 185 186 186 327 328 328 F F F <---
200872024-04-17T17:23:10.613Z 29 580-599 178 178 178 316 316 316 F F F
200882024-04-17T17:23:10.613ZMax gen: 186, Max flush: 331
200892024-04-17T17:23:10.613ZError: Difference in extent metadata found!
200902024-04-17T17:23:10.613ZA Difference in extent metadata is expected here
200912024-04-17T17:23:10.613ZOn loop 93
200922024-04-17T17:23:10.613Z
200932024-04-17T17:23:10.617ZVerifying data now
200942024-04-17T17:23:15.258ZNo change after verify
200952024-04-17T17:23:15.258ZLoop: 93 Downstairs dump after verify (and repair):
200962024-04-17T17:23:15.273ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
200972024-04-17T17:23:15.273Z 0 000-019 187 187 187 332 332 332 F F F
200982024-04-17T17:23:15.273Z 1 020-039 187 187 187 332 332 332 F F F
200992024-04-17T17:23:15.273Z 2 040-059 186 186 186 330 330 330 F F F
201002024-04-17T17:23:15.273Z 3 060-079 186 186 186 330 330 330 F F F
201012024-04-17T17:23:15.273Z 4 080-099 174 174 174 304 304 304 F F F
201022024-04-17T17:23:15.281Z 5 100-119 183 183 183 324 324 324 F F F
201032024-04-17T17:23:15.281Z 6 120-139 178 178 178 316 316 316 F F F
201042024-04-17T17:23:15.281Z 7 140-159 187 187 187 332 332 332 F F F
201052024-04-17T17:23:15.281Z 8 160-179 186 186 186 328 328 328 F F F
201062024-04-17T17:23:15.281Z 9 180-199 186 186 186 328 328 328 F F F
201072024-04-17T17:23:15.281Z 10 200-219 186 186 186 328 328 328 F F F
201082024-04-17T17:23:15.281Z 11 220-239 186 186 186 328 328 328 F F F
201092024-04-17T17:23:15.281Z 12 240-259 172 172 172 302 302 302 F F F
201102024-04-17T17:23:15.281Z 13 260-279 181 181 181 322 322 322 F F F
201112024-04-17T17:23:15.281Z 14 280-299 181 181 181 322 322 322 F F F
201122024-04-17T17:23:15.281Z 15 300-319 184 184 184 325 325 325 F F F
201132024-04-17T17:23:15.281Z 16 320-339 182 182 182 323 323 323 F F F
201142024-04-17T17:23:15.281Z 17 340-359 181 181 181 322 322 322 F F F
201152024-04-17T17:23:15.282Z 18 360-379 186 186 186 329 329 329 F F F
201162024-04-17T17:23:15.282Z 19 380-399 177 177 177 314 314 314 F F F
201172024-04-17T17:23:15.282Z 20 400-419 186 186 186 328 328 328 F F F
201182024-04-17T17:23:15.282Z 21 420-439 183 183 183 324 324 324 F F F
201192024-04-17T17:23:15.282Z 22 440-459 185 185 185 327 327 327 F F F
201202024-04-17T17:23:15.282Z 23 460-479 166 166 166 291 291 291 F F F
201212024-04-17T17:23:15.282Z 24 480-499 186 186 186 331 331 331 F F F
201222024-04-17T17:23:15.282Z 25 500-519 186 186 186 331 331 331 F F F
201232024-04-17T17:23:15.282Z 26 520-539 178 178 178 316 316 316 F F F
201242024-04-17T17:23:15.282Z 27 540-559 185 185 185 327 327 327 F F F
201252024-04-17T17:23:15.282Z 28 560-579 186 186 186 328 328 328 F F F
201262024-04-17T17:23:15.282Z 29 580-599 178 178 178 316 316 316 F F F
201272024-04-17T17:23:15.282ZMax gen: 187, Max flush: 332
201282024-04-17T17:23:15.282Z
201292024-04-17T17:23:15.282ZBegin loop 94
201302024-04-17T17:23:15.282ZDownstairs to restart: 0
201312024-04-17T17:23:15.296Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.906786809Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923}
201322024-04-17T17:23:15.297Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.907108458Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923}
201332024-04-17T17:23:15.303Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30Crucible runtime is spawned
201342024-04-17T17:23:15.303Z,"time":"2024-04-17T17:23:13.90714468Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923}
201352024-04-17T17:23:15.303Z{"msg":"Crucible 62c9e1a4-cc14-4b3e-81d6-acb491897a3b has session id: 92304004-fd04-4c44-b8c4-ba507fb6e4f2","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.907229125Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201362024-04-17T17:23:15.303Z{"msg":"Upstairs opts: Upstairs UUID: 62c9e1a4-cc14-4b3e-81d6-acb491897a3b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.90730663Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201372024-04-17T17:23:15.303Z{"msg":"Crucible stats registered with UUID: 62c9e1a4-cc14-4b3e-81d6-acb491897a3b","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.907348273Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201382024-04-17T17:23:15.303Z{"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.907364884Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923}
201392024-04-17T17:23:15.303Z{"msg":"62c9e1a4-cc14-4b3e-81d6-acb491897a3b active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.907383195Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201402024-04-17T17:23:15.303Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.907415087Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"client":"0","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201412024-04-17T17:23:15.303Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.907449619Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"client":"0","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201422024-04-17T17:23:15.304Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.907491041Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"client":"1","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201432024-04-17T17:23:15.304Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.907525103Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"client":"1","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201442024-04-17T17:23:15.304Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.907548945Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"client":"2","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201452024-04-17T17:23:15.304Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.907585757Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"client":"2","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201462024-04-17T17:23:15.304Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.907611009Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"":"io task","client":"2","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201472024-04-17T17:23:15.304Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.907639921Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"":"io task","client":"0","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201482024-04-17T17:23:15.304Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.907722455Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"":"io task","client":"1","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201492024-04-17T17:23:15.304Z{"msg":"ds_connection connected from Ok(127.0.0.1:52644)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.907781079Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"":"io task","client":"0","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201502024-04-17T17:23:15.304Z{"msg":"ds_connection connected from Ok(127.0.0.1:45517)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.907810821Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"":"io task","client":"2","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201512024-04-17T17:23:15.304Z{"msg":"ds_connection connected from Ok(127.0.0.1:52903)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.907853723Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"":"io task","client":"1","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201522024-04-17T17:23:15.304Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.908595578Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"client":"1","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201532024-04-17T17:23:15.304Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.90863087Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"client":"1","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201542024-04-17T17:23:15.304Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.908646931Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"client":"0","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201552024-04-17T17:23:15.304Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.908737327Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"client":"0","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201562024-04-17T17:23:15.304Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.90878732Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"client":"2","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201572024-04-17T17:23:15.304Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.908840433Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"client":"2","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201582024-04-17T17:23:15.304Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.90912278Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"client":"2","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201592024-04-17T17:23:15.304Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.909263859Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"client":"1","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201602024-04-17T17:23:15.304Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.909369765Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"client":"0","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201612024-04-17T17:23:15.304Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.909665343Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"client":"2","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201622024-04-17T17:23:15.304Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.909707696Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"client":"2","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201632024-04-17T17:23:15.305Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.909724587Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201642024-04-17T17:23:15.305Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.909742338Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"client":"1","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201652024-04-17T17:23:15.305Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.909763449Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"client":"1","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201662024-04-17T17:23:15.305Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.90977747Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201672024-04-17T17:23:15.305Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.909908578Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"client":"0","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201682024-04-17T17:23:15.305Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.90994306Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"client":"0","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201692024-04-17T17:23:15.305Z{"msg":"[0]R flush_numbers[0..12]: [332, 332, 330, 330, 304, 324, 316, 332, 328, 328, 328, 328]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.909967642Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"":"downstairs","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201702024-04-17T17:23:15.305Z{"msg":"[0]R generation[0..12]: [187, 187, 186, 186, 174, 183, 178, 187, 186, 186, 186, 186]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.909993653Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"":"downstairs","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201712024-04-17T17:23:15.305Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.910008734Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"":"downstairs","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201722024-04-17T17:23:15.305Z{"msg":"[1]R flush_numbers[0..12]: [332, 332, 330, 330, 304, 324, 316, 332, 328, 328, 328, 328]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.910022735Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"":"downstairs","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201732024-04-17T17:23:15.305Z{"msg":"[1]R generation[0..12]: [187, 187, 186, 186, 174, 183, 178, 187, 186, 186, 186, 186]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.910043456Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"":"downstairs","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201742024-04-17T17:23:15.305Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.910057827Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"":"downstairs","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201752024-04-17T17:23:15.305Z{"msg":"[2]R flush_numbers[0..12]: [332, 332, 330, 330, 304, 324, 316, 332, 328, 328, 328, 328]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.910071858Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"":"downstairs","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201762024-04-17T17:23:15.305Z{"msg":"[2]R generation[0..12]: [187, 187, 186, 186, 174, 183, 178, 187, 186, 186, 186, 186]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.910085499Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"":"downstairs","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201772024-04-17T17:23:15.305Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.91010027Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"":"downstairs","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201782024-04-17T17:23:15.305Z{"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.91011322Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"":"downstairs","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201792024-04-17T17:23:15.305Z{"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
201802024-04-17T17:23:15.305Z,"time":"2024-04-17T17:23:13.910133301Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"":"downstairs","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201812024-04-17T17:23:15.305Z{"msg":"Next flush: 333","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.910146742Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"":"downstairs","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201822024-04-17T17:23:15.305Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.910159533Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"":"downstairs","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201832024-04-17T17:23:15.305Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.910179434Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201842024-04-17T17:23:15.305Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.910191495Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"client":"0","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201852024-04-17T17:23:15.305Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.910204396Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"client":"1","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201862024-04-17T17:23:15.305Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.910224197Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"client":"2","":"downstairs","session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201872024-04-17T17:23:15.306Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.910237398Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201882024-04-17T17:23:15.306Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.910249079Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201892024-04-17T17:23:15.306Z{"msg":"62c9e1a4-cc14-4b3e-81d6-acb491897a3b is now active with session: 92304004-fd04-4c44-b8c4-ba507fb6e4f2","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.91026809Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201902024-04-17T17:23:15.306Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.910280181Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923,"session_id":"92304004-fd04-4c44-b8c4-ba507fb6e4f2"}
201912024-04-17T17:23:15.306Z{"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:13.910292181Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923}
201922024-04-17T17:23:15.306ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
201932024-04-17T17:23:15.306ZLoading write count information from file "/tmp/test_repair_verify.data"
201942024-04-17T17:23:15.306ZRun Repair workload
201952024-04-17T17:23:15.312Z01/30 Write block 365 len 36864 data: 12 11 11 16 15 16 17 14 15
201962024-04-17T17:23:15.312Z02/30 Write block 144 len 8192 data: 13 15
201972024-04-17T17:23:15.312Z03/30 Write block 558 len 16384 data: 11 14 15 15
201982024-04-17T17:23:15.312Z04/30 Read block 340 len 40960
201992024-04-17T17:23:15.312Z05/30 Read block 596 len 12288
202002024-04-17T17:23:15.312Z06/30 Read block 311 len 8192
202012024-04-17T17:23:15.312Z07/30 Write block 418 len 24576 data: 13 14 13 9 11 10
202022024-04-17T17:23:15.312Z08/30 Read block 111 len 20480
202032024-04-17T17:23:15.313Z09/30 Read block 572 len 8192
202042024-04-17T17:23:15.316Z10/30 Write block 307 len 40960 data: 16 17 16 17 15 18 19 22 21 19
202052024-04-17T17:23:15.316Z11/30 Write block 13 len 4096 data: 15
202062024-04-17T17:23:15.316Z12/30 Write block 456 len 20480 data: 11 11 13 15 16
202072024-04-17T17:23:15.316Z13/30 Read block 489 len 12288
202082024-04-17T17:23:15.316Z14/30 Read block 168 len 40960
202092024-04-17T17:23:15.316Z15/30 Read block 486 len 36864
202102024-04-17T17:23:15.316Z16/30 Read block 326 len 28672
202112024-04-17T17:23:15.320Z17/30 Write block 343 len 40960 data: 19 20 21 20 18 15 15 15 10 10
202122024-04-17T17:23:15.320Z18/30 Read block 239 len 8192
202132024-04-17T17:23:15.320Z19/30 Read block 92 len 8192
202142024-04-17T17:23:15.320Z20/30 Read block 553 len 12288
202152024-04-17T17:23:15.320Z21/30 Read block 284 len 40960
202162024-04-17T17:23:15.321Z22/30 Read block 525 len 12288
202172024-04-17T17:23:15.324Z23/30 Read block 52 len 20480
202182024-04-17T17:23:15.324Z24/30 Write block 105 len 32768 data: 14 13 13 15 15 15 17 17
202192024-04-17T17:23:15.324Z25/30 Write block 425 len 36864 data: 11 11 12 11 13 13 14 18 18
202202024-04-17T17:23:15.324Z26/30 Write block 197 len 20480 data: 10 14 11 12 16
202212024-04-17T17:23:15.324Z27/30 Read block 168 len 36864
202222024-04-17T17:23:15.328Z28/30 Read block 488 len 32768
202232024-04-17T17:23:15.328Z29/30 Read block 36 len 8192
202242024-04-17T17:23:15.329Z30/30 Read block 186 len 40960
202252024-04-17T17:23:15.329Z----------------------------------------------------------------
202262024-04-17T17:23:15.329Z Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:30
202272024-04-17T17:23:15.329ZStates: Active Active Active
202282024-04-17T17:23:15.329ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
202292024-04-17T17:23:15.329Z 1 Acked 1000 Write 9 Done Done Done false
202302024-04-17T17:23:15.329Z 2 Acked 1001 Write 2 Done Done Done false
202312024-04-17T17:23:15.329Z 3 Acked 1002 Write 4 Done Done Done false
202322024-04-17T17:23:15.329Z 4 Acked 1003 Read 10 Done Done Done false
202332024-04-17T17:23:15.329Z 5 Acked 1004 Read 3 Done Done Done false
202342024-04-17T17:23:15.329Z 6 Acked 1005 Read 2 Done Done Done false
202352024-04-17T17:23:15.329Z 7 Acked 1006 Write 6 Sent Done Done false
202362024-04-17T17:23:15.329Z 8 Acked 1007 Read 5 Sent Done Done false
202372024-04-17T17:23:15.329Z 9 Acked 1008 Read 2 Sent Done Done false
202382024-04-17T17:23:15.329Z 10 Acked 1009 Write 10 Sent Done Done false
202392024-04-17T17:23:15.329Z 11 Acked 1010 Write 1 Sent Done Done false
202402024-04-17T17:23:15.329Z 12 Acked 1011 Write 5 Sent Done Done false
202412024-04-17T17:23:15.329Z 13 Acked 1012 Read 3 Sent Done Done false
202422024-04-17T17:23:15.329Z 14 Acked 1013 Read 10 Sent Done Done false
202432024-04-17T17:23:15.329Z 15 Acked 1014 Read 9 Sent Done Done false
202442024-04-17T17:23:15.329Z 16 Acked 1015 Read 7 Sent Done Done false
202452024-04-17T17:23:15.329Z 17 Acked 1016 Write 10 Sent Done Done false
202462024-04-17T17:23:15.329Z 18 Acked 1017 Read 2 Sent Done Done false
202472024-04-17T17:23:15.330Z 19 Acked 1018 Read 2 Sent Done Done false
202482024-04-17T17:23:15.330Z 20 Acked 1019 Read 3 Sent Done Done false
202492024-04-17T17:23:15.330Z 21 Acked 1020 Read 10 Sent Done Done false
202502024-04-17T17:23:15.330Z 22 Acked 1021 Read 3 Sent Done Done false
202512024-04-17T17:23:15.330Z 23 Acked 1022 Read 5 Sent Done Done false
202522024-04-17T17:23:15.330Z 24 Acked 1023 Write 8 Sent Done Done false
202532024-04-17T17:23:15.330Z 25 Acked 1024 Write 9 Sent Done Done false
202542024-04-17T17:23:15.330Z 26 Acked 1025 Write 5 Sent Done Done false
202552024-04-17T17:23:15.330Z 27 Acked 1026 Read 9 Sent Done Done false
202562024-04-17T17:23:15.330Z 28 Acked 1027 Read 8 Sent Done Done false
202572024-04-17T17:23:15.330Z 29 Acked 1028 Read 2 Sent Done Done false
202582024-04-17T17:23:15.330Z 30 Acked 1029 Read 10 Sent Sent Done false
202592024-04-17T17:23:15.330Z STATES DS:0 DS:1 DS:2 TOTAL
202602024-04-17T17:23:15.330Z New 0 0 0 0
202612024-04-17T17:23:15.330Z Sent 24 1 0 25
202622024-04-17T17:23:15.330Z Done 6 29 30 65
202632024-04-17T17:23:15.330Z Skipped 0 0 0 0
202642024-04-17T17:23:15.330Z Error 0 0 0 0
202652024-04-17T17:23:15.330ZLast Flush: 0 0 0
202662024-04-17T17:23:15.330ZDownstairs last five completed:
202672024-04-17T17:23:15.330ZUpstairs last five completed: 30 29 28 27 26
202682024-04-17T17:23:15.330Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:23:13.938033372Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1923}
202692024-04-17T17:23:15.330ZWrote out file "/tmp/test_repair_verify.data"
202702024-04-17T17:23:15.337Z
202712024-04-17T17:23:15.337ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
202722024-04-17T17:23:15.356ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
202732024-04-17T17:23:15.356Z 0 000-019 187 187 187 332 332 332 F T T <---
202742024-04-17T17:23:15.356Z 1 020-039 187 187 187 332 332 332 F F F
202752024-04-17T17:23:15.356Z 2 040-059 186 186 186 330 330 330 F F F
202762024-04-17T17:23:15.356Z 3 060-079 186 186 186 330 330 330 F F F
202772024-04-17T17:23:15.357Z 4 080-099 174 174 174 304 304 304 F F F
202782024-04-17T17:23:15.357Z 5 100-119 183 183 183 324 324 324 F T T <---
202792024-04-17T17:23:15.357Z 6 120-139 178 178 178 316 316 316 F F F
202802024-04-17T17:23:15.357Z 7 140-159 187 187 187 332 332 332 T T T
202812024-04-17T17:23:15.365Z 8 160-179 186 186 186 328 328 328 F F F
202822024-04-17T17:23:15.365Z 9 180-199 186 186 186 328 328 328 F T T <---
202832024-04-17T17:23:15.365Z 10 200-219 186 186 186 328 328 328 F T T <---
202842024-04-17T17:23:15.365Z 11 220-239 186 186 186 328 328 328 F F F
202852024-04-17T17:23:15.365Z 12 240-259 172 172 172 302 302 302 F F F
202862024-04-17T17:23:15.365Z 13 260-279 181 181 181 322 322 322 F F F
202872024-04-17T17:23:15.365Z 14 280-299 181 181 181 322 322 322 F F F
202882024-04-17T17:23:15.365Z 15 300-319 184 184 184 325 325 325 F T T <---
202892024-04-17T17:23:15.365Z 16 320-339 182 182 182 323 323 323 F F F
202902024-04-17T17:23:15.365Z 17 340-359 181 181 181 322 322 322 F T T <---
202912024-04-17T17:23:15.365Z 18 360-379 186 186 186 329 329 329 T T T
202922024-04-17T17:23:15.365Z 19 380-399 177 177 177 314 314 314 F F F
202932024-04-17T17:23:15.365Z 20 400-419 186 186 186 328 328 328 F T T <---
202942024-04-17T17:23:15.365Z 21 420-439 183 183 183 324 324 324 F T T <---
202952024-04-17T17:23:15.365Z 22 440-459 185 185 185 327 327 327 F T T <---
202962024-04-17T17:23:15.366Z 23 460-479 166 166 166 291 291 291 F T T <---
202972024-04-17T17:23:15.366Z 24 480-499 186 186 186 331 331 331 F F F
202982024-04-17T17:23:15.366Z 25 500-519 186 186 186 331 331 331 F F F
202992024-04-17T17:23:15.366Z 26 520-539 178 178 178 316 316 316 F F F
203002024-04-17T17:23:15.366Z 27 540-559 185 185 185 327 327 327 T T T
203012024-04-17T17:23:15.366Z 28 560-579 186 186 186 328 328 328 T T T
203022024-04-17T17:23:15.366Z 29 580-599 178 178 178 316 316 316 F F F
203032024-04-17T17:23:15.366ZMax gen: 187, Max flush: 332
203042024-04-17T17:23:15.366ZError: Difference in extent metadata found!
203052024-04-17T17:23:15.366ZA Difference in extent metadata is expected here
203062024-04-17T17:23:15.366ZOn loop 94
203072024-04-17T17:23:15.366Z
203082024-04-17T17:23:15.366ZVerifying data now
203092024-04-17T17:23:30.046ZNo change after verify
203102024-04-17T17:23:30.046ZLoop: 94 Downstairs dump after verify (and repair):
203112024-04-17T17:23:30.059ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
203122024-04-17T17:23:30.060Z 0 000-019 188 188 188 333 333 333 F F F
203132024-04-17T17:23:30.064Z 1 020-039 187 187 187 332 332 332 F F F
203142024-04-17T17:23:30.064Z 2 040-059 186 186 186 330 330 330 F F F
203152024-04-17T17:23:30.064Z 3 060-079 186 186 186 330 330 330 F F F
203162024-04-17T17:23:30.064Z 4 080-099 174 174 174 304 304 304 F F F
203172024-04-17T17:23:30.064Z 5 100-119 188 188 188 333 333 333 F F F
203182024-04-17T17:23:30.064Z 6 120-139 178 178 178 316 316 316 F F F
203192024-04-17T17:23:30.064Z 7 140-159 188 188 188 333 333 333 F F F
203202024-04-17T17:23:30.064Z 8 160-179 186 186 186 328 328 328 F F F
203212024-04-17T17:23:30.064Z 9 180-199 188 188 188 333 333 333 F F F
203222024-04-17T17:23:30.064Z 10 200-219 188 188 188 333 333 333 F F F
203232024-04-17T17:23:30.064Z 11 220-239 186 186 186 328 328 328 F F F
203242024-04-17T17:23:30.064Z 12 240-259 172 172 172 302 302 302 F F F
203252024-04-17T17:23:30.064Z 13 260-279 181 181 181 322 322 322 F F F
203262024-04-17T17:23:30.065Z 14 280-299 181 181 181 322 322 322 F F F
203272024-04-17T17:23:30.065Z 15 300-319 188 188 188 333 333 333 F F F
203282024-04-17T17:23:30.065Z 16 320-339 182 182 182 323 323 323 F F F
203292024-04-17T17:23:30.065Z 17 340-359 188 188 188 333 333 333 F F F
203302024-04-17T17:23:30.065Z 18 360-379 188 188 188 333 333 333 F F F
203312024-04-17T17:23:30.065Z 19 380-399 177 177 177 314 314 314 F F F
203322024-04-17T17:23:30.065Z 20 400-419 188 188 188 333 333 333 F F F
203332024-04-17T17:23:30.065Z 21 420-439 188 188 188 333 333 333 F F F
203342024-04-17T17:23:30.065Z 22 440-459 188 188 188 333 333 333 F F F
203352024-04-17T17:23:30.065Z 23 460-479 188 188 188 333 333 333 F F F
203362024-04-17T17:23:30.065Z 24 480-499 186 186 186 331 331 331 F F F
203372024-04-17T17:23:30.065Z 25 500-519 186 186 186 331 331 331 F F F
203382024-04-17T17:23:30.065Z 26 520-539 178 178 178 316 316 316 F F F
203392024-04-17T17:23:30.065Z 27 540-559 188 188 188 333 333 333 F F F
203402024-04-17T17:23:30.065Z 28 560-579 188 188 188 333 333 333 F F F
203412024-04-17T17:23:30.065Z 29 580-599 178 178 178 316 316 316 F F F
203422024-04-17T17:23:30.065ZMax gen: 188, Max flush: 333
203432024-04-17T17:23:30.065Z
203442024-04-17T17:23:30.065ZBegin loop 95
203452024-04-17T17:23:30.065ZDownstairs to restart: 2
203462024-04-17T17:23:30.083Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.693530011Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931}
203472024-04-17T17:23:30.083Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.693815048Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931}
203482024-04-17T17:23:30.089Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.693858301Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931}
203492024-04-17T17:23:30.089Z{"msg":Crucible runtime is spawned
203502024-04-17T17:23:30.090Z"Crucible 3d71387d-fe9a-4bb6-ae90-c1839db248a7 has session id: 8880708b-a835-402e-86fe-507324921abe","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.693897933Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"session_id":"8880708b-a835-402e-86fe-507324921abe"}
203512024-04-17T17:23:30.090Z{"msg":"Upstairs opts: Upstairs UUID: 3d71387d-fe9a-4bb6-ae90-c1839db248a7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.693971868Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"session_id":"8880708b-a835-402e-86fe-507324921abe"}
203522024-04-17T17:23:30.090Z{"msg":"Crucible stats registered with UUID: 3d71387d-fe9a-4bb6-ae90-c1839db248a7","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.694018021Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"session_id":"8880708b-a835-402e-86fe-507324921abe"}
203532024-04-17T17:23:30.090Z{"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.694038092Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931}
203542024-04-17T17:23:30.090Z{"msg":"3d71387d-fe9a-4bb6-ae90-c1839db248a7 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.694081835Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"session_id":"8880708b-a835-402e-86fe-507324921abe"}
203552024-04-17T17:23:30.090Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.694109836Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"client":"0","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203562024-04-17T17:23:30.090Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.694142328Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"client":"0","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203572024-04-17T17:23:30.090Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.69417124Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"client":"1","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203582024-04-17T17:23:30.090Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.694215163Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"client":"1","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203592024-04-17T17:23:30.090Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.694304648Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"client":"2","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203602024-04-17T17:23:30.090Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.69433226Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"client":"2","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203612024-04-17T17:23:30.090Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.694362432Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"":"io task","client":"2","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203622024-04-17T17:23:30.090Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.694389813Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"":"io task","client":"0","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203632024-04-17T17:23:30.090Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.694421425Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"":"io task","client":"1","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203642024-04-17T17:23:30.090Z{"msg":"ds_connection connected from Ok(127.0.0.1:64491)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.694447257Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"":"io task","client":"1","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203652024-04-17T17:23:30.090Z{"msg":"ds_connection connected from Ok(127.0.0.1:36838)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.694649399Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"":"io task","client":"0","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203662024-04-17T17:23:30.090Z{"msg":"ds_connection connected from Ok(127.0.0.1:38968)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.694703042Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"":"io task","client":"2","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203672024-04-17T17:23:30.090Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.695201272Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"client":"0","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203682024-04-17T17:23:30.090Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.695259796Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"client":"0","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203692024-04-17T17:23:30.090Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.695315349Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"client":"1","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203702024-04-17T17:23:30.090Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.695350692Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"client":"1","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203712024-04-17T17:23:30.090Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.695475169Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"client":"2","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203722024-04-17T17:23:30.090Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.695509671Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"client":"2","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203732024-04-17T17:23:30.091Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.695887984Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"client":"0","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203742024-04-17T17:23:30.091Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.695943777Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"client":"1","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203752024-04-17T17:23:30.091Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.696095457Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"client":"2","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203762024-04-17T17:23:30.091Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.696457108Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"client":"1","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203772024-04-17T17:23:30.091Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.696506282Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"client":"1","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203782024-04-17T17:23:30.091Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.696543074Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"session_id":"8880708b-a835-402e-86fe-507324921abe"}
203792024-04-17T17:23:30.091Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.696557605Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"client":"0","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203802024-04-17T17:23:30.091Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.696571546Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"client":"0","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203812024-04-17T17:23:30.091Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.696597437Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"session_id":"8880708b-a835-402e-86fe-507324921abe"}
203822024-04-17T17:23:30.091Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.696609978Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"client":"2","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203832024-04-17T17:23:30.091Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.696627879Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"client":"2","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203842024-04-17T17:23:30.091Z{"msg":"[0]R flush_numbers[0..12]: [333, 332, 330, 330, 304, 333, 316, 333, 328, 333, 333, 328]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.696663651Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"":"downstairs","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203852024-04-17T17:23:30.091Z{"msg":"[0]R generation[0..12]: [188, 187, 186, 186, 174, 188, 178, 188, 186, 188, 188, 186]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.696681442Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"":"downstairs","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203862024-04-17T17:23:30.091Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.696703454Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"":"downstairs","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203872024-04-17T17:23:30.091Z{"msg":"[1]R flush_numbers[0..12]: [333, 332, 330, 330, 304, 333, 316, 333, 328, 333, 333, 328]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.696717714Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"":"downstairs","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203882024-04-17T17:23:30.091Z{"msg":"[1]R generation[0..12]: [188, 187, 186, 186, 174, 188, 178, 188, 186, 188, 188, 186]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.696737816Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"":"downstairs","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203892024-04-17T17:23:30.091Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.696751757Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"":"downstairs","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203902024-04-17T17:23:30.091Z{"msg":"[2]R flush_numbers[0..12]: [333, 332, 330, 330, 304, 333, 316, 333, 328, 333, 333, 328]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.696765457Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"":"downstairs","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203912024-04-17T17:23:30.091Z{"msg":"[2]R generation[0..12]: [188, 187, 186, 186, 174, 188, 178, 188, 186, 188, 188, 186]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.696778998Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"":"downstairs","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203922024-04-17T17:23:30.091Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO
203932024-04-17T17:23:30.091Ztime":"2024-04-17T17:23:28.69680561Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"":"downstairs","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203942024-04-17T17:23:30.091Z{"msg":"Max found gen is 189","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.696823051Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"":"downstairs","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203952024-04-17T17:23:30.091Z{"msg":"Generation requested: 190 >= found:189","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.696836222Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"":"downstairs","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203962024-04-17T17:23:30.092Z{"msg":"Next flush: 334","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.696856303Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"":"downstairs","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203972024-04-17T17:23:30.092Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.696869264Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"":"downstairs","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
203982024-04-17T17:23:30.092Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.696882224Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"session_id":"8880708b-a835-402e-86fe-507324921abe"}
203992024-04-17T17:23:30.092Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.696900755Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"client":"0","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
204002024-04-17T17:23:30.092Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.696913806Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"client":"1","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
204012024-04-17T17:23:30.092Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.696926497Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"client":"2","":"downstairs","session_id":"8880708b-a835-402e-86fe-507324921abe"}
204022024-04-17T17:23:30.092Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.696952329Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"session_id":"8880708b-a835-402e-86fe-507324921abe"}
204032024-04-17T17:23:30.092Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.69697836Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"session_id":"8880708b-a835-402e-86fe-507324921abe"}
204042024-04-17T17:23:30.092Z{"msg":"3d71387d-fe9a-4bb6-ae90-c1839db248a7 is now active with session: 8880708b-a835-402e-86fe-507324921abe","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.696998751Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"session_id":"8880708b-a835-402e-86fe-507324921abe"}
204052024-04-17T17:23:30.092Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.697017102Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931,"session_id":"8880708b-a835-402e-86fe-507324921abe"}
204062024-04-17T17:23:30.092Z{"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:28.697034263Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931}
204072024-04-17T17:23:30.092ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
204082024-04-17T17:23:30.092ZLoading write count information from file "/tmp/test_repair_verify.data"
204092024-04-17T17:23:30.092ZRun Repair workload
204102024-04-17T17:23:30.096Z01/30 Write block 372 len 4096 data: 15
204112024-04-17T17:23:30.096Z02/30 Write block 98 len 40960 data: 13 17 16 14 12 14 13 15 14 14
204122024-04-17T17:23:30.096Z03/30 Read block 420 len 20480
204132024-04-17T17:23:30.096Z04/30 Flush
204142024-04-17T17:23:30.096Z05/30 Write block 88 len 32768 data: 19 18 18 16 20 20 16 13
204152024-04-17T17:23:30.096Z06/30 Read block 75 len 28672
204162024-04-17T17:23:30.096Z07/30 Read block 376 len 28672
204172024-04-17T17:23:30.103Z08/30 Write block 509 len 16384 data: 13 13 12 13
204182024-04-17T17:23:30.103Z09/30 Write block 26 len 36864 data: 13 10 11 16 15 17 23 24 27
204192024-04-17T17:23:30.103Z10/30 Write block 239 len 12288 data: 12 11 14
204202024-04-17T17:23:30.103Z11/30 Write block 216 len 24576 data: 15 12 13 13 15 13
204212024-04-17T17:23:30.106Z12/30 Write block 54 len 36864 data: 19 17 17 18 20 19 20 19 16
204222024-04-17T17:23:30.106Z13/30 Read block 506 len 16384
204232024-04-17T17:23:30.106Z14/30 Read block 9 len 40960
204242024-04-17T17:23:30.106Z15/30 Read block 560 len 28672
204252024-04-17T17:23:30.107Z16/30 Write block 455 len 28672 data: 11 12 12 14 16 17 14
204262024-04-17T17:23:30.107Z17/30 Write block 283 len 16384 data: 11 12 12 10
204272024-04-17T17:23:30.110Z18/30 Write block 250 len 24576 data: 12 12 14 12 11 9
204282024-04-17T17:23:30.110Z19/30 Write block 461 len 8192 data: 15 10
204292024-04-17T17:23:30.110Z20/30 Write block 420 len 4096 data: 14
204302024-04-17T17:23:30.110Z21/30 Read block 322 len 36864
204312024-04-17T17:23:30.110Z22/30 Read block 191 len 24576
204322024-04-17T17:23:30.114Z23/30 Read block 252 len 28672
204332024-04-17T17:23:30.114Z24/30 Write block 404 len 36864 data: 14 14 14 12 9 11 11 13 13
204342024-04-17T17:23:30.114Z25/30 Write block 104 len 4096 data: 14
204352024-04-17T17:23:30.117Z26/30 Write block 190 len 40960 data: 15 13 12 11 9 10 11 11 15 12
204362024-04-17T17:23:30.117Z27/30 Read block 417 len 40960
204372024-04-17T17:23:30.117Z28/30 Read block 336 len 28672
204382024-04-17T17:23:30.117Z29/30 Write block 583 len 28672 data: 14 15 15 16 15 10 9
204392024-04-17T17:23:30.117Z30/30 Read block 210 len 12288
204402024-04-17T17:23:30.122Z----------------------------------------------------------------
204412024-04-17T17:23:30.122Z Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:26
204422024-04-17T17:23:30.122ZStates: Active Active Active
204432024-04-17T17:23:30.122ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
204442024-04-17T17:23:30.122Z 5 Acked 1004 Write 8 Done Done Done false
204452024-04-17T17:23:30.122Z 6 Acked 1005 Read 7 Done Done Sent false
204462024-04-17T17:23:30.122Z 7 Acked 1006 Read 7 Done Done Sent false
204472024-04-17T17:23:30.122Z 8 Acked 1007 Write 4 Done Done Sent false
204482024-04-17T17:23:30.122Z 9 Acked 1008 Write 9 Done Done Sent false
204492024-04-17T17:23:30.123Z 10 Acked 1009 Write 3 Done Done Sent false
204502024-04-17T17:23:30.123Z 11 Acked 1010 Write 6 Done Done Sent false
204512024-04-17T17:23:30.123Z 12 Acked 1011 Write 9 Done Done Sent false
204522024-04-17T17:23:30.123Z 13 Acked 1012 Read 4 Done Done Sent false
204532024-04-17T17:23:30.123Z 14 Acked 1013 Read 10 Done Done Sent false
204542024-04-17T17:23:30.123Z 15 Acked 1014 Read 7 Done Done Sent false
204552024-04-17T17:23:30.123Z 16 Acked 1015 Write 7 Done Done Sent false
204562024-04-17T17:23:30.123Z 17 Acked 1016 Write 4 Done Done Sent false
204572024-04-17T17:23:30.123Z 18 Acked 1017 Write 6 Done Done Sent false
204582024-04-17T17:23:30.123Z 19 Acked 1018 Write 2 Done Done Sent false
204592024-04-17T17:23:30.123Z 20 Acked 1019 Write 1 Done Done Sent false
204602024-04-17T17:23:30.123Z 21 Acked 1020 Read 9 Done Done Sent false
204612024-04-17T17:23:30.123Z 22 Acked 1021 Read 6 Done Done Sent false
204622024-04-17T17:23:30.123Z 23 Acked 1022 Read 7 Done Done Sent false
204632024-04-17T17:23:30.123Z 24 Acked 1023 Write 9 Done Done Sent false
204642024-04-17T17:23:30.123Z 25 Acked 1024 Write 1 Done Done Sent false
204652024-04-17T17:23:30.123Z 26 Acked 1025 Write 10 Done Done Sent false
204662024-04-17T17:23:30.123Z 27 Acked 1026 Read 10 Done Done Sent false
204672024-04-17T17:23:30.123Z 28 Acked 1027 Read 7 Done Done Sent false
204682024-04-17T17:23:30.123Z 29 Acked 1028 Write 7 Done Done Sent false
204692024-04-17T17:23:30.123Z 30 Acked 1029 Read 3 Done Sent Sent false
204702024-04-17T17:23:30.123Z STATES DS:0 DS:1 DS:2 TOTAL
204712024-04-17T17:23:30.123Z New 0 0 0 0
204722024-04-17T17:23:30.124Z Sent 0 1 25 26
204732024-04-17T17:23:30.124Z Done 26 25 1 52
204742024-04-17T17:23:30.124Z Skipped 0 0 0 0
204752024-04-17T17:23:30.124Z Error 0 0 0 0
204762024-04-17T17:23:30.124ZLast Flush: 1003 1003 1003
204772024-04-17T17:23:30.124ZDownstairs last five completed: 1003 1002 1001 1000
204782024-04-17T17:23:30.124ZUpstairs last five completed: 30 29 28 27 26
204792024-04-17T17:23:30.124Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:23:28.728872633Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1931}
204802024-04-17T17:23:30.124ZWrote out file "/tmp/test_repair_verify.data"
204812024-04-17T17:23:30.128Z
204822024-04-17T17:23:30.128ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
204832024-04-17T17:23:30.146ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
204842024-04-17T17:23:30.146Z 0 000-019 188 188 188 333 333 333 F F F
204852024-04-17T17:23:30.151Z 1 020-039 187 187 187 332 332 332 T T F <---
204862024-04-17T17:23:30.151Z 2 040-059 186 186 186 330 330 330 T T F <---
204872024-04-17T17:23:30.151Z 3 060-079 186 186 186 330 330 330 T T F <---
204882024-04-17T17:23:30.151Z 4 080-099 190 190 190 334 334 334 T T T
204892024-04-17T17:23:30.151Z 5 100-119 190 190 190 334 334 334 T T F <---
204902024-04-17T17:23:30.151Z 6 120-139 178 178 178 316 316 316 F F F
204912024-04-17T17:23:30.151Z 7 140-159 188 188 188 333 333 333 F F F
204922024-04-17T17:23:30.151Z 8 160-179 186 186 186 328 328 328 F F F
204932024-04-17T17:23:30.151Z 9 180-199 188 188 188 333 333 333 T T F <---
204942024-04-17T17:23:30.151Z 10 200-219 188 188 188 333 333 333 T T F <---
204952024-04-17T17:23:30.151Z 11 220-239 186 186 186 328 328 328 T T F <---
204962024-04-17T17:23:30.151Z 12 240-259 172 172 172 302 302 302 T T F <---
204972024-04-17T17:23:30.151Z 13 260-279 181 181 181 322 322 322 F F F
204982024-04-17T17:23:30.151Z 14 280-299 181 181 181 322 322 322 T T F <---
204992024-04-17T17:23:30.151Z 15 300-319 188 188 188 333 333 333 F F F
205002024-04-17T17:23:30.151Z 16 320-339 182 182 182 323 323 323 F F F
205012024-04-17T17:23:30.151Z 17 340-359 188 188 188 333 333 333 F F F
205022024-04-17T17:23:30.151Z 18 360-379 190 190 190 334 334 334 F F F
205032024-04-17T17:23:30.151Z 19 380-399 177 177 177 314 314 314 F F F
205042024-04-17T17:23:30.152Z 20 400-419 188 188 188 333 333 333 T T F <---
205052024-04-17T17:23:30.152Z 21 420-439 188 188 188 333 333 333 T T F <---
205062024-04-17T17:23:30.152Z 22 440-459 188 188 188 333 333 333 T T F <---
205072024-04-17T17:23:30.152Z 23 460-479 188 188 188 333 333 333 T T F <---
205082024-04-17T17:23:30.152Z 24 480-499 186 186 186 331 331 331 F F F
205092024-04-17T17:23:30.152Z 25 500-519 186 186 186 331 331 331 T T F <---
205102024-04-17T17:23:30.152Z 26 520-539 178 178 178 316 316 316 F F F
205112024-04-17T17:23:30.152Z 27 540-559 188 188 188 333 333 333 F F F
205122024-04-17T17:23:30.152Z 28 560-579 188 188 188 333 333 333 F F F
205132024-04-17T17:23:30.152Z 29 580-599 178 178 178 316 316 316 T T F <---
205142024-04-17T17:23:30.152ZMax gen: 190, Max flush: 334
205152024-04-17T17:23:30.152ZError: Difference in extent metadata found!
205162024-04-17T17:23:30.152ZA Difference in extent metadata is expected here
205172024-04-17T17:23:30.152ZOn loop 95
205182024-04-17T17:23:30.152Z
205192024-04-17T17:23:30.156ZVerifying data now
205202024-04-17T17:23:34.934ZNo change after verify
205212024-04-17T17:23:34.934ZLoop: 95 Downstairs dump after verify (and repair):
205222024-04-17T17:23:34.949ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
205232024-04-17T17:23:34.949Z 0 000-019 188 188 188 333 333 333 F F F
205242024-04-17T17:23:34.949Z 1 020-039 191 191 191 335 335 335 F F F
205252024-04-17T17:23:34.949Z 2 040-059 191 191 191 335 335 335 F F F
205262024-04-17T17:23:34.949Z 3 060-079 191 191 191 335 335 335 F F F
205272024-04-17T17:23:34.949Z 4 080-099 191 191 191 335 335 335 F F F
205282024-04-17T17:23:34.949Z 5 100-119 191 191 191 335 335 335 F F F
205292024-04-17T17:23:34.953Z 6 120-139 178 178 178 316 316 316 F F F
205302024-04-17T17:23:34.953Z 7 140-159 188 188 188 333 333 333 F F F
205312024-04-17T17:23:34.953Z 8 160-179 186 186 186 328 328 328 F F F
205322024-04-17T17:23:34.953Z 9 180-199 191 191 191 335 335 335 F F F
205332024-04-17T17:23:34.953Z 10 200-219 191 191 191 335 335 335 F F F
205342024-04-17T17:23:34.953Z 11 220-239 191 191 191 335 335 335 F F F
205352024-04-17T17:23:34.953Z 12 240-259 191 191 191 335 335 335 F F F
205362024-04-17T17:23:34.953Z 13 260-279 181 181 181 322 322 322 F F F
205372024-04-17T17:23:34.953Z 14 280-299 191 191 191 335 335 335 F F F
205382024-04-17T17:23:34.953Z 15 300-319 188 188 188 333 333 333 F F F
205392024-04-17T17:23:34.953Z 16 320-339 182 182 182 323 323 323 F F F
205402024-04-17T17:23:34.953Z 17 340-359 188 188 188 333 333 333 F F F
205412024-04-17T17:23:34.953Z 18 360-379 190 190 190 334 334 334 F F F
205422024-04-17T17:23:34.954Z 19 380-399 177 177 177 314 314 314 F F F
205432024-04-17T17:23:34.954Z 20 400-419 191 191 191 335 335 335 F F F
205442024-04-17T17:23:34.954Z 21 420-439 191 191 191 335 335 335 F F F
205452024-04-17T17:23:34.954Z 22 440-459 191 191 191 335 335 335 F F F
205462024-04-17T17:23:34.954Z 23 460-479 191 191 191 335 335 335 F F F
205472024-04-17T17:23:34.954Z 24 480-499 186 186 186 331 331 331 F F F
205482024-04-17T17:23:34.954Z 25 500-519 191 191 191 335 335 335 F F F
205492024-04-17T17:23:34.954Z 26 520-539 178 178 178 316 316 316 F F F
205502024-04-17T17:23:34.954Z 27 540-559 188 188 188 333 333 333 F F F
205512024-04-17T17:23:34.954Z 28 560-579 188 188 188 333 333 333 F F F
205522024-04-17T17:23:34.954Z 29 580-599 191 191 191 335 335 335 F F F
205532024-04-17T17:23:34.954ZMax gen: 191, Max flush: 335
205542024-04-17T17:23:34.954Z
205552024-04-17T17:23:34.954ZBegin loop 96
205562024-04-17T17:23:34.954ZDownstairs to restart: 0
205572024-04-17T17:23:34.976Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.583402974Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939}
205582024-04-17T17:23:34.976Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.583704412Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939}
205592024-04-17T17:23:34.976Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.583742094Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939}
205602024-04-17T17:23:34.982Z{"msg":"Crucible 20ac1043-bd12-4e90-a717-fe38de8046bb has session id: 7673a59b-a69d-47a2-b890-a7daf78bc2f2","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.583766165Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205612024-04-17T17:23:34.982Z{"msg":"Crucible runtime is spawned
205622024-04-17T17:23:34.982ZUpstairs opts: Upstairs UUID: 20ac1043-bd12-4e90-a717-fe38de8046bb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.583820769Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205632024-04-17T17:23:34.982Z{"msg":"Crucible stats registered with UUID: 20ac1043-bd12-4e90-a717-fe38de8046bb","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.583866382Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205642024-04-17T17:23:34.982Z{"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.583889803Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939}
205652024-04-17T17:23:34.982Z{"msg":"20ac1043-bd12-4e90-a717-fe38de8046bb active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.583927495Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205662024-04-17T17:23:34.983Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.583951157Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"client":"0","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205672024-04-17T17:23:34.983Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.583986219Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"client":"0","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205682024-04-17T17:23:34.983Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.58401112Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"client":"1","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205692024-04-17T17:23:34.983Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.584045242Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"client":"1","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205702024-04-17T17:23:34.983Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.584069454Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"client":"2","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205712024-04-17T17:23:34.983Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.584117207Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"client":"2","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205722024-04-17T17:23:34.983Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.584144518Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"":"io task","client":"2","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205732024-04-17T17:23:34.983Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.584208702Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"":"io task","client":"0","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205742024-04-17T17:23:34.983Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.584273836Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"":"io task","client":"1","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205752024-04-17T17:23:34.983Z{"msg":"ds_connection connected from Ok(127.0.0.1:34675)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.584346391Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"":"io task","client":"1","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205762024-04-17T17:23:34.983Z{"msg":"ds_connection connected from Ok(127.0.0.1:61227)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.584393283Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"":"io task","client":"0","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205772024-04-17T17:23:34.983Z{"msg":"ds_connection connected from Ok(127.0.0.1:34373)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.584424355Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"":"io task","client":"2","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205782024-04-17T17:23:34.983Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.585132428Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"client":"1","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205792024-04-17T17:23:34.983Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.58517021Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"client":"1","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205802024-04-17T17:23:34.983Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.585197842Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"client":"2","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205812024-04-17T17:23:34.983Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.585257886Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"client":"2","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205822024-04-17T17:23:34.983Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.585289688Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"client":"0","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205832024-04-17T17:23:34.983Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.585304919Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"client":"0","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205842024-04-17T17:23:34.983Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.585711053Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"client":"1","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205852024-04-17T17:23:34.983Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.585833371Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"client":"2","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205862024-04-17T17:23:34.983Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.585870543Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"client":"0","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205872024-04-17T17:23:34.983Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.586139999Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"client":"1","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205882024-04-17T17:23:34.983Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.586176711Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"client":"1","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205892024-04-17T17:23:34.984Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.586192412Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205902024-04-17T17:23:34.984Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.586355172Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"client":"0","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205912024-04-17T17:23:34.984Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.586405776Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"client":"0","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205922024-04-17T17:23:34.984Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.586448748Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205932024-04-17T17:23:34.984Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.58648226Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"client":"2","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205942024-04-17T17:23:34.984Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.586525493Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"client":"2","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205952024-04-17T17:23:34.984Z{"msg":"[0]R flush_numbers[0..12]: [333, 335, 335, 335, 335, 335, 316, 333, 328, 335, 335, 335]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.586562285Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"":"downstairs","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205962024-04-17T17:23:34.984Z{"msg":"[0]R generation[0..12]: [188, 191, 191, 191, 191, 191, 178, 188, 186, 191, 191, 191]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.586597907Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"":"downstairs","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205972024-04-17T17:23:34.984Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.586632729Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"":"downstairs","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205982024-04-17T17:23:34.984Z{"msg":"[1]R flush_numbers[0..12]: [333, 335, 335, 335, 335, 335, 316, 333, 328, 335, 335, 335]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.586658761Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"":"downstairs","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
205992024-04-17T17:23:34.984Z{"Wait for a query_work_queue command to finish before sending IO
206002024-04-17T17:23:34.984Zmsg":"[1]R generation[0..12]: [188, 191, 191, 191, 191, 191, 178, 188, 186, 191, 191, 191]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.586697873Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"":"downstairs","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
206012024-04-17T17:23:34.984Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.586736556Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"":"downstairs","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
206022024-04-17T17:23:34.984Z{"msg":"[2]R flush_numbers[0..12]: [333, 335, 335, 335, 335, 335, 316, 333, 328, 335, 335, 335]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.586775448Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"":"downstairs","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
206032024-04-17T17:23:34.984Z{"msg":"[2]R generation[0..12]: [188, 191, 191, 191, 191, 191, 178, 188, 186, 191, 191, 191]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.58681446Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"":"downstairs","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
206042024-04-17T17:23:34.984Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.586852603Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"":"downstairs","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
206052024-04-17T17:23:34.984Z{"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.586890085Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"":"downstairs","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
206062024-04-17T17:23:34.984Z{"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.586928277Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"":"downstairs","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
206072024-04-17T17:23:34.984Z{"msg":"Next flush: 336","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.58697243Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"":"downstairs","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
206082024-04-17T17:23:34.984Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.586999881Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"":"downstairs","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
206092024-04-17T17:23:34.984Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.587014502Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
206102024-04-17T17:23:34.984Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.587026413Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"client":"0","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
206112024-04-17T17:23:34.984Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.587046504Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"client":"1","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
206122024-04-17T17:23:34.985Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.587072776Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"client":"2","":"downstairs","session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
206132024-04-17T17:23:34.985Z{"msg":"All required reconciliation work is completed","v":0,"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
206142024-04-17T17:23:34.985Zname":"crucible","level":30,"time":"2024-04-17T17:23:33.587090557Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
206152024-04-17T17:23:34.985Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.587109688Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
206162024-04-17T17:23:34.985Z{"msg":"20ac1043-bd12-4e90-a717-fe38de8046bb is now active with session: 7673a59b-a69d-47a2-b890-a7daf78bc2f2","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.587122669Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
206172024-04-17T17:23:34.985Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.58713436Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939,"session_id":"7673a59b-a69d-47a2-b890-a7daf78bc2f2"}
206182024-04-17T17:23:34.985Z{"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:33.587153011Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939}
206192024-04-17T17:23:34.985ZLoading write count information from file "/tmp/test_repair_verify.data"
206202024-04-17T17:23:34.985ZRun Repair workload
206212024-04-17T17:23:34.985Z01/30 Write block 536 len 32768 data: 9 7 10 15 16 17 22 20
206222024-04-17T17:23:34.985Z02/30 Write block 50 len 20480 data: 11 13 13 14 20
206232024-04-17T17:23:34.989Z03/30 Read block 527 len 20480
206242024-04-17T17:23:34.989Z04/30 Read block 130 len 40960
206252024-04-17T17:23:34.989Z05/30 Read block 389 len 12288
206262024-04-17T17:23:34.990Z06/30 Flush
206272024-04-17T17:23:34.990Z07/30 Write block 70 len 12288 data: 17 17 17
206282024-04-17T17:23:34.990Z08/30 Write block 6 len 36864 data: 13 14 15 14 14 12 14 16 15
206292024-04-17T17:23:34.990Z09/30 Write block 297 len 20480 data: 10 11 13 12 13
206302024-04-17T17:23:34.990Z10/30 Write block 400 len 8192 data: 11 12
206312024-04-17T17:23:34.990Z11/30 Write block 409 len 4096 data: 12
206322024-04-17T17:23:34.994Z12/30 Write block 441 len 16384 data: 15 17 17 15
206332024-04-17T17:23:34.994Z13/30 Write block 11 len 36864 data: 13 15 17 16 12 11 14 13 16
206342024-04-17T17:23:34.994Z14/30 Read block 113 len 20480
206352024-04-17T17:23:34.994Z15/30 Write block 159 len 24576 data: 15 14 14 13 15 13
206362024-04-17T17:23:34.997Z16/30 Write block 448 len 32768 data: 11 13 12 10 11 8 9 12
206372024-04-17T17:23:34.997Z17/30 Read block 441 len 12288
206382024-04-17T17:23:34.997Z18/30 Read block 55 len 28672
206392024-04-17T17:23:34.997Z19/30 Read block 110 len 12288
206402024-04-17T17:23:34.997Z20/30 Read block 449 len 12288
206412024-04-17T17:23:35.001Z21/30 Read block 231 len 12288
206422024-04-17T17:23:35.001Z22/30 Flush
206432024-04-17T17:23:35.005Z23/30 Write block 236 len 32768 data: 17 14 12 13 12 15 13 15
206442024-04-17T17:23:35.005Z24/30 Read block 510 len 4096
206452024-04-17T17:23:35.005Z25/30 Read block 156 len 36864
206462024-04-17T17:23:35.006Z26/30 Read block 304 len 28672
206472024-04-17T17:23:35.009Z27/30 Write block 467 len 40960 data: 14 14 15 16 15 14 13 14 12 14
206482024-04-17T17:23:35.010Z28/30 Read block 525 len 4096
206492024-04-17T17:23:35.010Z29/30 Read block 434 len 40960
206502024-04-17T17:23:35.010Z30/30 Write block 180 len 12288 data: 11 14 15
206512024-04-17T17:23:35.010Z----------------------------------------------------------------
206522024-04-17T17:23:35.010Z Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:30
206532024-04-17T17:23:35.010ZStates: Active Active Active
206542024-04-17T17:23:35.010ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
206552024-04-17T17:23:35.010Z 1 Acked 1000 Write 8 Done Done Done false
206562024-04-17T17:23:35.010Z 2 Acked 1001 Write 5 Sent Done Done false
206572024-04-17T17:23:35.010Z 3 Acked 1002 Read 5 Sent Done Done false
206582024-04-17T17:23:35.010Z 4 Acked 1003 Read 10 Sent Done Done false
206592024-04-17T17:23:35.010Z 5 Acked 1004 Read 3 Sent Done Done false
206602024-04-17T17:23:35.010Z 6 Acked 1005 Flush 0 Sent Done Done false
206612024-04-17T17:23:35.011Z 7 Acked 1006 Write 3 Sent Done Done false
206622024-04-17T17:23:35.011Z 8 Acked 1007 Write 9 Sent Done Done false
206632024-04-17T17:23:35.011Z 9 Acked 1008 Write 5 Sent Done Done false
206642024-04-17T17:23:35.011Z 10 Acked 1009 Write 2 Sent Done Done false
206652024-04-17T17:23:35.011Z 11 Acked 1010 Write 1 Sent Done Done false
206662024-04-17T17:23:35.011Z 12 Acked 1011 Write 4 Sent Done Done false
206672024-04-17T17:23:35.011Z 13 Acked 1012 Write 9 Sent Done Done false
206682024-04-17T17:23:35.011Z 14 Acked 1013 Read 5 Sent Done Done false
206692024-04-17T17:23:35.011Z 15 Acked 1014 Write 6 Sent Done Done false
206702024-04-17T17:23:35.011Z 16 Acked 1015 Write 8 Sent Done Done false
206712024-04-17T17:23:35.011Z 17 Acked 1016 Read 3 Sent Done Done false
206722024-04-17T17:23:35.011Z 18 Acked 1017 Read 7 Sent Done Done false
206732024-04-17T17:23:35.011Z 19 Acked 1018 Read 3 Sent Done Done false
206742024-04-17T17:23:35.011Z 20 Acked 1019 Read 3 Sent Done Done false
206752024-04-17T17:23:35.011Z 21 Acked 1020 Read 3 Sent Done Done false
206762024-04-17T17:23:35.012Z 22 Acked 1021 Flush 0 Sent Done Done false
206772024-04-17T17:23:35.012Z 23 Acked 1022 Write 8 Sent Done Done false
206782024-04-17T17:23:35.012Z 24 Acked 1023 Read 1 Sent Done Done false
206792024-04-17T17:23:35.012Z 25 Acked 1024 Read 9 Sent Done Done false
206802024-04-17T17:23:35.012Z 26 Acked 1025 Read 7 Sent Done Done false
206812024-04-17T17:23:35.012Z 27 Acked 1026 Write 10 Sent Done Done false
206822024-04-17T17:23:35.012Z 28 Acked 1027 Read 1 Sent Done Done false
206832024-04-17T17:23:35.012Z 29 Acked 1028 Read 10 Sent Done Done false
206842024-04-17T17:23:35.012Z 30 Acked 1029 Write 3 Sent Sent Sent false
206852024-04-17T17:23:35.012Z STATES DS:0 DS:1 DS:2 TOTAL
206862024-04-17T17:23:35.012Z New 0 0 0 0
206872024-04-17T17:23:35.012Z Sent 29 1 1 31
206882024-04-17T17:23:35.012Z Done 1 29 29 59
206892024-04-17T17:23:35.012Z Skipped 0 0 0 0
206902024-04-17T17:23:35.012Z Error 0 0 0 0
206912024-04-17T17:23:35.012ZLast Flush: 0 1021 1021
206922024-04-17T17:23:35.012ZDownstairs last five completed:
206932024-04-17T17:23:35.012ZUpstairs last five completed: 30 29 28 27 26
206942024-04-17T17:23:35.012Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:23:33.61931997Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1939}
206952024-04-17T17:23:35.012ZWrote out file "/tmp/test_repair_verify.data"
206962024-04-17T17:23:35.016Z
206972024-04-17T17:23:35.017ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
206982024-04-17T17:23:35.032ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
206992024-04-17T17:23:35.033Z 0 000-019 188 192 192 333 337 337 F F F <---
207002024-04-17T17:23:35.033Z 1 020-039 191 191 191 335 335 335 F F F
207012024-04-17T17:23:35.033Z 2 040-059 191 192 192 335 336 336 F F F <---
207022024-04-17T17:23:35.033Z 3 060-079 191 192 192 335 337 337 F F F <---
207032024-04-17T17:23:35.033Z 4 080-099 191 191 191 335 335 335 F F F
207042024-04-17T17:23:35.033Z 5 100-119 191 191 191 335 335 335 F F F
207052024-04-17T17:23:35.033Z 6 120-139 178 178 178 316 316 316 F F F
207062024-04-17T17:23:35.033Z 7 140-159 188 192 192 333 337 337 F F F <---
207072024-04-17T17:23:35.037Z 8 160-179 186 192 192 328 337 337 F F F <---
207082024-04-17T17:23:35.037Z 9 180-199 191 191 191 335 335 335 F T T <---
207092024-04-17T17:23:35.037Z 10 200-219 191 191 191 335 335 335 F F F
207102024-04-17T17:23:35.037Z 11 220-239 191 191 191 335 335 335 F T T <---
207112024-04-17T17:23:35.037Z 12 240-259 191 191 191 335 335 335 F T T <---
207122024-04-17T17:23:35.037Z 13 260-279 181 181 181 322 322 322 F F F
207132024-04-17T17:23:35.037Z 14 280-299 191 192 192 335 337 337 F F F <---
207142024-04-17T17:23:35.037Z 15 300-319 188 192 192 333 337 337 F F F <---
207152024-04-17T17:23:35.037Z 16 320-339 182 182 182 323 323 323 F F F
207162024-04-17T17:23:35.037Z 17 340-359 188 188 188 333 333 333 F F F
207172024-04-17T17:23:35.037Z 18 360-379 190 190 190 334 334 334 F F F
207182024-04-17T17:23:35.037Z 19 380-399 177 177 177 314 314 314 F F F
207192024-04-17T17:23:35.037Z 20 400-419 191 192 192 335 337 337 F F F <---
207202024-04-17T17:23:35.037Z 21 420-439 191 191 191 335 335 335 F F F
207212024-04-17T17:23:35.037Z 22 440-459 191 192 192 335 337 337 F F F <---
207222024-04-17T17:23:35.037Z 23 460-479 191 191 191 335 335 335 F T T <---
207232024-04-17T17:23:35.037Z 24 480-499 186 186 186 331 331 331 F F F
207242024-04-17T17:23:35.038Z 25 500-519 191 191 191 335 335 335 F F F
207252024-04-17T17:23:35.038Z 26 520-539 178 192 192 316 336 336 T F F <---
207262024-04-17T17:23:35.038Z 27 540-559 188 192 192 333 336 336 T F F <---
207272024-04-17T17:23:35.038Z 28 560-579 188 188 188 333 333 333 F F F
207282024-04-17T17:23:35.038Z 29 580-599 191 191 191 335 335 335 F F F
207292024-04-17T17:23:35.038ZMax gen: 192, Max flush: 337
207302024-04-17T17:23:35.038ZError: Difference in extent metadata found!
207312024-04-17T17:23:35.038ZA Difference in extent metadata is expected here
207322024-04-17T17:23:35.038ZOn loop 96
207332024-04-17T17:23:35.038Z
207342024-04-17T17:23:35.042ZVerifying data now
207352024-04-17T17:23:49.716ZNo change after verify
207362024-04-17T17:23:49.716ZLoop: 96 Downstairs dump after verify (and repair):
207372024-04-17T17:23:49.731ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
207382024-04-17T17:23:49.731Z 0 000-019 192 192 192 337 337 337 F F F
207392024-04-17T17:23:49.735Z 1 020-039 191 191 191 335 335 335 F F F
207402024-04-17T17:23:49.735Z 2 040-059 192 192 192 336 336 336 F F F
207412024-04-17T17:23:49.735Z 3 060-079 192 192 192 337 337 337 F F F
207422024-04-17T17:23:49.735Z 4 080-099 191 191 191 335 335 335 F F F
207432024-04-17T17:23:49.735Z 5 100-119 191 191 191 335 335 335 F F F
207442024-04-17T17:23:49.735Z 6 120-139 178 178 178 316 316 316 F F F
207452024-04-17T17:23:49.735Z 7 140-159 192 192 192 337 337 337 F F F
207462024-04-17T17:23:49.735Z 8 160-179 192 192 192 337 337 337 F F F
207472024-04-17T17:23:49.735Z 9 180-199 193 193 193 338 338 338 F F F
207482024-04-17T17:23:49.735Z 10 200-219 191 191 191 335 335 335 F F F
207492024-04-17T17:23:49.735Z 11 220-239 193 193 193 338 338 338 F F F
207502024-04-17T17:23:49.735Z 12 240-259 193 193 193 338 338 338 F F F
207512024-04-17T17:23:49.735Z 13 260-279 181 181 181 322 322 322 F F F
207522024-04-17T17:23:49.735Z 14 280-299 192 192 192 337 337 337 F F F
207532024-04-17T17:23:49.735Z 15 300-319 192 192 192 337 337 337 F F F
207542024-04-17T17:23:49.735Z 16 320-339 182 182 182 323 323 323 F F F
207552024-04-17T17:23:49.735Z 17 340-359 188 188 188 333 333 333 F F F
207562024-04-17T17:23:49.736Z 18 360-379 190 190 190 334 334 334 F F F
207572024-04-17T17:23:49.736Z 19 380-399 177 177 177 314 314 314 F F F
207582024-04-17T17:23:49.736Z 20 400-419 192 192 192 337 337 337 F F F
207592024-04-17T17:23:49.736Z 21 420-439 191 191 191 335 335 335 F F F
207602024-04-17T17:23:49.736Z 22 440-459 192 192 192 337 337 337 F F F
207612024-04-17T17:23:49.736Z 23 460-479 193 193 193 338 338 338 F F F
207622024-04-17T17:23:49.736Z 24 480-499 186 186 186 331 331 331 F F F
207632024-04-17T17:23:49.736Z 25 500-519 191 191 191 335 335 335 F F F
207642024-04-17T17:23:49.736Z 26 520-539 192 192 192 336 336 336 F F F
207652024-04-17T17:23:49.736Z 27 540-559 192 192 192 336 336 336 F F F
207662024-04-17T17:23:49.736Z 28 560-579 188 188 188 333 333 333 F F F
207672024-04-17T17:23:49.736Z 29 580-599 191 191 191 335 335 335 F F F
207682024-04-17T17:23:49.736ZMax gen: 193, Max flush: 338
207692024-04-17T17:23:49.736Z
207702024-04-17T17:23:49.736ZBegin loop 97
207712024-04-17T17:23:49.736ZDownstairs to restart: 1
207722024-04-17T17:23:49.757Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.364269531Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947}
207732024-04-17T17:23:49.757Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.364525556Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947}
207742024-04-17T17:23:49.763Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.364563038Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947}
207752024-04-17T17:23:49.763Z{"msg":"Crucible 8ab4d0c6-7de1-4aee-b390-01975a80b25f has session id: 36a4d2ab-a2c9-4ca4-b675-5c247a6aa995","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.36458702Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
207762024-04-17T17:23:49.763Z{Crucible runtime is spawned
207772024-04-17T17:23:49.763Z"msg":"Upstairs opts: Upstairs UUID: 8ab4d0c6-7de1-4aee-b390-01975a80b25f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.364771041Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
207782024-04-17T17:23:49.763Z{"msg":"Crucible stats registered with UUID: 8ab4d0c6-7de1-4aee-b390-01975a80b25f","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.364843115Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
207792024-04-17T17:23:49.764Z{"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.364898359Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947}
207802024-04-17T17:23:49.764Z{"msg":"8ab4d0c6-7de1-4aee-b390-01975a80b25f active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.364926861Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
207812024-04-17T17:23:49.764Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.364953302Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"client":"0","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
207822024-04-17T17:23:49.764Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.364994985Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"client":"0","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
207832024-04-17T17:23:49.764Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.365059839Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"client":"1","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
207842024-04-17T17:23:49.764Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.365104471Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"client":"1","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
207852024-04-17T17:23:49.764Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.365131453Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"client":"2","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
207862024-04-17T17:23:49.764Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.365156045Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"client":"2","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
207872024-04-17T17:23:49.764Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.365178876Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"":"io task","client":"0","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
207882024-04-17T17:23:49.764Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.365220569Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"":"io task","client":"1","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
207892024-04-17T17:23:49.764Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.36524634Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"":"io task","client":"2","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
207902024-04-17T17:23:49.764Z{"msg":"ds_connection connected from Ok(127.0.0.1:64966)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.365273392Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"":"io task","client":"2","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
207912024-04-17T17:23:49.764Z{"msg":"ds_connection connected from Ok(127.0.0.1:60859)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.365312514Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"":"io task","client":"1","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
207922024-04-17T17:23:49.764Z{"msg":"ds_connection connected from Ok(127.0.0.1:50602)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.365330785Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"":"io task","client":"0","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
207932024-04-17T17:23:49.764Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.365963563Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"client":"2","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
207942024-04-17T17:23:49.764Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.366009166Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"client":"2","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
207952024-04-17T17:23:49.764Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.366056849Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"client":"0","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
207962024-04-17T17:23:49.764Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.366101372Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"client":"0","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
207972024-04-17T17:23:49.764Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.366143334Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"client":"1","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
207982024-04-17T17:23:49.764Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.366178136Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"client":"1","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
207992024-04-17T17:23:49.764Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.366510807Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"client":"2","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208002024-04-17T17:23:49.764Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.366558079Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"client":"0","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208012024-04-17T17:23:49.764Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.366826316Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"client":"1","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208022024-04-17T17:23:49.765Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.367035759Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"client":"2","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208032024-04-17T17:23:49.765Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.367073881Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"client":"2","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208042024-04-17T17:23:49.765Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.367099412Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208052024-04-17T17:23:49.765Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.367154706Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"client":"0","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208062024-04-17T17:23:49.765Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.367205709Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"client":"0","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208072024-04-17T17:23:49.765Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.36722078Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208082024-04-17T17:23:49.765Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.367381659Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"client":"1","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208092024-04-17T17:23:49.765Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.367417872Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"client":"1","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208102024-04-17T17:23:49.765Z{"msg":"[0]R flush_numbers[0..12]: [337, 335, 336, 337, 335, 335, 316, 337, 337, 338, 335, 338]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.367434623Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"":"downstairs","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208112024-04-17T17:23:49.765Z{"msg":"[0]R generation[0..12]: [192, 191, 192, 192, 191, 191, 178, 192, 192, 193, 191, 193]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.367457364Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"":"downstairs","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208122024-04-17T17:23:49.765Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.367472075Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"":"downstairs","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208132024-04-17T17:23:49.765Z{"msg":"[1]R flush_numbers[0..12]: [337, 335, 336, 337, 335, 335, 316, 337, 337, 338, 335, 338]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.367492836Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"":"downstairs","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208142024-04-17T17:23:49.765Z{"msg":"[1]R generation[0..12]: [192, 191, 192, 192, 191, 191, 178, 192, 192, 193, 191, 193]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.367507237Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"":"downstairs","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208152024-04-17T17:23:49.765Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.367521088Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"":"downstairs","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208162024-04-17T17:23:49.765Z{"msg":"[2]R flush_numbers[0..12]: [337, 335, 336, 337, 335, 335, 316, 337, 337, 338, 335, 338]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
208172024-04-17T17:23:49.765Z,"time":"2024-04-17T17:23:48.367545779Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"":"downstairs","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208182024-04-17T17:23:49.765Z{"msg":"[2]R generation[0..12]: [192, 191, 192, 192, 191, 191, 178, 192, 192, 193, 191, 193]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.367564391Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"":"downstairs","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208192024-04-17T17:23:49.765Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.367578791Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"":"downstairs","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208202024-04-17T17:23:49.765Z{"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.367591902Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"":"downstairs","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208212024-04-17T17:23:49.765Z{"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.367604813Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"":"downstairs","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208222024-04-17T17:23:49.765Z{"msg":"Next flush: 339","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.367617434Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"":"downstairs","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208232024-04-17T17:23:49.765Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.367642855Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"":"downstairs","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208242024-04-17T17:23:49.766Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.367660116Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208252024-04-17T17:23:49.766Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.367675987Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"client":"0","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208262024-04-17T17:23:49.766Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.367689498Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"client":"1","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208272024-04-17T17:23:49.766Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.367710519Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"client":"2","":"downstairs","session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208282024-04-17T17:23:49.766Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.36772407Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208292024-04-17T17:23:49.766Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.367754062Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208302024-04-17T17:23:49.766Z{"msg":"8ab4d0c6-7de1-4aee-b390-01975a80b25f is now active with session: 36a4d2ab-a2c9-4ca4-b675-5c247a6aa995","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.367769353Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208312024-04-17T17:23:49.766Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.367801005Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947,"session_id":"36a4d2ab-a2c9-4ca4-b675-5c247a6aa995"}
208322024-04-17T17:23:49.766Z{"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:23:48.367817986Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947}
208332024-04-17T17:23:49.766ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
208342024-04-17T17:23:49.766ZLoading write count information from file "/tmp/test_repair_verify.data"
208352024-04-17T17:23:49.766ZRun Repair workload
208362024-04-17T17:23:49.766Z01/30 Write block 144 len 8192 data: 14 16
208372024-04-17T17:23:49.766Z02/30 Read block 60 len 32768
208382024-04-17T17:23:49.766Z03/30 Read block 186 len 40960
208392024-04-17T17:23:49.770Z04/30 Write block 548 len 32768 data: 18 20 16 17 21 16 16 17
208402024-04-17T17:23:49.770Z05/30 Read block 262 len 32768
208412024-04-17T17:23:49.770Z06/30 Flush
208422024-04-17T17:23:49.770Z07/30 Write block 28 len 40960 data: 12 17 16 18 24 25 28 22 19 17
208432024-04-17T17:23:49.770Z08/30 Write block 593 len 24576 data: 11 10 9 8 8 7
208442024-04-17T17:23:49.770Z09/30 Read block 404 len 24576
208452024-04-17T17:23:49.770Z10/30 Write block 475 len 12288 data: 13 15 18
208462024-04-17T17:23:49.770Z11/30 Read block 528 len 40960
208472024-04-17T17:23:49.775Z12/30 Write block 328 len 36864 data: 10 11 9 12 11 11 13 15 19
208482024-04-17T17:23:49.775Z13/30 Read block 574 len 32768
208492024-04-17T17:23:49.775Z14/30 Read block 540 len 32768
208502024-04-17T17:23:49.780Z15/30 Write block 570 len 36864 data: 13 12 14 15 18 19 16 13 12
208512024-04-17T17:23:49.780Z16/30 Flush
208522024-04-17T17:23:49.783Z17/30 Write block 463 len 40960 data: 12 13 17 15 15 15 16 17 16 15
208532024-04-17T17:23:49.783Z18/30 Write block 124 len 24576 data: 11 13 13 15 13 14
208542024-04-17T17:23:49.783Z19/30 Write block 280 len 4096 data: 13
208552024-04-17T17:23:49.784Z20/30 Write block 560 len 32768 data: 16 16 15 15 16 11 11 14
208562024-04-17T17:23:49.790Z21/30 Write block 183 len 40960 data: 13 13 12 12 18 20 19 16 14 13
208572024-04-17T17:23:49.790Z22/30 Write block 504 len 4096 data: 14
208582024-04-17T17:23:49.790Z23/30 Read block 94 len 24576
208592024-04-17T17:23:49.790Z24/30 Write block 113 len 28672 data: 16 14 14 14 13 10 12
208602024-04-17T17:23:49.796Z25/30 Write block 42 len 12288 data: 17 15 13
208612024-04-17T17:23:49.796Z26/30 Write block 262 len 4096 data: 14
208622024-04-17T17:23:49.796Z27/30 Read block 120 len 24576
208632024-04-17T17:23:49.796Z28/30 Write block 308 len 16384 data: 18 17 18 16
208642024-04-17T17:23:49.796Z29/30 Read block 284 len 20480
208652024-04-17T17:23:49.796Z30/30 Write block 152 len 28672 data: 12 12 15 15 16 16 13
208662024-04-17T17:23:49.796Z----------------------------------------------------------------
208672024-04-17T17:23:49.796Z Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:30
208682024-04-17T17:23:49.796ZStates: Active Active Active
208692024-04-17T17:23:49.796ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
208702024-04-17T17:23:49.796Z 1 Acked 1000 Write 2 Done Done Done false
208712024-04-17T17:23:49.796Z 2 Acked 1001 Read 8 Done Done Done false
208722024-04-17T17:23:49.796Z 3 Acked 1002 Read 10 Done Done Done false
208732024-04-17T17:23:49.796Z 4 Acked 1003 Write 8 Done Done Done false
208742024-04-17T17:23:49.796Z 5 Acked 1004 Read 8 Done Done Done false
208752024-04-17T17:23:49.796Z 6 Acked 1005 Flush 0 Done Sent Done false
208762024-04-17T17:23:49.797Z 7 Acked 1006 Write 10 Done Sent Done false
208772024-04-17T17:23:49.797Z 8 Acked 1007 Write 6 Done Sent Done false
208782024-04-17T17:23:49.797Z 9 Acked 1008 Read 6 Done Sent Done false
208792024-04-17T17:23:49.797Z 10 Acked 1009 Write 3 Done Sent Done false
208802024-04-17T17:23:49.797Z 11 Acked 1010 Read 10 Done Sent Done false
208812024-04-17T17:23:49.797Z 12 Acked 1011 Write 9 Done Sent Done false
208822024-04-17T17:23:49.797Z 13 Acked 1012 Read 8 Done Sent Done false
208832024-04-17T17:23:49.797Z 14 Acked 1013 Read 8 Done Sent Done false
208842024-04-17T17:23:49.797Z 15 Acked 1014 Write 9 Done Sent Done false
208852024-04-17T17:23:49.797Z 16 Acked 1015 Flush 0 Done Sent Done false
208862024-04-17T17:23:49.797Z 17 Acked 1016 Write 10 Done Sent Done false
208872024-04-17T17:23:49.797Z 18 Acked 1017 Write 6 Done Sent Done false
208882024-04-17T17:23:49.797Z 19 Acked 1018 Write 1 Done Sent Done false
208892024-04-17T17:23:49.797Z 20 Acked 1019 Write 8 Done Sent Done false
208902024-04-17T17:23:49.797Z 21 Acked 1020 Write 10 Done Sent Done false
208912024-04-17T17:23:49.797Z 22 Acked 1021 Write 1 Done Sent Done false
208922024-04-17T17:23:49.797Z 23 Acked 1022 Read 6 Done Sent Done false
208932024-04-17T17:23:49.797Z 24 Acked 1023 Write 7 Done Sent Done false
208942024-04-17T17:23:49.797Z 25 Acked 1024 Write 3 Done Sent Done false
208952024-04-17T17:23:49.797Z 26 Acked 1025 Write 1 Done Sent Done false
208962024-04-17T17:23:49.797Z 27 Acked 1026 Read 6 Done Sent Done false
208972024-04-17T17:23:49.797Z 28 Acked 1027 Write 4 Done Sent Done false
208982024-04-17T17:23:49.797Z 29 Acked 1028 Read 5 Done Sent Done false
208992024-04-17T17:23:49.798Z 30 Acked 1029 Write 7 Sent Sent Sent false
209002024-04-17T17:23:49.798Z STATES DS:0 DS:1 DS:2 TOTAL
209012024-04-17T17:23:49.798Z New 0 0 0 0
209022024-04-17T17:23:49.798Z Sent 1 25 1 27
209032024-04-17T17:23:49.798Z Done 29 5 29 63
209042024-04-17T17:23:49.798Z Skipped 0 0 0 0
209052024-04-17T17:23:49.798Z Error 0 0 0 0
209062024-04-17T17:23:49.798ZLast Flush: 1015 0 1015
209072024-04-17T17:23:49.798ZDownstairs last five completed:
209082024-04-17T17:23:49.798ZUpstairs last five completed: 30 29 28 27 26
209092024-04-17T17:23:49.798Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:23:48.403312157Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1947}
209102024-04-17T17:23:49.798ZWrote out file "/tmp/test_repair_verify.data"
209112024-04-17T17:23:49.802Z
209122024-04-17T17:23:49.803ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
209132024-04-17T17:23:49.823ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
209142024-04-17T17:23:49.823Z 0 000-019 192 192 192 337 337 337 F F F
209152024-04-17T17:23:49.824Z 1 020-039 194 191 194 340 335 340 F F F <---
209162024-04-17T17:23:49.824Z 2 040-059 192 192 192 336 336 336 T F T <---
209172024-04-17T17:23:49.824Z 3 060-079 192 192 192 337 337 337 F F F
209182024-04-17T17:23:49.824Z 4 080-099 191 191 191 335 335 335 F F F
209192024-04-17T17:23:49.836Z 5 100-119 191 191 191 335 335 335 T F T <---
209202024-04-17T17:23:49.836Z 6 120-139 178 178 178 316 316 316 T F T <---
209212024-04-17T17:23:49.837Z 7 140-159 194 192 194 339 337 339 T T T <---
209222024-04-17T17:23:49.837Z 8 160-179 192 192 192 337 337 337 F F F
209232024-04-17T17:23:49.837Z 9 180-199 193 193 193 338 338 338 T F T <---
209242024-04-17T17:23:49.837Z 10 200-219 191 191 191 335 335 335 F F F
209252024-04-17T17:23:49.837Z 11 220-239 193 193 193 338 338 338 F F F
209262024-04-17T17:23:49.837Z 12 240-259 193 193 193 338 338 338 F F F
209272024-04-17T17:23:49.837Z 13 260-279 181 181 181 322 322 322 T F T <---
209282024-04-17T17:23:49.837Z 14 280-299 192 192 192 337 337 337 T F T <---
209292024-04-17T17:23:49.837Z 15 300-319 192 192 192 337 337 337 T F T <---
209302024-04-17T17:23:49.837Z 16 320-339 194 182 194 340 323 340 F F F <---
209312024-04-17T17:23:49.837Z 17 340-359 188 188 188 333 333 333 F F F
209322024-04-17T17:23:49.837Z 18 360-379 190 190 190 334 334 334 F F F
209332024-04-17T17:23:49.837Z 19 380-399 177 177 177 314 314 314 F F F
209342024-04-17T17:23:49.837Z 20 400-419 192 192 192 337 337 337 F F F
209352024-04-17T17:23:49.837Z 21 420-439 191 191 191 335 335 335 F F F
209362024-04-17T17:23:49.837Z 22 440-459 192 192 192 337 337 337 F F F
209372024-04-17T17:23:49.837Z 23 460-479 194 193 194 340 338 340 T F T <---
209382024-04-17T17:23:49.837Z 24 480-499 186 186 186 331 331 331 F F F
209392024-04-17T17:23:49.837Z 25 500-519 191 191 191 335 335 335 T F T <---
209402024-04-17T17:23:49.837Z 26 520-539 192 192 192 336 336 336 F F F
209412024-04-17T17:23:49.837Z 27 540-559 194 192 194 339 336 339 F T F <---
209422024-04-17T17:23:49.837Z 28 560-579 194 188 194 340 333 340 T F T <---
209432024-04-17T17:23:49.838Z 29 580-599 194 191 194 340 335 340 F F F <---
209442024-04-17T17:23:49.838ZMax gen: 194, Max flush: 340
209452024-04-17T17:23:49.838ZError: Difference in extent metadata found!
209462024-04-17T17:23:49.838ZA Difference in extent metadata is expected here
209472024-04-17T17:23:49.838ZOn loop 97
209482024-04-17T17:23:49.838Z
209492024-04-17T17:23:49.838ZVerifying data now
209502024-04-17T17:24:04.555ZNo change after verify
209512024-04-17T17:24:04.555ZLoop: 97 Downstairs dump after verify (and repair):
209522024-04-17T17:24:04.570ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
209532024-04-17T17:24:04.570Z 0 000-019 192 192 192 337 337 337 F F F
209542024-04-17T17:24:04.587Z 1 020-039 194 194 194 340 340 340 F F F
209552024-04-17T17:24:04.588Z 2 040-059 195 195 195 341 341 341 F F F
209562024-04-17T17:24:04.588Z 3 060-079 192 192 192 337 337 337 F F F
209572024-04-17T17:24:04.588Z 4 080-099 191 191 191 335 335 335 F F F
209582024-04-17T17:24:04.588Z 5 100-119 195 195 195 341 341 341 F F F
209592024-04-17T17:24:04.588Z 6 120-139 195 195 195 341 341 341 F F F
209602024-04-17T17:24:04.589Z 7 140-159 195 195 195 341 341 341 F F F
209612024-04-17T17:24:04.589Z 8 160-179 192 192 192 337 337 337 F F F
209622024-04-17T17:24:04.589Z 9 180-199 195 195 195 341 341 341 F F F
209632024-04-17T17:24:04.589Z 10 200-219 191 191 191 335 335 335 F F F
209642024-04-17T17:24:04.590Z 11 220-239 193 193 193 338 338 338 F F F
209652024-04-17T17:24:04.590Z 12 240-259 193 193 193 338 338 338 F F F
209662024-04-17T17:24:04.590Z 13 260-279 195 195 195 341 341 341 F F F
209672024-04-17T17:24:04.590Z 14 280-299 195 195 195 341 341 341 F F F
209682024-04-17T17:24:04.599Z 15 300-319 195 195 195 341 341 341 F F F
209692024-04-17T17:24:04.599Z 16 320-339 194 194 194 340 340 340 F F F
209702024-04-17T17:24:04.600Z 17 340-359 188 188 188 333 333 333 F F F
209712024-04-17T17:24:04.600Z 18 360-379 190 190 190 334 334 334 F F F
209722024-04-17T17:24:04.600Z 19 380-399 177 177 177 314 314 314 F F F
209732024-04-17T17:24:04.600Z 20 400-419 192 192 192 337 337 337 F F F
209742024-04-17T17:24:04.600Z 21 420-439 191 191 191 335 335 335 F F F
209752024-04-17T17:24:04.600Z 22 440-459 192 192 192 337 337 337 F F F
209762024-04-17T17:24:04.600Z 23 460-479 195 195 195 341 341 341 F F F
209772024-04-17T17:24:04.601Z 24 480-499 186 186 186 331 331 331 F F F
209782024-04-17T17:24:04.601Z 25 500-519 195 195 195 341 341 341 F F F
209792024-04-17T17:24:04.601Z 26 520-539 192 192 192 336 336 336 F F F
209802024-04-17T17:24:04.601Z 27 540-559 194 194 194 339 339 339 F F F
209812024-04-17T17:24:04.601Z 28 560-579 195 195 195 341 341 341 F F F
209822024-04-17T17:24:04.601Z 29 580-599 194 194 194 340 340 340 F F F
209832024-04-17T17:24:04.601ZMax gen: 195, Max flush: 341
209842024-04-17T17:24:04.601Z
209852024-04-17T17:24:04.602ZBegin loop 98
209862024-04-17T17:24:04.602ZDownstairs to restart: 1
209872024-04-17T17:24:04.622Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.200315159Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955}
209882024-04-17T17:24:04.622Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.200542043Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955}
209892024-04-17T17:24:04.622Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.200567645Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955}
209902024-04-17T17:24:04.623Z{"msg":"Crucible fcd324db-1d32-40b0-9ec0-6d69e46c2179 has session id: bfc5c709-d7bf-4118-ac7d-ba96a5841680","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.200581555Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
209912024-04-17T17:24:04.623Z{"msg":"Upstairs opts: Upstairs UUID: fcd324db-1d32-40b0-9ec0-6d69e46c2179, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.200636109Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
209922024-04-17T17:24:04.623Z{"msg":"Crucible stats registered with UUID: fcd324db-1d32-40b0-9ec0-6d69e46c2179","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.200677311Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
209932024-04-17T17:24:04.623ZCrucible runtime is spawned
209942024-04-17T17:24:04.623Z{"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.200795388Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955}
209952024-04-17T17:24:04.624Z{"msg":"fcd324db-1d32-40b0-9ec0-6d69e46c2179 active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.200911686Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
209962024-04-17T17:24:04.624Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.200940897Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"client":"0","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
209972024-04-17T17:24:04.624Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.200959349Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"client":"0","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
209982024-04-17T17:24:04.624Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.20098366Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"client":"1","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
209992024-04-17T17:24:04.625Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.201031933Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"client":"1","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210002024-04-17T17:24:04.625Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.201059135Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"client":"2","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210012024-04-17T17:24:04.625Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.201094907Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"client":"2","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210022024-04-17T17:24:04.625Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.201120608Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"":"io task","client":"1","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210032024-04-17T17:24:04.625Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.20115847Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"":"io task","client":"0","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210042024-04-17T17:24:04.625Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.201192803Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"":"io task","client":"2","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210052024-04-17T17:24:04.626Z{"msg":"ds_connection connected from Ok(127.0.0.1:64092)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.201353612Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"":"io task","client":"0","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210062024-04-17T17:24:04.626Z{"msg":"ds_connection connected from Ok(127.0.0.1:50002)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.201413316Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"":"io task","client":"2","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210072024-04-17T17:24:04.626Z{"msg":"ds_connection connected from Ok(127.0.0.1:57396)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.201464989Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"":"io task","client":"1","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210082024-04-17T17:24:04.626Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.202097017Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"client":"2","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210092024-04-17T17:24:04.626Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.20213528Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"client":"2","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210102024-04-17T17:24:04.626Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.202160941Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"client":"1","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210112024-04-17T17:24:04.627Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.202190713Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"client":"1","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210122024-04-17T17:24:04.627Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.202216825Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"client":"0","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210132024-04-17T17:24:04.627Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.202247976Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"client":"0","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210142024-04-17T17:24:04.627Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.202665392Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"client":"2","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210152024-04-17T17:24:04.627Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.202721895Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"client":"0","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210162024-04-17T17:24:04.628Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.202811791Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"client":"1","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210172024-04-17T17:24:04.628Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.203224076Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"client":"2","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210182024-04-17T17:24:04.628Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.203276059Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"client":"2","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210192024-04-17T17:24:04.628Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.203327672Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210202024-04-17T17:24:04.628Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.203365184Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"client":"0","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210212024-04-17T17:24:04.628Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.203393076Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"client":"0","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210222024-04-17T17:24:04.629Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.203427638Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210232024-04-17T17:24:04.629Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.20345849Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"client":"1","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210242024-04-17T17:24:04.629Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.203493572Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"client":"1","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210252024-04-17T17:24:04.629Z{"msg":"[0]R flush_numbers[0..12]: [337, 340, 341, 337, 335, 341, 341, 341, 337, 341, 335, 338]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.203528894Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"":"downstairs","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210262024-04-17T17:24:04.629Z{"msg":"[0]R generation[0..12]: [192, 194, 195, 192, 191, 195, 195, 195, 192, 195, 191, 193]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.203564036Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"":"downstairs","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210272024-04-17T17:24:04.629Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
210282024-04-17T17:24:04.630Z,"time":"2024-04-17T17:24:03.203601558Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"":"downstairs","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210292024-04-17T17:24:04.630Z{"msg":"[1]R flush_numbers[0..12]: [337, 340, 341, 337, 335, 341, 341, 341, 337, 341, 335, 338]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.203691594Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"":"downstairs","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210302024-04-17T17:24:04.630Z{"msg":"[1]R generation[0..12]: [192, 194, 195, 192, 191, 195, 195, 195, 192, 195, 191, 193]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.203731517Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"":"downstairs","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210312024-04-17T17:24:04.630Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.203762328Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"":"downstairs","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210322024-04-17T17:24:04.630Z{"msg":"[2]R flush_numbers[0..12]: [337, 340, 341, 337, 335, 341, 341, 341, 337, 341, 335, 338]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.203802741Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"":"downstairs","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210332024-04-17T17:24:04.630Z{"msg":"[2]R generation[0..12]: [192, 194, 195, 192, 191, 195, 195, 195, 192, 195, 191, 193]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.203843943Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"":"downstairs","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210342024-04-17T17:24:04.631Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.203885866Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"":"downstairs","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210352024-04-17T17:24:04.631Z{"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.203922318Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"":"downstairs","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210362024-04-17T17:24:04.631Z{"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
210372024-04-17T17:24:04.631Z,"time":"2024-04-17T17:24:03.20395771Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"":"downstairs","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210382024-04-17T17:24:04.632Z{"msg":"Next flush: 342","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.203994062Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"":"downstairs","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210392024-04-17T17:24:04.632Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.204026994Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"":"downstairs","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210402024-04-17T17:24:04.632Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.204042175Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210412024-04-17T17:24:04.632Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.204062337Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"client":"0","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210422024-04-17T17:24:04.632Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.204076417Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"client":"1","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210432024-04-17T17:24:04.632Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.204096139Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"client":"2","":"downstairs","session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210442024-04-17T17:24:04.632Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.204109729Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210452024-04-17T17:24:04.632Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.204128621Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210462024-04-17T17:24:04.632Z{"msg":"fcd324db-1d32-40b0-9ec0-6d69e46c2179 is now active with session: bfc5c709-d7bf-4118-ac7d-ba96a5841680","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.204141551Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210472024-04-17T17:24:04.632Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.204160132Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955,"session_id":"bfc5c709-d7bf-4118-ac7d-ba96a5841680"}
210482024-04-17T17:24:04.632Z{"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:03.204172363Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955}
210492024-04-17T17:24:04.632ZLoading write count information from file "/tmp/test_repair_verify.data"
210502024-04-17T17:24:04.632ZRun Repair workload
210512024-04-17T17:24:04.632Z01/30 Write block 479 len 12288 data: 19 20 18
210522024-04-17T17:24:04.632Z02/30 Read block 520 len 4096
210532024-04-17T17:24:04.632Z03/30 Read block 82 len 12288
210542024-04-17T17:24:04.632Z04/30 Write block 86 len 16384 data: 16 22 20 19
210552024-04-17T17:24:04.632Z05/30 Write block 330 len 36864 data: 10 13 12 12 14 16 20 21 20
210562024-04-17T17:24:04.633Z06/30 Read block 174 len 32768
210572024-04-17T17:24:04.633Z07/30 Write block 164 len 16384 data: 14 14 19 20
210582024-04-17T17:24:04.633Z08/30 Write block 46 len 20480 data: 14 13 10 13 12
210592024-04-17T17:24:04.633Z09/30 Read block 217 len 20480
210602024-04-17T17:24:04.633Z10/30 Write block 150 len 36864 data: 13 14 13 13 16 16 17 17 14
210612024-04-17T17:24:04.633Z11/30 Flush
210622024-04-17T17:24:04.633Z12/30 Write block 591 len 8192 data: 7 11
210632024-04-17T17:24:04.633Z13/30 Write block 209 len 20480 data: 18 17 18 18 19
210642024-04-17T17:24:04.642Z14/30 Write block 235 len 8192 data: 21 18
210652024-04-17T17:24:04.642Z15/30 Read block 152 len 4096
210662024-04-17T17:24:04.642Z16/30 Write block 410 len 4096 data: 12
210672024-04-17T17:24:04.642Z17/30 Read block 6 len 36864
210682024-04-17T17:24:04.642Z18/30 Read block 143 len 8192
210692024-04-17T17:24:04.642Z19/30 Read block 321 len 24576
210702024-04-17T17:24:04.642Z20/30 Read block 435 len 28672
210712024-04-17T17:24:04.642Z21/30 Write block 525 len 20480 data: 11 13 14 15 15
210722024-04-17T17:24:04.642Z22/30 Read block 263 len 12288
210732024-04-17T17:24:04.642Z23/30 Read block 13 len 32768
210742024-04-17T17:24:04.642Z24/30 Read block 543 len 16384
210752024-04-17T17:24:04.642Z25/30 Read block 438 len 28672
210762024-04-17T17:24:04.642Z26/30 Write block 299 len 24576 data: 14 13 14 13 16 17
210772024-04-17T17:24:04.642Z27/30 Read block 577 len 40960
210782024-04-17T17:24:04.643Z28/30 Write block 226 len 28672 data: 15 17 16 17 17 15 16
210792024-04-17T17:24:04.643Z29/30 Read block 163 len 28672
210802024-04-17T17:24:04.643Z30/30 Read block 327 len 8192
210812024-04-17T17:24:04.643Z----------------------------------------------------------------
210822024-04-17T17:24:04.643Z Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:30
210832024-04-17T17:24:04.643ZStates: Active Active Active
210842024-04-17T17:24:04.643ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
210852024-04-17T17:24:04.643Z 1 Acked 1000 Write 3 Done Done Done false
210862024-04-17T17:24:04.643Z 2 Acked 1001 Read 1 Done Done Done false
210872024-04-17T17:24:04.643Z 3 Acked 1002 Read 3 Done Done Done false
210882024-04-17T17:24:04.643Z 4 Acked 1003 Write 4 Done Sent Done false
210892024-04-17T17:24:04.643Z 5 Acked 1004 Write 9 Done Sent Done false
210902024-04-17T17:24:04.643Z 6 Acked 1005 Read 8 Done Sent Done false
210912024-04-17T17:24:04.643Z 7 Acked 1006 Write 4 Done Sent Done false
210922024-04-17T17:24:04.643Z 8 Acked 1007 Write 5 Done Sent Done false
210932024-04-17T17:24:04.644Z 9 Acked 1008 Read 5 Done Sent Done false
210942024-04-17T17:24:04.644Z 10 Acked 1009 Write 9 Done Sent Done false
210952024-04-17T17:24:04.644Z 11 Acked 1010 Flush 0 Done Sent Done false
210962024-04-17T17:24:04.644Z 12 Acked 1011 Write 2 Done Sent Done false
210972024-04-17T17:24:04.644Z 13 Acked 1012 Write 5 Done Sent Done false
210982024-04-17T17:24:04.644Z 14 Acked 1013 Write 2 Done Sent Done false
210992024-04-17T17:24:04.644Z 15 Acked 1014 Read 1 Done Sent Done false
211002024-04-17T17:24:04.644Z 16 Acked 1015 Write 1 Done Sent Done false
211012024-04-17T17:24:04.644Z 17 Acked 1016 Read 9 Done Sent Done false
211022024-04-17T17:24:04.644Z 18 Acked 1017 Read 2 Done Sent Done false
211032024-04-17T17:24:04.644Z 19 Acked 1018 Read 6 Done Sent Done false
211042024-04-17T17:24:04.644Z 20 Acked 1019 Read 7 Done Sent Done false
211052024-04-17T17:24:04.644Z 21 Acked 1020 Write 5 Done Sent Done false
211062024-04-17T17:24:04.645Z 22 Acked 1021 Read 3 Done Sent Done false
211072024-04-17T17:24:04.645Z 23 Acked 1022 Read 8 Done Sent Done false
211082024-04-17T17:24:04.645Z 24 Acked 1023 Read 4 Done Sent Done false
211092024-04-17T17:24:04.645Z 25 Acked 1024 Read 7 Done Sent Done false
211102024-04-17T17:24:04.645Z 26 Acked 1025 Write 6 Done Sent Done false
211112024-04-17T17:24:04.645Z 27 Acked 1026 Read 10 Done Sent Done false
211122024-04-17T17:24:04.645Z 28 Acked 1027 Write 7 Done Sent Done false
211132024-04-17T17:24:04.645Z 29 Acked 1028 Read 7 Done Sent Done false
211142024-04-17T17:24:04.645Z 30 Acked 1029 Read 2 Done Sent Done false
211152024-04-17T17:24:04.645Z STATES DS:0 DS:1 DS:2 TOTAL
211162024-04-17T17:24:04.645Z New 0 0 0 0
211172024-04-17T17:24:04.645Z Sent 0 27 0 27
211182024-04-17T17:24:04.645Z Done 30 3 30 63
211192024-04-17T17:24:04.645Z Skipped 0 0 0 0
211202024-04-17T17:24:04.646Z Error 0 0 0 0
211212024-04-17T17:24:04.646ZLast Flush: 1010 0 1010
211222024-04-17T17:24:04.646ZDownstairs last five completed:
211232024-04-17T17:24:04.646ZUpstairs last five completed: 30 29 28 27 26
211242024-04-17T17:24:04.646Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:24:03.232280547Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1955}
211252024-04-17T17:24:04.646ZWrote out file "/tmp/test_repair_verify.data"
211262024-04-17T17:24:04.646Z
211272024-04-17T17:24:04.646ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
211282024-04-17T17:24:04.656ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
211292024-04-17T17:24:04.656Z 0 000-019 192 192 192 337 337 337 F F F
211302024-04-17T17:24:04.656Z 1 020-039 194 194 194 340 340 340 F F F
211312024-04-17T17:24:04.656Z 2 040-059 196 195 196 342 341 342 F F F <---
211322024-04-17T17:24:04.656Z 3 060-079 192 192 192 337 337 337 F F F
211332024-04-17T17:24:04.656Z 4 080-099 196 191 196 342 335 342 F F F <---
211342024-04-17T17:24:04.656Z 5 100-119 195 195 195 341 341 341 F F F
211352024-04-17T17:24:04.656Z 6 120-139 195 195 195 341 341 341 F F F
211362024-04-17T17:24:04.656Z 7 140-159 196 195 196 342 341 342 F F F <---
211372024-04-17T17:24:04.656Z 8 160-179 196 192 196 342 337 342 F F F <---
211382024-04-17T17:24:04.657Z 9 180-199 195 195 195 341 341 341 F F F
211392024-04-17T17:24:04.657Z 10 200-219 191 191 191 335 335 335 T F T <---
211402024-04-17T17:24:04.657Z 11 220-239 193 193 193 338 338 338 T F T <---
211412024-04-17T17:24:04.657Z 12 240-259 193 193 193 338 338 338 F F F
211422024-04-17T17:24:04.657Z 13 260-279 195 195 195 341 341 341 F F F
211432024-04-17T17:24:04.658Z 14 280-299 195 195 195 341 341 341 T F T <---
211442024-04-17T17:24:04.658Z 15 300-319 195 195 195 341 341 341 T F T <---
211452024-04-17T17:24:04.658Z 16 320-339 196 194 196 342 340 342 F F F <---
211462024-04-17T17:24:04.658Z 17 340-359 188 188 188 333 333 333 F F F
211472024-04-17T17:24:04.658Z 18 360-379 190 190 190 334 334 334 F F F
211482024-04-17T17:24:04.658Z 19 380-399 177 177 177 314 314 314 F F F
211492024-04-17T17:24:04.659Z 20 400-419 192 192 192 337 337 337 T F T <---
211502024-04-17T17:24:04.659Z 21 420-439 191 191 191 335 335 335 F F F
211512024-04-17T17:24:04.659Z 22 440-459 192 192 192 337 337 337 F F F
211522024-04-17T17:24:04.659Z 23 460-479 196 195 196 342 341 342 F T F <---
211532024-04-17T17:24:04.659Z 24 480-499 196 186 196 342 331 342 F T F <---
211542024-04-17T17:24:04.659Z 25 500-519 195 195 195 341 341 341 F F F
211552024-04-17T17:24:04.660Z 26 520-539 192 192 192 336 336 336 T F T <---
211562024-04-17T17:24:04.660Z 27 540-559 194 194 194 339 339 339 F F F
211572024-04-17T17:24:04.660Z 28 560-579 195 195 195 341 341 341 F F F
211582024-04-17T17:24:04.660Z 29 580-599 194 194 194 340 340 340 T F T <---
211592024-04-17T17:24:04.660ZMax gen: 196, Max flush: 342
211602024-04-17T17:24:04.660ZError: Difference in extent metadata found!
211612024-04-17T17:24:04.661ZA Difference in extent metadata is expected here
211622024-04-17T17:24:04.661ZOn loop 98
211632024-04-17T17:24:04.662Z
211642024-04-17T17:24:04.667ZVerifying data now
211652024-04-17T17:24:19.312ZNo change after verify
211662024-04-17T17:24:19.312ZLoop: 98 Downstairs dump after verify (and repair):
211672024-04-17T17:24:19.328ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
211682024-04-17T17:24:19.328Z 0 000-019 192 192 192 337 337 337 F F F
211692024-04-17T17:24:19.335Z 1 020-039 194 194 194 340 340 340 F F F
211702024-04-17T17:24:19.335Z 2 040-059 196 196 196 342 342 342 F F F
211712024-04-17T17:24:19.335Z 3 060-079 192 192 192 337 337 337 F F F
211722024-04-17T17:24:19.335Z 4 080-099 196 196 196 342 342 342 F F F
211732024-04-17T17:24:19.335Z 5 100-119 195 195 195 341 341 341 F F F
211742024-04-17T17:24:19.335Z 6 120-139 195 195 195 341 341 341 F F F
211752024-04-17T17:24:19.335Z 7 140-159 196 196 196 342 342 342 F F F
211762024-04-17T17:24:19.335Z 8 160-179 196 196 196 342 342 342 F F F
211772024-04-17T17:24:19.335Z 9 180-199 195 195 195 341 341 341 F F F
211782024-04-17T17:24:19.335Z 10 200-219 197 197 197 343 343 343 F F F
211792024-04-17T17:24:19.335Z 11 220-239 197 197 197 343 343 343 F F F
211802024-04-17T17:24:19.335Z 12 240-259 193 193 193 338 338 338 F F F
211812024-04-17T17:24:19.335Z 13 260-279 195 195 195 341 341 341 F F F
211822024-04-17T17:24:19.335Z 14 280-299 197 197 197 343 343 343 F F F
211832024-04-17T17:24:19.335Z 15 300-319 197 197 197 343 343 343 F F F
211842024-04-17T17:24:19.335Z 16 320-339 196 196 196 342 342 342 F F F
211852024-04-17T17:24:19.335Z 17 340-359 188 188 188 333 333 333 F F F
211862024-04-17T17:24:19.336Z 18 360-379 190 190 190 334 334 334 F F F
211872024-04-17T17:24:19.336Z 19 380-399 177 177 177 314 314 314 F F F
211882024-04-17T17:24:19.336Z 20 400-419 197 197 197 343 343 343 F F F
211892024-04-17T17:24:19.336Z 21 420-439 191 191 191 335 335 335 F F F
211902024-04-17T17:24:19.336Z 22 440-459 192 192 192 337 337 337 F F F
211912024-04-17T17:24:19.336Z 23 460-479 196 196 196 342 342 342 F F F
211922024-04-17T17:24:19.336Z 24 480-499 196 196 196 342 342 342 F F F
211932024-04-17T17:24:19.336Z 25 500-519 195 195 195 341 341 341 F F F
211942024-04-17T17:24:19.336Z 26 520-539 197 197 197 343 343 343 F F F
211952024-04-17T17:24:19.336Z 27 540-559 194 194 194 339 339 339 F F F
211962024-04-17T17:24:19.336Z 28 560-579 195 195 195 341 341 341 F F F
211972024-04-17T17:24:19.336Z 29 580-599 197 197 197 343 343 343 F F F
211982024-04-17T17:24:19.336ZMax gen: 197, Max flush: 343
211992024-04-17T17:24:19.336Z
212002024-04-17T17:24:19.336ZBegin loop 99
212012024-04-17T17:24:19.336ZDownstairs to restart: 0
212022024-04-17T17:24:19.351Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.960731336Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963}
212032024-04-17T17:24:19.351Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9dd6771ac5213adb2b97af7a72c1375fe36d2636\",\n git_commit_timestamp: \"2024-04-17T16:00:39.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.960988532Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963}
212042024-04-17T17:24:19.351Z{"msg":"Upstairs <-> Downstairs Message Version: 6","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.961014413Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963}
212052024-04-17T17:24:19.358Z{"msg":"Crucible f9146ddb-8fcb-49c1-b24e-38ac36932d1a has session id: e6734b4c-e8a5-4457-8358-10ce2ded493d","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.961033354Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212062024-04-17T17:24:19.359Z{"msg":"Upstairs opts: Upstairs UUID: f9146ddb-8fcb-49c1-b24e-38ac36932d1a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.961090768Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212072024-04-17T17:24:19.359ZCrucible runtime is spawned
212082024-04-17T17:24:19.359Z{"msg":"Crucible stats registered with UUID: f9146ddb-8fcb-49c1-b24e-38ac36932d1a","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.961142821Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212092024-04-17T17:24:19.359Z{"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.961187024Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963}
212102024-04-17T17:24:19.359Z{"msg":"f9146ddb-8fcb-49c1-b24e-38ac36932d1a active request set","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.961300211Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212112024-04-17T17:24:19.359Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.961331603Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"client":"0","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212122024-04-17T17:24:19.359Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.961369415Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"client":"0","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212132024-04-17T17:24:19.359Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.961394847Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"client":"1","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212142024-04-17T17:24:19.359Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.961420678Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"client":"1","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212152024-04-17T17:24:19.359Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.96145606Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"client":"2","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212162024-04-17T17:24:19.359Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.961482412Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"client":"2","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212172024-04-17T17:24:19.359Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.961515714Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"":"io task","client":"0","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212182024-04-17T17:24:19.359Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.961564907Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"":"io task","client":"2","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212192024-04-17T17:24:19.359Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.961697695Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"":"io task","client":"1","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212202024-04-17T17:24:19.359Z{"msg":"ds_connection connected from Ok(127.0.0.1:64835)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.961734747Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"":"io task","client":"0","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212212024-04-17T17:24:19.359Z{"msg":"ds_connection connected from Ok(127.0.0.1:62378)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.961762309Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"":"io task","client":"2","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212222024-04-17T17:24:19.359Z{"msg":"ds_connection connected from Ok(127.0.0.1:42828)","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.961797461Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"":"io task","client":"1","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212232024-04-17T17:24:19.359Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.962466341Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"client":"1","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212242024-04-17T17:24:19.359Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.962507114Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"client":"1","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212252024-04-17T17:24:19.359Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.962591319Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"client":"2","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212262024-04-17T17:24:19.359Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.962681654Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"client":"2","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212272024-04-17T17:24:19.359Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.96277415Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"client":"0","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212282024-04-17T17:24:19.359Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.962821093Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"client":"0","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212292024-04-17T17:24:19.360Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.96310817Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"client":"2","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212302024-04-17T17:24:19.360Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.963246469Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"client":"1","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212312024-04-17T17:24:19.360Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.963511865Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"client":"0","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212322024-04-17T17:24:19.360Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.963719387Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"client":"2","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212332024-04-17T17:24:19.360Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.963754639Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"client":"2","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212342024-04-17T17:24:19.360Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.963769951Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212352024-04-17T17:24:19.360Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.963800152Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"client":"1","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212362024-04-17T17:24:19.360Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.963833724Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"client":"1","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212372024-04-17T17:24:19.360Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.963866386Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212382024-04-17T17:24:19.360Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.964023286Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"client":"0","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212392024-04-17T17:24:19.360Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.964060348Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"client":"0","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212402024-04-17T17:24:19.360Z{"msg":"[0]R flush_numbers[0..12]: [337, 340, 342, 337, 342, 341, 341, 342, 342, 341, 343, 343]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.964076489Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"":"downstairs","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212412024-04-17T17:24:19.360Z{"msg":"[0]R generation[0..12]: [192, 194, 196, 192, 196, 195, 195, 196, 196, 195, 197, 197]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.96409854Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"":"downstairs","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212422024-04-17T17:24:19.360Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.964113521Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"":"downstairs","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212432024-04-17T17:24:19.360Z{"msg":"[1]R flush_numbers[0..12]: [337, 340, 342, 337, 342, 341, 341, 342, 342, 341, 343, 343]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.964162424Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"":"downstairs","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212442024-04-17T17:24:19.360Z{"msg":"[1]R generation[0..12]: [192, 194, 196, 192, 196, 195, 195, 196, 196, 195, 197, 197]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.964176735Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"":"downstairs","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212452024-04-17T17:24:19.360Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
212462024-04-17T17:24:19.360Z,"time":"2024-04-17T17:24:17.964191216Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"":"downstairs","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212472024-04-17T17:24:19.360Z{"msg":"[2]R flush_numbers[0..12]: [337, 340, 342, 337, 342, 341, 341, 342, 342, 341, 343, 343]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.964217718Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"":"downstairs","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212482024-04-17T17:24:19.360Z{"msg":"[2]R generation[0..12]: [192, 194, 196, 192, 196, 195, 195, 196, 196, 195, 197, 197]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.964238289Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"":"downstairs","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212492024-04-17T17:24:19.360Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.96425274Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"":"downstairs","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212502024-04-17T17:24:19.360Z{"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.964272131Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"":"downstairs","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212512024-04-17T17:24:19.360Z{"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.964285832Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"":"downstairs","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212522024-04-17T17:24:19.361Z{"msg":"Next flush: 344","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.964304863Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"":"downstairs","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212532024-04-17T17:24:19.361Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.964318804Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"":"downstairs","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212542024-04-17T17:24:19.361Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.964338665Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212552024-04-17T17:24:19.361Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.964350946Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"client":"0","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212562024-04-17T17:24:19.361Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.964371527Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"client":"1","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212572024-04-17T17:24:19.361Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.964384798Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"client":"2","":"downstairs","session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212582024-04-17T17:24:19.361Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.964404329Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212592024-04-17T17:24:19.361Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.96441656Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212602024-04-17T17:24:19.361Z{"msg":"f9146ddb-8fcb-49c1-b24e-38ac36932d1a is now active with session: e6734b4c-e8a5-4457-8358-10ce2ded493d","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.964435741Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212612024-04-17T17:24:19.361Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.964447872Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963,"session_id":"e6734b4c-e8a5-4457-8358-10ce2ded493d"}
212622024-04-17T17:24:19.361Z{"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-04-17T17:24:17.964467173Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963}
212632024-04-17T17:24:19.361ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
212642024-04-17T17:24:19.361ZLoading write count information from file "/tmp/test_repair_verify.data"
212652024-04-17T17:24:19.361ZRun Repair workload
212662024-04-17T17:24:19.361Z01/30 Write block 8 len 8192 data: 16 15
212672024-04-17T17:24:19.365Z02/30 Write block 186 len 20480 data: 13 19 21 20 17
212682024-04-17T17:24:19.365Z03/30 Write block 103 len 20480 data: 15 15 16 15 15
212692024-04-17T17:24:19.365Z04/30 Write block 290 len 36864 data: 20 21 18 21 19 14 11 11 12
212702024-04-17T17:24:19.365Z05/30 Read block 63 len 12288
212712024-04-17T17:24:19.365Z06/30 Write block 582 len 12288 data: 13 15 16
212722024-04-17T17:24:19.365Z07/30 Write block 497 len 24576 data: 14 17 18 18 19 17
212732024-04-17T17:24:19.365Z08/30 Write block 52 len 8192 data: 14 15
212742024-04-17T17:24:19.365Z09/30 Read block 589 len 16384
212752024-04-17T17:24:19.365Z10/30 Write block 489 len 40960 data: 15 15 13 14 13 14 16 14 15 18
212762024-04-17T17:24:19.366Z11/30 Read block 308 len 32768
212772024-04-17T17:24:19.370Z12/30 Read block 146 len 40960
212782024-04-17T17:24:19.370Z13/30 Read block 378 len 20480
212792024-04-17T17:24:19.370Z14/30 Read block 34 len 28672
212802024-04-17T17:24:19.370Z15/30 Read block 567 len 24576
212812024-04-17T17:24:19.370Z16/30 Flush
212822024-04-17T17:24:19.374Z17/30 Write block 439 len 40960 data: 21 17 16 18 18 16 11 12 10 12
212832024-04-17T17:24:19.374Z18/30 Write block 164 len 36864 data: 15 15 20 21 18 17 16 18 16
212842024-04-17T17:24:19.380Z19/30 Write block 209 len 24576 data: 19 18 19 19 20 16
212852024-04-17T17:24:19.380Z20/30 Read block 181 len 12288
212862024-04-17T17:24:19.380Z21/30 Write block 415 len 8192 data: 15 18
212872024-04-17T17:24:19.380Z22/30 Write block 298 len 20480 data: 13 15 14 15 14
212882024-04-17T17:24:19.380Z23/30 Flush
212892024-04-17T17:24:19.383Z24/30 Write block 231 len 36864 data: 16 17 15 16 22 19 15 13 14
212902024-04-17T17:24:19.384Z25/30 Write block 337 len 24576 data: 22 21 21 19 16 17
212912024-04-17T17:24:19.384Z26/30 Read block 6 len 32768
212922024-04-17T17:24:19.390Z27/30 Read block 222 len 12288
212932024-04-17T17:24:19.390Z28/30 Read block 448 len 24576
212942024-04-17T17:24:19.390Z29/30 Read block 242 len 28672
212952024-04-17T17:24:19.390Z30/30 Write block 552 len 20480 data: 22 17 17 18 13
212962024-04-17T17:24:19.390Z----------------------------------------------------------------
212972024-04-17T17:24:19.390Z Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:30
212982024-04-17T17:24:19.391ZStates: Active Active Active
212992024-04-17T17:24:19.391ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
213002024-04-17T17:24:19.391Z 1 Acked 1000 Write 2 Sent Done Done false
213012024-04-17T17:24:19.391Z 2 Acked 1001 Write 5 Sent Done Done false
213022024-04-17T17:24:19.391Z 3 Acked 1002 Write 5 Sent Done Done false
213032024-04-17T17:24:19.391Z 4 Acked 1003 Write 9 Sent Done Done false
213042024-04-17T17:24:19.391Z 5 Acked 1004 Read 3 Sent Done Done false
213052024-04-17T17:24:19.391Z 6 Acked 1005 Write 3 Sent Done Done false
213062024-04-17T17:24:19.391Z 7 Acked 1006 Write 6 Sent Done Done false
213072024-04-17T17:24:19.391Z 8 Acked 1007 Write 2 Sent Done Done false
213082024-04-17T17:24:19.391Z 9 Acked 1008 Read 4 Sent Done Done false
213092024-04-17T17:24:19.391Z 10 Acked 1009 Write 10 Sent Done Done false
213102024-04-17T17:24:19.391Z 11 Acked 1010 Read 8 Sent Done Done false
213112024-04-17T17:24:19.391Z 12 Acked 1011 Read 10 Sent Done Done false
213122024-04-17T17:24:19.391Z 13 Acked 1012 Read 5 Sent Done Done false
213132024-04-17T17:24:19.391Z 14 Acked 1013 Read 7 Sent Done Done false
213142024-04-17T17:24:19.391Z 15 Acked 1014 Read 6 Sent Done Done false
213152024-04-17T17:24:19.391Z 16 Acked 1015 Flush 0 Sent Done Done false
213162024-04-17T17:24:19.391Z 17 Acked 1016 Write 10 Sent Done Done false
213172024-04-17T17:24:19.391Z 18 Acked 1017 Write 9 Sent Done Done false
213182024-04-17T17:24:19.391Z 19 Acked 1018 Write 6 Sent Done Done false
213192024-04-17T17:24:19.391Z 20 Acked 1019 Read 3 Sent Done Done false
213202024-04-17T17:24:19.391Z 21 Acked 1020 Write 2 Sent Done Done false
213212024-04-17T17:24:19.391Z 22 Acked 1021 Write 5 Sent Done Done false
213222024-04-17T17:24:19.392Z 23 Acked 1022 Flush 0 Sent Done Done false
213232024-04-17T17:24:19.392Z 24 Acked 1023 Write 9 Sent Done Done false
213242024-04-17T17:24:19.392Z 25 Acked 1024 Write 6 Sent Done Done false
213252024-04-17T17:24:19.392Z 26 Acked 1025 Read 8 Sent Done Done false
213262024-04-17T17:24:19.392Z 27 Acked 1026 Read 3 Sent Done Done false
213272024-04-17T17:24:19.392Z 28 Acked 1027 Read 6 Sent Done Done false
213282024-04-17T17:24:19.392Z 29 Acked 1028 Read 7 Sent Done Done false
213292024-04-17T17:24:19.392Z 30 Acked 1029 Write 5 Sent Sent Sent false
213302024-04-17T17:24:19.392Z STATES DS:0 DS:1 DS:2 TOTAL
213312024-04-17T17:24:19.392Z New 0 0 0 0
213322024-04-17T17:24:19.392Z Sent 30 1 1 32
213332024-04-17T17:24:19.392Z Done 0 29 29 58
213342024-04-17T17:24:19.392Z Skipped 0 0 0 0
213352024-04-17T17:24:19.392Z Error 0 0 0 0
213362024-04-17T17:24:19.392ZLast Flush: 0 1022 1022
213372024-04-17T17:24:19.392ZDownstairs last five completed:
213382024-04-17T17:24:19.392ZUpstairs last five completed: 30 29 28 27 26
213392024-04-17T17:24:19.392Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-04-17T17:24:17.998140034Z","hostname":"w-01HVPGDYTTAYG62ZSA4VY7JVXQ","pid":1963}
213402024-04-17T17:24:19.398ZWrote out file "/tmp/test_repair_verify.data"
213412024-04-17T17:24:19.398Z
213422024-04-17T17:24:19.398ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
213432024-04-17T17:24:19.410ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
213442024-04-17T17:24:19.410Z 0 000-019 192 198 198 337 344 344 F F F <---
213452024-04-17T17:24:19.410Z 1 020-039 194 194 194 340 340 340 F F F
213462024-04-17T17:24:19.410Z 2 040-059 196 198 198 342 344 344 F F F <---
213472024-04-17T17:24:19.411Z 3 060-079 192 192 192 337 337 337 F F F
213482024-04-17T17:24:19.411Z 4 080-099 196 196 196 342 342 342 F F F
213492024-04-17T17:24:19.415Z 5 100-119 195 198 198 341 344 344 F F F <---
213502024-04-17T17:24:19.415Z 6 120-139 195 195 195 341 341 341 F F F
213512024-04-17T17:24:19.415Z 7 140-159 196 196 196 342 342 342 F F F
213522024-04-17T17:24:19.415Z 8 160-179 196 198 198 342 345 345 F F F <---
213532024-04-17T17:24:19.415Z 9 180-199 195 198 198 341 344 344 F F F <---
213542024-04-17T17:24:19.415Z 10 200-219 197 198 198 343 345 345 F F F <---
213552024-04-17T17:24:19.415Z 11 220-239 197 197 197 343 343 343 F T T <---
213562024-04-17T17:24:19.415Z 12 240-259 193 193 193 338 338 338 F F F
213572024-04-17T17:24:19.415Z 13 260-279 195 195 195 341 341 341 F F F
213582024-04-17T17:24:19.415Z 14 280-299 197 198 198 343 345 345 F F F <---
213592024-04-17T17:24:19.415Z 15 300-319 197 198 198 343 345 345 F F F <---
213602024-04-17T17:24:19.415Z 16 320-339 196 196 196 342 342 342 F T T <---
213612024-04-17T17:24:19.415Z 17 340-359 188 188 188 333 333 333 F T T <---
213622024-04-17T17:24:19.415Z 18 360-379 190 190 190 334 334 334 F F F
213632024-04-17T17:24:19.415Z 19 380-399 177 177 177 314 314 314 F F F
213642024-04-17T17:24:19.415Z 20 400-419 197 198 198 343 345 345 F F F <---
213652024-04-17T17:24:19.415Z 21 420-439 191 198 198 335 345 345 F F F <---
213662024-04-17T17:24:19.415Z 22 440-459 192 198 198 337 345 345 F F F <---
213672024-04-17T17:24:19.416Z 23 460-479 196 196 196 342 342 342 F F F
213682024-04-17T17:24:19.416Z 24 480-499 196 198 198 342 344 344 F F F <---
213692024-04-17T17:24:19.416Z 25 500-519 195 198 198 341 344 344 F F F <---
213702024-04-17T17:24:19.416Z 26 520-539 197 197 197 343 343 343 F F F
213712024-04-17T17:24:19.416Z 27 540-559 194 194 194 339 339 339 F T T <---
213722024-04-17T17:24:19.416Z 28 560-579 195 195 195 341 341 341 F F F
213732024-04-17T17:24:19.416Z 29 580-599 197 198 198 343 344 344 F F F <---
213742024-04-17T17:24:19.416ZMax gen: 198, Max flush: 345
213752024-04-17T17:24:19.416ZError: Difference in extent metadata found!
213762024-04-17T17:24:19.416ZA Difference in extent metadata is expected here
213772024-04-17T17:24:19.416ZOn loop 99
213782024-04-17T17:24:19.416Z
213792024-04-17T17:24:19.421ZVerifying data now
213802024-04-17T17:24:24.240ZNo change after verify
213812024-04-17T17:24:24.240ZLoop: 99 Downstairs dump after verify (and repair):
213822024-04-17T17:24:24.255ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
213832024-04-17T17:24:24.255Z 0 000-019 198 198 198 344 344 344 F F F
213842024-04-17T17:24:24.255Z 1 020-039 194 194 194 340 340 340 F F F
213852024-04-17T17:24:24.255Z 2 040-059 198 198 198 344 344 344 F F F
213862024-04-17T17:24:24.255Z 3 060-079 192 192 192 337 337 337 F F F
213872024-04-17T17:24:24.255Z 4 080-099 196 196 196 342 342 342 F F F
213882024-04-17T17:24:24.255Z 5 100-119 198 198 198 344 344 344 F F F
213892024-04-17T17:24:24.259Z 6 120-139 195 195 195 341 341 341 F F F
213902024-04-17T17:24:24.259Z 7 140-159 196 196 196 342 342 342 F F F
213912024-04-17T17:24:24.259Z 8 160-179 198 198 198 345 345 345 F F F
213922024-04-17T17:24:24.259Z 9 180-199 198 198 198 344 344 344 F F F
213932024-04-17T17:24:24.259Z 10 200-219 198 198 198 345 345 345 F F F
213942024-04-17T17:24:24.259Z 11 220-239 199 199 199 346 346 346 F F F
213952024-04-17T17:24:24.259Z 12 240-259 193 193 193 338 338 338 F F F
213962024-04-17T17:24:24.259Z 13 260-279 195 195 195 341 341 341 F F F
213972024-04-17T17:24:24.260Z 14 280-299 198 198 198 345 345 345 F F F
213982024-04-17T17:24:24.260Z 15 300-319 198 198 198 345 345 345 F F F
213992024-04-17T17:24:24.260Z 16 320-339 199 199 199 346 346 346 F F F
214002024-04-17T17:24:24.260Z 17 340-359 199 199 199 346 346 346 F F F
214012024-04-17T17:24:24.260Z 18 360-379 190 190 190 334 334 334 F F F
214022024-04-17T17:24:24.260Z 19 380-399 177 177 177 314 314 314 F F F
214032024-04-17T17:24:24.260Z 20 400-419 198 198 198 345 345 345 F F F
214042024-04-17T17:24:24.260Z 21 420-439 198 198 198 345 345 345 F F F
214052024-04-17T17:24:24.260Z 22 440-459 198 198 198 345 345 345 F F F
214062024-04-17T17:24:24.260Z 23 460-479 196 196 196 342 342 342 F F F
214072024-04-17T17:24:24.260Z 24 480-499 198 198 198 344 344 344 F F F
214082024-04-17T17:24:24.260Z 25 500-519 198 198 198 344 344 344 F F F
214092024-04-17T17:24:24.260Z 26 520-539 197 197 197 343 343 343 F F F
214102024-04-17T17:24:24.260Z 27 540-559 199 199 199 346 346 346 F F F
214112024-04-17T17:24:24.260Z 28 560-579 195 195 195 341 341 341 F F F
214122024-04-17T17:24:24.260Z 29 580-599 198 198 198 344 344 344 F F F
214132024-04-17T17:24:24.260ZMax gen: 199, Max flush: 346
214142024-04-17T17:24:24.260Z13:38 Test duration
214152024-04-17T17:24:24.260ZTest completed
214162024-04-17T17:24:24.265Z
214172024-04-17T17:24:24.265Zreal 13:38.827452637
214182024-04-17T17:24:24.265Zuser 1:30.308891470
214192024-04-17T17:24:24.265Zsys 19.011143360
214202024-04-17T17:24:24.265Ztrap 0.085503695
214212024-04-17T17:24:24.265Ztflt 0.052402137
214222024-04-17T17:24:24.265Zdflt 0.048076971
214232024-04-17T17:24:24.265Zkflt 0.004868176
214242024-04-17T17:24:24.266Zlock 19:45:46.586914948
214252024-04-17T17:24:24.266Zslp 1:05:41.044047399
214262024-04-17T17:24:24.266Zlat 7.132372212
214272024-04-17T17:24:24.266Zstop 0.052716935
214282024-04-17T17:24:24.266Zprocess exited: duration 826193 ms, exit code 0
 
214292024-04-17T17:24:24.278Zfound 4 output files
214302024-04-17T17:24:24.278Zuploading: /tmp/test_repair_ds8810.txt (28676 bytes)
214312024-04-17T17:24:25.293Zuploaded: /tmp/test_repair_ds8810.txt
214322024-04-17T17:24:25.294Zuploading: /tmp/test_repair_ds8820.txt (57024 bytes)
214332024-04-17T17:24:26.316Zuploaded: /tmp/test_repair_ds8820.txt
214342024-04-17T17:24:26.325Zuploading: /tmp/test_repair_ds8830.txt (78877 bytes)
214352024-04-17T17:24:27.339Zuploaded: /tmp/test_repair_ds8830.txt
214362024-04-17T17:24:27.339Zuploading: /tmp/test_repair_out.txt (86572 bytes)
214372024-04-17T17:24:28.357Zuploaded: /tmp/test_repair_out.txt